diff --git a/.idea/dataSources.xml b/.idea/dataSources.xml index 77a0c08..6cd4c71 100644 --- a/.idea/dataSources.xml +++ b/.idea/dataSources.xml @@ -29,5 +29,12 @@ jdbc:sqlite:C:\Users\Frede\IdeaProjects\db_interactor\local.db $ProjectFileDir$ + + sqlite.xerial + true + org.sqlite.JDBC + jdbc:sqlite:$PROJECT_DIR$/local.db + $ProjectFileDir$ + \ No newline at end of file diff --git a/.idea/database_interacter.iml b/.idea/database_interacter.iml index 8b76679..880d55c 100644 --- a/.idea/database_interacter.iml +++ b/.idea/database_interacter.iml @@ -4,7 +4,7 @@ - + \ No newline at end of file diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml index 6787940..b910d22 100644 --- a/.idea/sqldialects.xml +++ b/.idea/sqldialects.xml @@ -1,6 +1,7 @@ + \ No newline at end of file diff --git a/checkpoint/Checkpoint.py b/checkpoint/Checkpoint.py new file mode 100644 index 0000000..e058df7 --- /dev/null +++ b/checkpoint/Checkpoint.py @@ -0,0 +1,57 @@ +from dataclasses import dataclass +from enum import Enum + + +class CheckpointStatus(Enum): + STARTED = "STARTED" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + +@dataclass +class CheckpointMetadata: + _metadata: str + _type: str + + def __init__(self, metadata: str, _type: str): + self._metadata = metadata + self._type = _type + + @property + def metadata(self): + return self._metadata + + @property + def type(self): + return self._type + + +@dataclass +class Checkpoint: + _name: str + _hash: str + _status: CheckpointStatus + _id: int + + + def __init__(self, id: int, name: str, hash: str, status: CheckpointStatus): + self._name = name + self._hash = hash + self._status = status + self._id = id + + + @property + def name(self): + return self._name + + @property + def hash(self): + return self._hash + + @property + def status(self): + return self._status + + @property + def id(self): + return self._id \ No newline at end of file diff --git a/checkpoint/CheckpointService.py b/checkpoint/CheckpointService.py new file mode 100644 index 0000000..76db835 --- /dev/null +++ b/checkpoint/CheckpointService.py @@ -0,0 +1,63 @@ +import sqlite3 + +from checkpoint.Checkpoint import Checkpoint, CheckpointMetadata, CheckpointStatus + + +class CheckpointService: + _table: str + _database: str + _connection: str + _conn: sqlite3.Connection + _insert_metadata_query = """ INSERT INTO checkpoint_metadata (checkpoint_id, metadata, type) VALUES (?, ?, ?) """ + + def __init__(self, database: str = "local.db"): + self._database = database + self._conn = sqlite3.connect(database) + + self._initialize_table() + + def _initialize_table(self): + create_checkpoint_table_query = """ + CREATE TABLE IF NOT EXISTS checkpoint ( + id INTEGER primary key autoincrement, + name TEXT not null, + hash TEXT not null unique, + status TEXT not null + ); + """ + + create_checkpoint_metadata_table_query = """ + CREATE TABLE IF NOT EXISTS checkpoint_metadata ( + id INTEGER primary key autoincrement, + checkpoint_id INTEGER NOT NULL, + metadata TEXT NOT NULL, + type TEXT not null, + constraint checkpoint_metadata_checkpoint_fk + foreign key (checkpoint_id) references checkpoint (id) + ); + """ + + self._conn.execute(create_checkpoint_table_query) + self._conn.execute(create_checkpoint_metadata_table_query) + self._conn.commit() + + def start_checkpoint(self, name, hash, metadata: CheckpointMetadata) -> Checkpoint: + insert_checkpoint_query = """INSERT INTO checkpoint (name, hash, status) VALUES (?, ?, ?)""" + + checkpoint_id = self._conn.execute(insert_checkpoint_query, (name, hash, CheckpointStatus.STARTED)).lastrowid + self._conn.execute(self._insert_metadata_query, (checkpoint_id, metadata.metadata, metadata.type)) + self._conn.commit() + + return Checkpoint(checkpoint_id, name, hash, CheckpointStatus.STARTED) + + def end_checkpoint(self, checkpoint_id, metadata: CheckpointMetadata): + update_checkpoint_query = """UPDATE checkpoint SET status = ? WHERE checkpoint_id = ?""" + self._conn.execute(update_checkpoint_query, (CheckpointStatus.COMPLETED, checkpoint_id)) + self._conn.execute(self._insert_metadata_query, (checkpoint_id, metadata.metadata, metadata.type)) + self._conn.commit() + + def get_checkpoint(self, checkpoint_id: int, hash: str): + ... + + def delete_checkpoint(self): + ... \ No newline at end of file diff --git a/checkpoint/__init__.py b/checkpoint/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/database/db_adapter.py b/database/db_adapter.py index 55e887d..dbea5d1 100644 --- a/database/db_adapter.py +++ b/database/db_adapter.py @@ -1,6 +1,7 @@ import logging import os import time +from pathlib import Path import pandas as pd import sqlalchemy as sq @@ -62,6 +63,7 @@ class DBAdapter: conn.execute(sq.text(f"alter session set current_schema = {schema}")) case DatabaseType.PSQL: conn.execute(sq.text(f"set schema '{schema}'")) + conn.execute(sq.text(f"set search_path = public")) case _: raise Exception( f'Database type {self._database_config.type} is not supported for readonly transactions') @@ -73,7 +75,9 @@ class DBAdapter: pass return f'{self._output_folder}{filename}' - def _export_to_file(self, export_type, output_file_name, result, query_parameter: QueryParameters = None): + def _export_to_file(self, export_type: ExportType, output_file_name: str, result: pd.DataFrame, query_parameter: QueryParameters = None): + os.makedirs(os.path.dirname(output_file_name + ".txt"), exist_ok=True) + match export_type: case ExportType.CSV: output_file_name += '.csv' @@ -172,7 +176,7 @@ class DBAdapter: self._logger.info( f'({query_file_index + 1}/{len(municipalities)}) Starting to process query with title: {query_parameter.title}') - if not len(queries) != 1: + if not len(queries) == 1: self._logger.error(f'Query file {query_parameter.title} failed due to multiple queries') raise Exception(f'Query file {query_parameter.title} failed due to multiple queries') diff --git a/main.py b/main.py index 78bea0f..1fd79ff 100644 --- a/main.py +++ b/main.py @@ -1,7 +1,9 @@ from config.Config import Config +from database.QueryParameters import QueryParameters from database.db_adapter import DBAdapter from keepass.Keepass import KeePass from logger.logger import init_logger +from models.ExportType import ExportType from models.Municipality import Municipality logger = init_logger() @@ -11,109 +13,12 @@ config = Config() keepass = KeePass(config.kee_pass, logger) municipalities = [ - #Municipality('Central common schema', '00000000', 'KY_CENTRAL', 'KY_CENTRAL'), - Municipality('Odense Kommune', '35209115', 'KY_0461', '461'), - Municipality('Svendborg Kommune', '29189730', 'KY_0479', '479'), - Municipality('Nordfyns Kommune', '29188947', 'KY_0480', '480'), - Municipality('Langeland Kommune', '29188955', 'KY_0482', '482'), - Municipality('Ærø Kommune', '28856075', 'KY_0492', '492'), - Municipality('Haderslev Kommune', '29189757', 'KY_0510', '510'), - Municipality('Billund Kommune', '29189765', 'KY_0530', '530'), - Municipality('Sønderborg Kommune', '29189773', 'KY_0540', '540'), - Municipality('Tønder Kommune', '29189781', 'KY_0550', '550'), - Municipality('Esbjerg Kommune', '29189803', 'KY_0561', '561'), - Municipality('Fanø Kommune', '31210917', 'KY_0563', '563'), - Municipality('Varde Kommune', '29189811', 'KY_0573', '573'), - Municipality('Vejen Kommune', '29189838', 'KY_0575', '575'), - Municipality('Aabenraa Kommune', '29189854', 'KY_0580', '580'), - Municipality('Fredericia Kommune', '69116418', 'KY_0607', '607'), - Municipality('Horsens Kommune', '29189889', 'KY_0615', '615'), - Municipality('Kolding Kommune', '29189897', 'KY_0621', '621'), - Municipality('Vejle Kommune', '29189900', 'KY_0630', '630'), - Municipality('Herning Kommune', '29189919', 'KY_0657', '657'), - Municipality('Holstebro Kommune', '29189927', 'KY_0661', '661'), - Municipality('Lemvig Kommune', '29189935', 'KY_0665', '665'), - Municipality('Struer Kommune', '29189951', 'KY_0671', '671'), - Municipality('Syddjurs Kommune', '29189978', 'KY_0706', '706'), - Municipality('Norddjurs Kommune', '29189986', 'KY_0707', '707'), - Municipality('Favrskov Kommune', '29189714', 'KY_0710', '710'), - Municipality('Odder Kommune', '32264328', 'KY_0727', '727'), - Municipality('Randers Kommune', '29189668', 'KY_0730', '730'), - Municipality('Silkeborg Kommune', '29189641', 'KY_0740', '740'), - Municipality('Samsø Kommune', '23795515', 'KY_0741', '741'), - Municipality('Skanderborg Kommune', '29189633', 'KY_0746', '746'), - Municipality('Aarhus Kommune', '55133018', 'KY_0751', '751'), - Municipality('Ikast-Brande Kommune', '29189617', 'KY_0756', '756'), - Municipality('Ringkøbing-Skjern Kommune', '29189609', 'KY_0760', '760'), - Municipality('Hedensted Kommune', '29189587', 'KY_0766', '766'), - Municipality('Morsø Kommune', '41333014', 'KY_0773', '773'), - Municipality('Skive Kommune', '29189579', 'KY_0779', '779'), - Municipality('Thisted Kommune', '29189560', 'KY_0787', '787'), - Municipality('Viborg Kommune', '29189846', 'KY_0791', '791'), - Municipality('Brønderslev Kommune', '29189501', 'KY_0810', '810'), - Municipality('Frederikshavn Kommune', '29189498', 'KY_0813', '813'), - Municipality('Vesthimmerlands Kommune', '29189471', 'KY_0820', '820'), - Municipality('Læsø Kommune', '45973328', 'KY_0825', '825'), - Municipality('Rebild Kommune', '29189463', 'KY_0840', '840'), - Municipality('Mariagerfjord Kommune', '29189455', 'KY_0846', '846'), - Municipality('Jammerbugt Kommune', '29189439', 'KY_0849', '849'), - Municipality('Aalborg Kommune', '29189420', 'KY_0851', '851'), - Municipality('Hjørring Kommune', '29189382', 'KY_0860', '860'), - Municipality('Københavns Kommune', '64942212', 'KY_0101', '101'), - Municipality('Frederiksberg Kommune', '11259979', 'KY_0147', '147'), - Municipality('Ballerup Kommune', '58271713', 'KY_0151', '151'), - Municipality('Brøndby Kommune', '65113015', 'KY_0153', '153'), - Municipality('Dragør Kommune', '12881517', 'KY_0155', '155'), - Municipality('Gentofte Kommune', '19438414', 'KY_0157', '157'), - Municipality('Gladsaxe Kommune', '62761113', 'KY_0159', '159'), - Municipality('Glostrup Kommune', '65120119', 'KY_0161', '161'), - Municipality('Herlev Kommune', '63640719', 'KY_0163', '163'), - Municipality('Albertslund Kommune', '66137112', 'KY_0165', '165'), - Municipality('Hvidovre Kommune', '55606617', 'KY_0167', '167'), - Municipality('Høje Taastrup Kommune', '19501817', 'KY_0169', '169'), - Municipality('Lyngby-Taarbæk Kommune', '11715311', 'KY_0173', '173'), - Municipality('Rødovre Kommune', '65307316', 'KY_0175', '175'), - Municipality('Ishøj Kommune', '11931316', 'KY_0183', '183'), - Municipality('Tårnby Kommune', '20310413', 'KY_0185', '185'), - Municipality('Vallensbæk Kommune', '19583910', 'KY_0187', '187'), - Municipality('Furesø Kommune', '29188327', 'KY_0190', '190'), - Municipality('Allerød Kommune', '60183112', 'KY_0201', '201'), - Municipality('Fredensborg Kommune', '29188335', 'KY_0210', '210'), - Municipality('Helsingør Kommune', '64502018', 'KY_0217', '217'), - Municipality('Hillerød Kommune', '29189366', 'KY_0219', '219'), - Municipality('Hørsholm Kommune', '70960516', 'KY_0223', '223'), - Municipality('Rudersdal Kommune', '29188378', 'KY_0230', '230'), - Municipality('Egedal Kommune', '29188386', 'KY_0240', '240'), - Municipality('Frederikssund Kommune', '29189129', 'KY_0250', '250'), - Municipality('Greve Kommune', '44023911', 'KY_0253', '253'), - Municipality('Køge Kommune', '29189374', 'KY_0259', '259'), - Municipality('Halsnæs Kommune', '29188416', 'KY_0260', '260'), - Municipality('Roskilde Kommune', '29189404', 'KY_0265', '265'), - Municipality('Solrød Kommune', '68534917', 'KY_0269', '269'), - Municipality('Gribskov Kommune', '29188440', 'KY_0270', '270'), - Municipality('Odsherred Kommune', '29188459', 'KY_0306', '306'), - Municipality('Holbæk Kommune', '29189447', 'KY_0316', '316'), - Municipality('Faxe Kommune', '29188475', 'KY_0320', '320'), - Municipality('Kalundborg Kommune', '29189595', 'KY_0326', '326'), - Municipality('Ringsted Kommune', '18957981', 'KY_0329', '329'), - Municipality('Slagelse Kommune', '29188505', 'KY_0330', '330'), - Municipality('Stevns Kommune', '29208654', 'KY_0336', '336'), - Municipality('Sorø Kommune', '29189994', 'KY_0340', '340'), - Municipality('Lejre Kommune', '29188548', 'KY_0350', '350'), - Municipality('Lolland Kommune', '29188572', 'KY_0360', '360'), - Municipality('Næstved Kommune', '29189625', 'KY_0370', '370'), - Municipality('Guldborgsund Kommune', '29188599', 'KY_0376', '376'), - Municipality('Vordingborg Kommune', '29189676', 'KY_0390', '390'), - Municipality('Bornholms Regionskommune', '26696348', 'KY_0400', '400'), - Municipality('Middelfart Kommune', '29189684', 'KY_0410', '410'), - Municipality('Assens Kommune', '29189692', 'KY_0420', '420'), - Municipality('Faaborg-Midtfyn Kommune', '29188645', 'KY_0430', '430'), - Municipality('Kerteminde Kommune', '29189706', 'KY_0440', '440'), - Municipality('Nyborg Kommune', '29189722', 'KY_0450', '450'), + Municipality("op_test", None, "OP", "test") ] +parameter = QueryParameters(title="Test rapport", input_path="query.sql", export_type=ExportType.XML) db_adapter = DBAdapter(keepass, config.database, logger) -db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities=municipalities) +db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities=municipalities, query_parameter=parameter) # db_adapter.run_sql_file_multiple_statements() diff --git a/query.sql b/query.sql index 26cb59d..8af97b0 100644 --- a/query.sql +++ b/query.sql @@ -1,2 +1,3 @@ -SELECT 1 -FROM DUAL; \ No newline at end of file +SELECT id, health_counter, created_at, updated_at +FROM simple_healthcheck_counters +;