started working on checkpoints

.idea/dataSources.xml

@@ -29,5 +29,12 @@
       <jdbc-url>jdbc:sqlite:C:\Users\Frede\IdeaProjects\db_interactor\local.db</jdbc-url>
       <working-dir>$ProjectFileDir$</working-dir>
     </data-source>
+    <data-source source="LOCAL" name="local [3]" uuid="1b93d1a3-fc5a-4057-9ac1-f582933b27d3">
+      <driver-ref>sqlite.xerial</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.sqlite.JDBC</jdbc-driver>
+      <jdbc-url>jdbc:sqlite:$PROJECT_DIR$/local.db</jdbc-url>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
   </component>
 </project>

.idea/database_interacter.iml

@@ -4,7 +4,7 @@
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.venv" />
     </content>
-    <orderEntry type="jdk" jdkName="uv (db_interactor)" jdkType="Python SDK" />
+    <orderEntry type="jdk" jdkName="Python 3.13 (db_interactor)" jdkType="Python SDK" />
     <orderEntry type="sourceFolder" forTests="false" />
   </component>
 </module>

.idea/sqldialects.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="SqlDialectMappings">
+    <file url="file://$PROJECT_DIR$/checkpoint/CheckpointService.py" dialect="GenericSQL" />
     <file url="file://$PROJECT_DIR$/query.sql" dialect="OracleSqlPlus" />
   </component>
 </project>

checkpoint/Checkpoint.py

@@ -0,0 +1,57 @@
+from dataclasses import dataclass
+from enum import Enum
+
+
+class CheckpointStatus(Enum):
+    STARTED = "STARTED"
+    COMPLETED = "COMPLETED"
+    FAILED = "FAILED"
+
+@dataclass
+class CheckpointMetadata:
+    _metadata: str
+    _type: str
+
+    def __init__(self, metadata: str, _type: str):
+        self._metadata = metadata
+        self._type = _type
+
+    @property
+    def metadata(self):
+        return self._metadata
+
+    @property
+    def type(self):
+        return self._type
+
+
+@dataclass
+class Checkpoint:
+    _name: str
+    _hash: str
+    _status: CheckpointStatus
+    _id: int
+
+
+    def __init__(self, id: int, name: str, hash: str, status: CheckpointStatus):
+        self._name = name
+        self._hash = hash
+        self._status = status
+        self._id = id
+
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def hash(self):
+        return self._hash
+
+    @property
+    def status(self):
+        return self._status
+
+    @property
+    def id(self):
+        return self._id

checkpoint/CheckpointService.py

@@ -0,0 +1,63 @@
+import sqlite3
+
+from checkpoint.Checkpoint import Checkpoint, CheckpointMetadata, CheckpointStatus
+
+
+class CheckpointService:
+    _table: str
+    _database: str
+    _connection: str
+    _conn: sqlite3.Connection
+    _insert_metadata_query = """ INSERT INTO checkpoint_metadata (checkpoint_id, metadata, type) VALUES (?, ?, ?) """
+
+    def __init__(self, database: str = "local.db"):
+        self._database = database
+        self._conn = sqlite3.connect(database)
+
+        self._initialize_table()
+
+    def _initialize_table(self):
+        create_checkpoint_table_query = """
+        CREATE TABLE IF NOT EXISTS checkpoint (
+            id INTEGER primary key autoincrement,
+            name TEXT not null,
+            hash TEXT not null unique,
+            status TEXT not null
+        );
+        """
+
+        create_checkpoint_metadata_table_query = """
+        CREATE TABLE IF NOT EXISTS checkpoint_metadata (
+            id INTEGER primary key autoincrement,
+            checkpoint_id INTEGER NOT NULL,
+            metadata TEXT NOT NULL,
+            type TEXT not null,
+                constraint checkpoint_metadata_checkpoint_fk
+                foreign key (checkpoint_id) references checkpoint (id)
+        );
+        """
+
+        self._conn.execute(create_checkpoint_table_query)
+        self._conn.execute(create_checkpoint_metadata_table_query)
+        self._conn.commit()
+
+    def start_checkpoint(self, name, hash, metadata: CheckpointMetadata) -> Checkpoint:
+        insert_checkpoint_query = """INSERT INTO checkpoint (name, hash, status) VALUES (?, ?, ?)"""
+
+        checkpoint_id = self._conn.execute(insert_checkpoint_query, (name, hash, CheckpointStatus.STARTED)).lastrowid
+        self._conn.execute(self._insert_metadata_query, (checkpoint_id, metadata.metadata, metadata.type))
+        self._conn.commit()
+
+        return Checkpoint(checkpoint_id, name, hash, CheckpointStatus.STARTED)
+
+    def end_checkpoint(self, checkpoint_id, metadata: CheckpointMetadata):
+        update_checkpoint_query = """UPDATE checkpoint SET status = ? WHERE checkpoint_id = ?"""
+        self._conn.execute(update_checkpoint_query, (CheckpointStatus.COMPLETED, checkpoint_id))
+        self._conn.execute(self._insert_metadata_query, (checkpoint_id, metadata.metadata, metadata.type))
+        self._conn.commit()
+
+    def get_checkpoint(self, checkpoint_id: int, hash: str):
+        ...
+
+    def delete_checkpoint(self):
+        ...

database/db_adapter.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import time
+from pathlib import Path
 
 import pandas as pd
 import sqlalchemy as sq
@@ -62,6 +63,7 @@ class DBAdapter:
                 conn.execute(sq.text(f"alter session set current_schema = {schema}"))
             case DatabaseType.PSQL:
                 conn.execute(sq.text(f"set schema '{schema}'"))
+                conn.execute(sq.text(f"set search_path = public"))
             case _:
                 raise Exception(
                     f'Database type {self._database_config.type} is not supported for readonly transactions')
@@ -73,7 +75,9 @@ class DBAdapter:
             pass
         return f'{self._output_folder}{filename}'
 
-    def _export_to_file(self, export_type, output_file_name, result, query_parameter: QueryParameters = None):
+    def _export_to_file(self, export_type: ExportType, output_file_name: str, result: pd.DataFrame, query_parameter: QueryParameters = None):
+        os.makedirs(os.path.dirname(output_file_name + ".txt"), exist_ok=True)
+
         match export_type:
             case ExportType.CSV:
                 output_file_name += '.csv'
@@ -172,7 +176,7 @@ class DBAdapter:
                         self._logger.info(
                             f'({query_file_index + 1}/{len(municipalities)}) Starting to process query with title: {query_parameter.title}')
 
-                        if not len(queries) != 1:
+                        if not len(queries) == 1:
                             self._logger.error(f'Query file {query_parameter.title} failed due to multiple queries')
                             raise Exception(f'Query file {query_parameter.title} failed due to multiple queries')
 

main.py

@@ -1,7 +1,9 @@
 from config.Config import Config
+from database.QueryParameters import QueryParameters
 from database.db_adapter import DBAdapter
 from keepass.Keepass import KeePass
 from logger.logger import init_logger
+from models.ExportType import ExportType
 from models.Municipality import Municipality
 
 logger = init_logger()
@@ -11,109 +13,12 @@ config = Config()
 keepass = KeePass(config.kee_pass, logger)
 
 municipalities = [
-    #Municipality('Central common schema', '00000000', 'KY_CENTRAL', 'KY_CENTRAL'),
-    Municipality('Odense Kommune', '35209115', 'KY_0461', '461'),
-    Municipality('Svendborg Kommune', '29189730', 'KY_0479', '479'),
-    Municipality('Nordfyns Kommune', '29188947', 'KY_0480', '480'),
-    Municipality('Langeland Kommune', '29188955', 'KY_0482', '482'),
-    Municipality('Ærø Kommune', '28856075', 'KY_0492', '492'),
-    Municipality('Haderslev Kommune', '29189757', 'KY_0510', '510'),
-    Municipality('Billund Kommune', '29189765', 'KY_0530', '530'),
-    Municipality('Sønderborg Kommune', '29189773', 'KY_0540', '540'),
-    Municipality('Tønder Kommune', '29189781', 'KY_0550', '550'),
-    Municipality('Esbjerg Kommune', '29189803', 'KY_0561', '561'),
-    Municipality('Fanø Kommune', '31210917', 'KY_0563', '563'),
-    Municipality('Varde Kommune', '29189811', 'KY_0573', '573'),
-    Municipality('Vejen Kommune', '29189838', 'KY_0575', '575'),
-    Municipality('Aabenraa Kommune', '29189854', 'KY_0580', '580'),
-    Municipality('Fredericia Kommune', '69116418', 'KY_0607', '607'),
-    Municipality('Horsens Kommune', '29189889', 'KY_0615', '615'),
-    Municipality('Kolding Kommune', '29189897', 'KY_0621', '621'),
-    Municipality('Vejle Kommune', '29189900', 'KY_0630', '630'),
-    Municipality('Herning Kommune', '29189919', 'KY_0657', '657'),
-    Municipality('Holstebro Kommune', '29189927', 'KY_0661', '661'),
-    Municipality('Lemvig Kommune', '29189935', 'KY_0665', '665'),
-    Municipality('Struer Kommune', '29189951', 'KY_0671', '671'),
-    Municipality('Syddjurs Kommune', '29189978', 'KY_0706', '706'),
-    Municipality('Norddjurs Kommune', '29189986', 'KY_0707', '707'),
-    Municipality('Favrskov Kommune', '29189714', 'KY_0710', '710'),
-    Municipality('Odder Kommune', '32264328', 'KY_0727', '727'),
-    Municipality('Randers Kommune', '29189668', 'KY_0730', '730'),
-    Municipality('Silkeborg Kommune', '29189641', 'KY_0740', '740'),
-    Municipality('Samsø Kommune', '23795515', 'KY_0741', '741'),
-    Municipality('Skanderborg Kommune', '29189633', 'KY_0746', '746'),
-    Municipality('Aarhus Kommune', '55133018', 'KY_0751', '751'),
-    Municipality('Ikast-Brande Kommune', '29189617', 'KY_0756', '756'),
-    Municipality('Ringkøbing-Skjern Kommune', '29189609', 'KY_0760', '760'),
-    Municipality('Hedensted Kommune', '29189587', 'KY_0766', '766'),
-    Municipality('Morsø Kommune', '41333014', 'KY_0773', '773'),
-    Municipality('Skive Kommune', '29189579', 'KY_0779', '779'),
-    Municipality('Thisted Kommune', '29189560', 'KY_0787', '787'),
-    Municipality('Viborg Kommune', '29189846', 'KY_0791', '791'),
-    Municipality('Brønderslev Kommune', '29189501', 'KY_0810', '810'),
-    Municipality('Frederikshavn Kommune', '29189498', 'KY_0813', '813'),
-    Municipality('Vesthimmerlands Kommune', '29189471', 'KY_0820', '820'),
-    Municipality('Læsø Kommune', '45973328', 'KY_0825', '825'),
-    Municipality('Rebild Kommune', '29189463', 'KY_0840', '840'),
-    Municipality('Mariagerfjord Kommune', '29189455', 'KY_0846', '846'),
-    Municipality('Jammerbugt Kommune', '29189439', 'KY_0849', '849'),
-    Municipality('Aalborg Kommune', '29189420', 'KY_0851', '851'),
-    Municipality('Hjørring Kommune', '29189382', 'KY_0860', '860'),
-    Municipality('Københavns Kommune', '64942212', 'KY_0101', '101'),
-    Municipality('Frederiksberg Kommune', '11259979', 'KY_0147', '147'),
-    Municipality('Ballerup Kommune', '58271713', 'KY_0151', '151'),
-    Municipality('Brøndby Kommune', '65113015', 'KY_0153', '153'),
-    Municipality('Dragør Kommune', '12881517', 'KY_0155', '155'),
-    Municipality('Gentofte Kommune', '19438414', 'KY_0157', '157'),
-    Municipality('Gladsaxe Kommune', '62761113', 'KY_0159', '159'),
-    Municipality('Glostrup Kommune', '65120119', 'KY_0161', '161'),
-    Municipality('Herlev Kommune', '63640719', 'KY_0163', '163'),
-    Municipality('Albertslund Kommune', '66137112', 'KY_0165', '165'),
-    Municipality('Hvidovre Kommune', '55606617', 'KY_0167', '167'),
-    Municipality('Høje Taastrup Kommune', '19501817', 'KY_0169', '169'),
-    Municipality('Lyngby-Taarbæk Kommune', '11715311', 'KY_0173', '173'),
-    Municipality('Rødovre Kommune', '65307316', 'KY_0175', '175'),
-    Municipality('Ishøj Kommune', '11931316', 'KY_0183', '183'),
-    Municipality('Tårnby Kommune', '20310413', 'KY_0185', '185'),
-    Municipality('Vallensbæk Kommune', '19583910', 'KY_0187', '187'),
-    Municipality('Furesø Kommune', '29188327', 'KY_0190', '190'),
-    Municipality('Allerød Kommune', '60183112', 'KY_0201', '201'),
-    Municipality('Fredensborg Kommune', '29188335', 'KY_0210', '210'),
-    Municipality('Helsingør Kommune', '64502018', 'KY_0217', '217'),
-    Municipality('Hillerød Kommune', '29189366', 'KY_0219', '219'),
-    Municipality('Hørsholm Kommune', '70960516', 'KY_0223', '223'),
-    Municipality('Rudersdal Kommune', '29188378', 'KY_0230', '230'),
-    Municipality('Egedal Kommune', '29188386', 'KY_0240', '240'),
-    Municipality('Frederikssund Kommune', '29189129', 'KY_0250', '250'),
-    Municipality('Greve Kommune', '44023911', 'KY_0253', '253'),
-    Municipality('Køge Kommune', '29189374', 'KY_0259', '259'),
-    Municipality('Halsnæs Kommune', '29188416', 'KY_0260', '260'),
-    Municipality('Roskilde Kommune', '29189404', 'KY_0265', '265'),
-    Municipality('Solrød Kommune', '68534917', 'KY_0269', '269'),
-    Municipality('Gribskov Kommune', '29188440', 'KY_0270', '270'),
-    Municipality('Odsherred Kommune', '29188459', 'KY_0306', '306'),
-    Municipality('Holbæk Kommune', '29189447', 'KY_0316', '316'),
-    Municipality('Faxe Kommune', '29188475', 'KY_0320', '320'),
-    Municipality('Kalundborg Kommune', '29189595', 'KY_0326', '326'),
-    Municipality('Ringsted Kommune', '18957981', 'KY_0329', '329'),
-    Municipality('Slagelse Kommune', '29188505', 'KY_0330', '330'),
-    Municipality('Stevns Kommune', '29208654', 'KY_0336', '336'),
-    Municipality('Sorø Kommune', '29189994', 'KY_0340', '340'),
-    Municipality('Lejre Kommune', '29188548', 'KY_0350', '350'),
-    Municipality('Lolland Kommune', '29188572', 'KY_0360', '360'),
-    Municipality('Næstved Kommune', '29189625', 'KY_0370', '370'),
-    Municipality('Guldborgsund Kommune', '29188599', 'KY_0376', '376'),
-    Municipality('Vordingborg Kommune', '29189676', 'KY_0390', '390'),
-    Municipality('Bornholms Regionskommune', '26696348', 'KY_0400', '400'),
-    Municipality('Middelfart Kommune', '29189684', 'KY_0410', '410'),
-    Municipality('Assens Kommune', '29189692', 'KY_0420', '420'),
-    Municipality('Faaborg-Midtfyn Kommune', '29188645', 'KY_0430', '430'),
-    Municipality('Kerteminde Kommune', '29189706', 'KY_0440', '440'),
-    Municipality('Nyborg Kommune', '29189722', 'KY_0450', '450'),
+    Municipality("op_test", None, "OP", "test")
 ]
 
+parameter = QueryParameters(title="Test rapport", input_path="query.sql", export_type=ExportType.XML)
 
 db_adapter = DBAdapter(keepass, config.database, logger)
-db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities=municipalities)
+db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities=municipalities, query_parameter=parameter)
 
 # db_adapter.run_sql_file_multiple_statements()

query.sql

@@ -1,2 +1,3 @@
-SELECT 1
-FROM DUAL;
+SELECT id, health_counter, created_at, updated_at
+FROM simple_healthcheck_counters
+;