Flamingo/db_interactor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Started doing some refactoring to make it more modulized to make it easier to make a collection of scripts later

Browse Source
This commit is contained in:
Frederik Jacobsen 2025-03-08 23:42:33 +01:00
parent 73c333229b
commit 6f451499c0
19 changed files with 1385 additions and 283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
config/Config.py Normal file
View File

@ -0,0 +1,25 @@
from dataclasses import dataclass
import toml
from models.DatabaseConfig import DatabaseConfig
from models.KeePassConfig import KeePassConfig
@dataclass
class Config:
_config: dict
_kee_pass_config: KeePassConfig
_database_config: DatabaseConfig
def __init__(self):
with open('config.toml', 'r') as f:
self._config: dict = toml.load(f)
self._kee_pass_config: KeePassConfig = KeePassConfig(self._config['keepass'])
self._database_config: DatabaseConfig = DatabaseConfig(self._config['database'])
@property
def kee_pass(self) -> KeePassConfig:
return self._kee_pass_config
@property
def database(self) -> DatabaseConfig:
return self._database_config

0
config/__init__.py Normal file
View File

0
database/__init__.py Normal file
View File

45
db_adapter.py → database/db_adapter.py
View File

@ -1,13 +1,16 @@
import logging import logging
import os import os
import time
import pandas as pd
import sqlalchemy as sq import sqlalchemy as sq
import sqlparse import sqlparse
import pandas as pd
from config import DatabaseConfig, DatabaseType from models.DatabaseConfig import DatabaseConfig
from keepass import KeePass from models.DatabaseType import DatabaseType
from models import Municipality, ExportType from keepass.keepass import KeePass
import time from models.ExportType import ExportType
from models.Municipality import Municipality
class DBAdapter: class DBAdapter:
@ -99,7 +102,7 @@ class DBAdapter:
self._logger.info(f'Created file {output_file_name}') self._logger.info(f'Created file {output_file_name}')
def run_query(self, query: str, read_only = True) -> sq.CursorResult: def run_query(self, query: str, read_only=True) -> sq.CursorResult:
""" """
Runs a single SQL query and returns the result as a CursorResult. Runs a single SQL query and returns the result as a CursorResult.
If more than one query, throws an error If more than one query, throws an error
@ -129,11 +132,12 @@ class DBAdapter:
self._logger.info(f'Transaction commited') self._logger.info(f'Transaction commited')
return result return result
def run_sql_file_one_statement(self, filename: str = "query.sql", read_only = True) -> sq.CursorResult: def run_sql_file_one_statement(self, filename: str = "query.sql", read_only=True) -> sq.CursorResult:
query = self._read_and_sql_file_and_strip_for_comments(filename) query = self._read_and_sql_file_and_strip_for_comments(filename)
return self.run_query(query, read_only) return self.run_query(query, read_only)
def run_sql_file_export_to_file(self, schema: str | None = None, input_name: str = "query.sql", output_name: str = "export", read_only = True, export_type = ExportType.CSV): def run_sql_file_export_to_file(self, schema: str | None = None, input_name: str = "query.sql",
output_name: str = "export", read_only=True, export_type=ExportType.CSV):
""" """
Runs a single SQL query and creates a csv file with the given output name and resulting contents. Runs a single SQL query and creates a csv file with the given output name and resulting contents.
If more than one query, throws an error If more than one query, throws an error
@ -156,7 +160,8 @@ class DBAdapter:
result = self._extract_dataframe(conn, query, read_only) result = self._extract_dataframe(conn, query, read_only)
self._export_to_file(export_type, self._generate_filename(output_name), result) self._export_to_file(export_type, self._generate_filename(output_name), result)
def _extract_dataframe(self, conn: sq.Connection, query: str, read_only: bool, schema: str | None = None) -> pd.DataFrame: def _extract_dataframe(self, conn: sq.Connection, query: str, read_only: bool,
schema: str | None = None) -> pd.DataFrame:
result: pd.DataFrame result: pd.DataFrame
self._verify_singular_query(query) self._verify_singular_query(query)
@ -181,18 +186,20 @@ class DBAdapter:
self._logger.info(f'Query took {(end - start):.4f} seconds') self._logger.info(f'Query took {(end - start):.4f} seconds')
return result return result
def run_sql_file_export_to_file_multiple_schemas(self, municipalities: list[Municipality], base_output_name: str = "", input_name: str = "query.sql", read_only = True, export_type = ExportType.CSV): def run_sql_file_export_to_file_multiple_schemas(self, municipalities: list[Municipality],
base_output_name: str = "", input_name: str = "query.sql",
read_only=True, export_type=ExportType.CSV):
query = self._read_and_sql_file_and_strip_for_comments(input_name) query = self._read_and_sql_file_and_strip_for_comments(input_name)
self._logger.info(f'Running on {len(municipalities)} schemas') self._logger.info(f'Running on {len(municipalities)} schemas')
self._logger.info(f'Running query: "{query}"') self._logger.info(f'Running query: "{query}"')
with self._engine.connect() as conn: with self._engine.connect() as conn:
for index, municipality in enumerate(municipalities): for index, municipality in enumerate(municipalities):
self._logger.info(f'({index + 1}/{len(municipalities)}) running for municipality {municipality.name}') self._logger.info(f'({index + 1}/{len(municipalities)}) running for municipality {municipality.name}')
result = self._extract_dataframe(conn, query, read_only, schema = municipality.schema) result = self._extract_dataframe(conn, query, read_only, schema=municipality.schema)
output_file_name = self._generate_filename(f'{base_output_name}{municipality.name}') output_file_name = self._generate_filename(f'{base_output_name}{municipality.name}')
self._export_to_file(export_type, output_file_name, result) self._export_to_file(export_type, output_file_name, result)
def run_sql_file_multiple_statements(self, filename: str = "query.sql", read_only = False): def run_sql_file_multiple_statements(self, filename: str = "query.sql", read_only=False):
""" """
Runs an SQL file, supports multiple statements, does not support plsql. Runs an SQL file, supports multiple statements, does not support plsql.
If any statements fail, throws an error and rolls back. If any statements fail, throws an error and rolls back.
@ -214,7 +221,8 @@ class DBAdapter:
start = time.time() start = time.time()
conn.execute(sq.text(query)) conn.execute(sq.text(query))
end = time.time() end = time.time()
self._logger.info(f'({index + 1} / {len(queries)}) Query took {(end - start):.4f} seconds ({query})') self._logger.info(
f'({index + 1} / {len(queries)}) Query took {(end - start):.4f} seconds ({query})')
conn.commit() conn.commit()
except Exception as e: except Exception as e:
self._logger.info(f'Transaction rollback') self._logger.info(f'Transaction rollback')
@ -222,8 +230,9 @@ class DBAdapter:
raise e raise e
self._logger.info(f'Transaction commited') self._logger.info(f'Transaction commited')
def run_sql_files_export_to_files_multiple_schemas(self, municipalities: list[Municipality],
def run_sql_files_export_to_files_multiple_schemas(self, municipalities: list[Municipality], input_querie_file_names: list[str] = None, read_only: bool = True, export_type = ExportType.XML): input_querie_file_names: list[str] = None,
read_only: bool = True, export_type=ExportType.XML):
"""" """"
Runs the list of granted sql files against the list of municipalities Runs the list of granted sql files against the list of municipalities
:param export_type: the type of files to export :param export_type: the type of files to export
@ -238,12 +247,14 @@ class DBAdapter:
self._set_transaction_readonly(conn) self._set_transaction_readonly(conn)
for municipality_index, municipality in enumerate(municipalities): for municipality_index, municipality in enumerate(municipalities):
self._logger.info(f'({municipality_index + 1}/{len(municipalities)}) Starting to process municipality {municipality.name} ({municipality.schema})') self._logger.info(
f'({municipality_index + 1}/{len(municipalities)}) Starting to process municipality {municipality.name} ({municipality.schema})')
self._set_schema(conn, municipality.schema) self._set_schema(conn, municipality.schema)
file_prefix = f'{municipality.kommunekode}/' file_prefix = f'{municipality.kommunekode}/'
for query_file_index, query_filename in enumerate(input_querie_file_names): for query_file_index, query_filename in enumerate(input_querie_file_names):
self._logger.info(f'({query_file_index + 1}/{len(municipalities)}) Starting to process query file: {query_filename}') self._logger.info(
f'({query_file_index + 1}/{len(municipalities)}) Starting to process query file: {query_filename}')
raw_query = self._read_and_sql_file_and_strip_for_comments(query_filename) raw_query = self._read_and_sql_file_and_strip_for_comments(query_filename)
if not self._verify_singular_query(raw_query): if not self._verify_singular_query(raw_query):

173
jupyter.ipynb
View File

@ -1,173 +0,0 @@
{
"cells": [
{
"metadata": {},
"cell_type": "code",
"source": [
"from config import Config\n",
"from db_adapter import DBAdapter\n",
"from keepass import KeePass\n",
"from logger import init_logger\n",
"from models import Municipality, ExportType\n",
"\n",
"logger = init_logger()"
],
"id": "ce6e9d363184c46b",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "code",
"source": [
"config = Config()\n",
"\n",
"keepass = KeePass(config.kee_pass, logger)\n",
"\n",
"db_adapter = DBAdapter(keepass, config.database, logger)"
],
"id": "3eec86ed2e7c0658",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "code",
"source": [
"municipalities = [\n",
" #Municipality('Central common schema', '00000000', 'KY_CENTRAL', 'KY_CENTRAL'),\n",
" Municipality('Odense Kommune', '35209115', 'KY_0461', '461'),\n",
" Municipality('Svendborg Kommune', '29189730', 'KY_0479', '479'),\n",
" Municipality('Nordfyns Kommune', '29188947', 'KY_0480', '480'),\n",
" Municipality('Langeland Kommune', '29188955', 'KY_0482', '482'),\n",
" Municipality('Ærø Kommune', '28856075', 'KY_0492', '492'),\n",
" Municipality('Haderslev Kommune', '29189757', 'KY_0510', '510'),\n",
" Municipality('Billund Kommune', '29189765', 'KY_0530', '530'),\n",
" Municipality('Sønderborg Kommune', '29189773', 'KY_0540', '540'),\n",
" Municipality('Tønder Kommune', '29189781', 'KY_0550', '550'),\n",
" Municipality('Esbjerg Kommune', '29189803', 'KY_0561', '561'),\n",
" Municipality('Fanø Kommune', '31210917', 'KY_0563', '563'),\n",
" Municipality('Varde Kommune', '29189811', 'KY_0573', '573'),\n",
" Municipality('Vejen Kommune', '29189838', 'KY_0575', '575'),\n",
" Municipality('Aabenraa Kommune', '29189854', 'KY_0580', '580'),\n",
" Municipality('Fredericia Kommune', '69116418', 'KY_0607', '607'),\n",
" Municipality('Horsens Kommune', '29189889', 'KY_0615', '615'),\n",
" Municipality('Kolding Kommune', '29189897', 'KY_0621', '621'),\n",
" Municipality('Vejle Kommune', '29189900', 'KY_0630', '630'),\n",
" Municipality('Herning Kommune', '29189919', 'KY_0657', '657'),\n",
" Municipality('Holstebro Kommune', '29189927', 'KY_0661', '661'),\n",
" Municipality('Lemvig Kommune', '29189935', 'KY_0665', '665'),\n",
" Municipality('Struer Kommune', '29189951', 'KY_0671', '671'),\n",
" Municipality('Syddjurs Kommune', '29189978', 'KY_0706', '706'),\n",
" Municipality('Norddjurs Kommune', '29189986', 'KY_0707', '707'),\n",
" Municipality('Favrskov Kommune', '29189714', 'KY_0710', '710'),\n",
" Municipality('Odder Kommune', '32264328', 'KY_0727', '727'),\n",
" Municipality('Randers Kommune', '29189668', 'KY_0730', '730'),\n",
" Municipality('Silkeborg Kommune', '29189641', 'KY_0740', '740'),\n",
" Municipality('Samsø Kommune', '23795515', 'KY_0741', '741'),\n",
" Municipality('Skanderborg Kommune', '29189633', 'KY_0746', '746'),\n",
" Municipality('Aarhus Kommune', '55133018', 'KY_0751', '751'),\n",
" Municipality('Ikast-Brande Kommune', '29189617', 'KY_0756', '756'),\n",
" Municipality('Ringkøbing-Skjern Kommune', '29189609', 'KY_0760', '760'),\n",
" Municipality('Hedensted Kommune', '29189587', 'KY_0766', '766'),\n",
" Municipality('Morsø Kommune', '41333014', 'KY_0773', '773'),\n",
" Municipality('Skive Kommune', '29189579', 'KY_0779', '779'),\n",
" Municipality('Thisted Kommune', '29189560', 'KY_0787', '787'),\n",
" Municipality('Viborg Kommune', '29189846', 'KY_0791', '791'),\n",
" Municipality('Brønderslev Kommune', '29189501', 'KY_0810', '810'),\n",
" Municipality('Frederikshavn Kommune', '29189498', 'KY_0813', '813'),\n",
" Municipality('Vesthimmerlands Kommune', '29189471', 'KY_0820', '820'),\n",
" Municipality('Læsø Kommune', '45973328', 'KY_0825', '825'),\n",
" Municipality('Rebild Kommune', '29189463', 'KY_0840', '840'),\n",
" Municipality('Mariagerfjord Kommune', '29189455', 'KY_0846', '846'),\n",
" Municipality('Jammerbugt Kommune', '29189439', 'KY_0849', '849'),\n",
" Municipality('Aalborg Kommune', '29189420', 'KY_0851', '851'),\n",
" Municipality('Hjørring Kommune', '29189382', 'KY_0860', '860'),\n",
" Municipality('Københavns Kommune', '64942212', 'KY_0101', '101'),\n",
" Municipality('Frederiksberg Kommune', '11259979', 'KY_0147', '147'),\n",
" Municipality('Ballerup Kommune', '58271713', 'KY_0151', '151'),\n",
" Municipality('Brøndby Kommune', '65113015', 'KY_0153', '153'),\n",
" Municipality('Dragør Kommune', '12881517', 'KY_0155', '155'),\n",
" Municipality('Gentofte Kommune', '19438414', 'KY_0157', '157'),\n",
" Municipality('Gladsaxe Kommune', '62761113', 'KY_0159', '159'),\n",
" Municipality('Glostrup Kommune', '65120119', 'KY_0161', '161'),\n",
" Municipality('Herlev Kommune', '63640719', 'KY_0163', '163'),\n",
" Municipality('Albertslund Kommune', '66137112', 'KY_0165', '165'),\n",
" Municipality('Hvidovre Kommune', '55606617', 'KY_0167', '167'),\n",
" Municipality('Høje Taastrup Kommune', '19501817', 'KY_0169', '169'),\n",
" Municipality('Lyngby-Taarbæk Kommune', '11715311', 'KY_0173', '173'),\n",
" Municipality('Rødovre Kommune', '65307316', 'KY_0175', '175'),\n",
" Municipality('Ishøj Kommune', '11931316', 'KY_0183', '183'),\n",
" Municipality('Tårnby Kommune', '20310413', 'KY_0185', '185'),\n",
" Municipality('Vallensbæk Kommune', '19583910', 'KY_0187', '187'),\n",
" Municipality('Furesø Kommune', '29188327', 'KY_0190', '190'),\n",
" Municipality('Allerød Kommune', '60183112', 'KY_0201', '201'),\n",
" Municipality('Fredensborg Kommune', '29188335', 'KY_0210', '210'),\n",
" Municipality('Helsingør Kommune', '64502018', 'KY_0217', '217'),\n",
" Municipality('Hillerød Kommune', '29189366', 'KY_0219', '219'),\n",
" Municipality('Hørsholm Kommune', '70960516', 'KY_0223', '223'),\n",
" Municipality('Rudersdal Kommune', '29188378', 'KY_0230', '230'),\n",
" Municipality('Egedal Kommune', '29188386', 'KY_0240', '240'),\n",
" Municipality('Frederikssund Kommune', '29189129', 'KY_0250', '250'),\n",
" Municipality('Greve Kommune', '44023911', 'KY_0253', '253'),\n",
" Municipality('Køge Kommune', '29189374', 'KY_0259', '259'),\n",
" Municipality('Halsnæs Kommune', '29188416', 'KY_0260', '260'),\n",
" Municipality('Roskilde Kommune', '29189404', 'KY_0265', '265'),\n",
" Municipality('Solrød Kommune', '68534917', 'KY_0269', '269'),\n",
" Municipality('Gribskov Kommune', '29188440', 'KY_0270', '270'),\n",
" Municipality('Odsherred Kommune', '29188459', 'KY_0306', '306'),\n",
" Municipality('Holbæk Kommune', '29189447', 'KY_0316', '316'),\n",
" Municipality('Faxe Kommune', '29188475', 'KY_0320', '320'),\n",
" Municipality('Kalundborg Kommune', '29189595', 'KY_0326', '326'),\n",
" Municipality('Ringsted Kommune', '18957981', 'KY_0329', '329'),\n",
" Municipality('Slagelse Kommune', '29188505', 'KY_0330', '330'),\n",
" Municipality('Stevns Kommune', '29208654', 'KY_0336', '336'),\n",
" Municipality('Sorø Kommune', '29189994', 'KY_0340', '340'),\n",
" Municipality('Lejre Kommune', '29188548', 'KY_0350', '350'),\n",
" Municipality('Lolland Kommune', '29188572', 'KY_0360', '360'),\n",
" Municipality('Næstved Kommune', '29189625', 'KY_0370', '370'),\n",
" Municipality('Guldborgsund Kommune', '29188599', 'KY_0376', '376'),\n",
" Municipality('Vordingborg Kommune', '29189676', 'KY_0390', '390'),\n",
" Municipality('Bornholms Regionskommune', '26696348', 'KY_0400', '400'),\n",
" Municipality('Middelfart Kommune', '29189684', 'KY_0410', '410'),\n",
" Municipality('Assens Kommune', '29189692', 'KY_0420', '420'),\n",
" Municipality('Faaborg-Midtfyn Kommune', '29188645', 'KY_0430', '430'),\n",
" Municipality('Kerteminde Kommune', '29189706', 'KY_0440', '440'),\n",
" Municipality('Nyborg Kommune', '29189722', 'KY_0450', '450'),\n",
"]"
],
"id": "6b6e8799942abcc1",
"outputs": [],
"execution_count": null
},
{
"metadata": {},
"cell_type": "code",
"source": "db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities, export_type=ExportType.EXCEL)",
"id": "7b3292ddcb9edd91",
"outputs": [],
"execution_count": null
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 2
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
"version": "2.7.6"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

0
keepass/__init__.py Normal file
View File

14
keepass.py → keepass/keepass.py
View File

@ -1,9 +1,11 @@
import getpass
import logging import logging
from dataclasses import dataclass
from pykeepass import PyKeePass from pykeepass import PyKeePass
from config import KeePassConfig
import getpass from models.KeePassConfig import KeePassConfig
from dataclasses import dataclass
@dataclass @dataclass
class KeePassEntry: class KeePassEntry:
@ -20,7 +22,6 @@ class KeePassEntry:
return self._name return self._name
class KeePass: class KeePass:
def __init__(self, config: KeePassConfig, logger: logging.Logger): def __init__(self, config: KeePassConfig, logger: logging.Logger):
self._logger: logging.Logger = logger self._logger: logging.Logger = logger
@ -31,7 +32,8 @@ class KeePass:
self._logger.info(f'Successfully connected to keepass') self._logger.info(f'Successfully connected to keepass')
def get_db_credentials(self) -> KeePassEntry: def get_db_credentials(self) -> KeePassEntry:
self._logger.info(f'Searching for database credentials on credential: {self._kee_pass_config.db_credentials_name}') self._logger.info(
f'Searching for database credentials on credential: {self._kee_pass_config.db_credentials_name}')
group = None group = None
if self._kee_pass_config.db_credentials_group.strip() is not None and self._kee_pass_config.db_credentials_group.strip() != "": if self._kee_pass_config.db_credentials_group.strip() is not None and self._kee_pass_config.db_credentials_group.strip() != "":
self._logger.info(f'Searching in group {self._kee_pass_config.db_credentials_group}') self._logger.info(f'Searching in group {self._kee_pass_config.db_credentials_group}')
@ -50,4 +52,4 @@ class KeePass:
raise Exception(f'Could not find password, found {len(group)} entries') raise Exception(f'Could not find password, found {len(group)} entries')
self._logger.info(f'Found credentials for database for username {group[0].username}') self._logger.info(f'Found credentials for database for username {group[0].username}')
return KeePassEntry(group[0].username, group[0].password) return KeePassEntry(group[0].username, group[0].password)

0
logger/__init__.py Normal file
View File

30
logger.py → logger/logger.py
View File

@ -1,12 +1,10 @@
import logging import logging
import os
import sqlite3 import sqlite3
DEFAULT_SEPARATOR = '|' DEFAULT_SEPARATOR = '|'
DEFAULT_DATA_TYPE = 'TEXT' DEFAULT_DATA_TYPE = 'TEXT'
# WARNING: attributes must be choosen from https://docs.python.org/3/library/logging.html#formatter-objects
#WARNING: attributes must be choosen from https://docs.python.org/3/library/logging.html#formatter-objects
DEFAULT_ATTRIBUTES_LIST = ['asctime', 'levelname', 'name', 'message'] DEFAULT_ATTRIBUTES_LIST = ['asctime', 'levelname', 'name', 'message']
@ -16,7 +14,7 @@ class SQLiteHandler(logging.Handler):
Based on Yarin Kessler's sqlite_handler.py https://gist.github.com/ykessler/2662203#file_sqlite_handler.py Based on Yarin Kessler's sqlite_handler.py https://gist.github.com/ykessler/2662203#file_sqlite_handler.py
""" """
def __init__(self, database:str = "local.db", table = "log", attributes_list=None): def __init__(self, database: str = "local.db", table="log", attributes_list=None):
""" """
SQLiteHandler class constructor SQLiteHandler class constructor
Parameters: Parameters:
@ -27,8 +25,8 @@ class SQLiteHandler(logging.Handler):
Returns: Returns:
None None
""" """
#super(SQLiteHandler, self).__init__() # for python 2.X # super(SQLiteHandler, self).__init__() # for python 2.X
super().__init__() # for python 3.X super().__init__() # for python 3.X
if attributes_list is None: if attributes_list is None:
attributes_list = DEFAULT_ATTRIBUTES_LIST attributes_list = DEFAULT_ATTRIBUTES_LIST
self.database = database self.database = database
@ -36,14 +34,14 @@ class SQLiteHandler(logging.Handler):
self.attributes = attributes_list self.attributes = attributes_list
# Create table if needed # Create table if needed
create_table_sql = 'CREATE TABLE IF NOT EXISTS ' + self.table + ' (' + ((' ' + DEFAULT_DATA_TYPE + ', ').join(self.attributes)) + ' ' + DEFAULT_DATA_TYPE + ');' create_table_sql = 'CREATE TABLE IF NOT EXISTS ' + self.table + ' (' + (
#print(create_table_sql) (' ' + DEFAULT_DATA_TYPE + ', ').join(self.attributes)) + ' ' + DEFAULT_DATA_TYPE + ');'
# print(create_table_sql)
conn = sqlite3.connect(self.database) conn = sqlite3.connect(self.database)
conn.execute(create_table_sql) conn.execute(create_table_sql)
conn.commit() conn.commit()
conn.close() conn.close()
def emit(self, record): def emit(self, record):
""" """
Save the log record Save the log record
@ -56,13 +54,15 @@ class SQLiteHandler(logging.Handler):
# Use default formatting if no formatter is set # Use default formatting if no formatter is set
self.format(record) self.format(record)
#print(record.__dict__) # print(record.__dict__)
record_values = [record.__dict__[k] for k in self.attributes] record_values = [record.__dict__[k] for k in self.attributes]
str_record_values = ', '.join("'{0}'".format(v.replace("'", '').replace('"', '').replace('\n', ' ')) for v in record_values) str_record_values = ', '.join(
#print(str_record_values) "'{0}'".format(v.replace("'", '').replace('"', '').replace('\n', ' ')) for v in record_values)
# print(str_record_values)
insert_sql = 'INSERT INTO ' + self.table + ' (' + (', '.join(self.attributes)) + ') VALUES (' + str_record_values + ');' insert_sql = 'INSERT INTO ' + self.table + ' (' + (
#print(insert_sql) ', '.join(self.attributes)) + ') VALUES (' + str_record_values + ');'
# print(insert_sql)
conn = sqlite3.connect(self.database) conn = sqlite3.connect(self.database)
conn.execute(insert_sql) conn.execute(insert_sql)
conn.commit() conn.commit()
@ -89,4 +89,4 @@ def init_logger() -> logging.Logger:
logger.addHandler(console_handler) logger.addHandler(console_handler)
logger.addHandler(sql_handler) logger.addHandler(sql_handler)
return logger return logger

119
main.py
View File

@ -1,8 +1,8 @@
from config import Config from config.Config import Config
from db_adapter import DBAdapter from database.db_adapter import DBAdapter
from keepass import KeePass from keepass.keepass import KeePass
from logger import init_logger from logger.logger import init_logger
from models import Municipality from models.Municipality import Municipality
logger = init_logger() logger = init_logger()
@ -10,7 +10,110 @@ config = Config()
keepass = KeePass(config.kee_pass, logger) keepass = KeePass(config.kee_pass, logger)
db_adapter = DBAdapter(keepass, config.database, logger) municipalities = [
db_adapter.run_sql_file_export_to_file_multiple_schemas([Municipality('København Kommune', '64942212', 'KY_0101', '101')]) #Municipality('Central common schema', '00000000', 'KY_CENTRAL', 'KY_CENTRAL'),
Municipality('Odense Kommune', '35209115', 'KY_0461', '461'),
Municipality('Svendborg Kommune', '29189730', 'KY_0479', '479'),
Municipality('Nordfyns Kommune', '29188947', 'KY_0480', '480'),
Municipality('Langeland Kommune', '29188955', 'KY_0482', '482'),
Municipality('Ærø Kommune', '28856075', 'KY_0492', '492'),
Municipality('Haderslev Kommune', '29189757', 'KY_0510', '510'),
Municipality('Billund Kommune', '29189765', 'KY_0530', '530'),
Municipality('Sønderborg Kommune', '29189773', 'KY_0540', '540'),
Municipality('Tønder Kommune', '29189781', 'KY_0550', '550'),
Municipality('Esbjerg Kommune', '29189803', 'KY_0561', '561'),
Municipality('Fanø Kommune', '31210917', 'KY_0563', '563'),
Municipality('Varde Kommune', '29189811', 'KY_0573', '573'),
Municipality('Vejen Kommune', '29189838', 'KY_0575', '575'),
Municipality('Aabenraa Kommune', '29189854', 'KY_0580', '580'),
Municipality('Fredericia Kommune', '69116418', 'KY_0607', '607'),
Municipality('Horsens Kommune', '29189889', 'KY_0615', '615'),
Municipality('Kolding Kommune', '29189897', 'KY_0621', '621'),
Municipality('Vejle Kommune', '29189900', 'KY_0630', '630'),
Municipality('Herning Kommune', '29189919', 'KY_0657', '657'),
Municipality('Holstebro Kommune', '29189927', 'KY_0661', '661'),
Municipality('Lemvig Kommune', '29189935', 'KY_0665', '665'),
Municipality('Struer Kommune', '29189951', 'KY_0671', '671'),
Municipality('Syddjurs Kommune', '29189978', 'KY_0706', '706'),
Municipality('Norddjurs Kommune', '29189986', 'KY_0707', '707'),
Municipality('Favrskov Kommune', '29189714', 'KY_0710', '710'),
Municipality('Odder Kommune', '32264328', 'KY_0727', '727'),
Municipality('Randers Kommune', '29189668', 'KY_0730', '730'),
Municipality('Silkeborg Kommune', '29189641', 'KY_0740', '740'),
Municipality('Samsø Kommune', '23795515', 'KY_0741', '741'),
Municipality('Skanderborg Kommune', '29189633', 'KY_0746', '746'),
Municipality('Aarhus Kommune', '55133018', 'KY_0751', '751'),
Municipality('Ikast-Brande Kommune', '29189617', 'KY_0756', '756'),
Municipality('Ringkøbing-Skjern Kommune', '29189609', 'KY_0760', '760'),
Municipality('Hedensted Kommune', '29189587', 'KY_0766', '766'),
Municipality('Morsø Kommune', '41333014', 'KY_0773', '773'),
Municipality('Skive Kommune', '29189579', 'KY_0779', '779'),
Municipality('Thisted Kommune', '29189560', 'KY_0787', '787'),
Municipality('Viborg Kommune', '29189846', 'KY_0791', '791'),
Municipality('Brønderslev Kommune', '29189501', 'KY_0810', '810'),
Municipality('Frederikshavn Kommune', '29189498', 'KY_0813', '813'),
Municipality('Vesthimmerlands Kommune', '29189471', 'KY_0820', '820'),
Municipality('Læsø Kommune', '45973328', 'KY_0825', '825'),
Municipality('Rebild Kommune', '29189463', 'KY_0840', '840'),
Municipality('Mariagerfjord Kommune', '29189455', 'KY_0846', '846'),
Municipality('Jammerbugt Kommune', '29189439', 'KY_0849', '849'),
Municipality('Aalborg Kommune', '29189420', 'KY_0851', '851'),
Municipality('Hjørring Kommune', '29189382', 'KY_0860', '860'),
Municipality('Københavns Kommune', '64942212', 'KY_0101', '101'),
Municipality('Frederiksberg Kommune', '11259979', 'KY_0147', '147'),
Municipality('Ballerup Kommune', '58271713', 'KY_0151', '151'),
Municipality('Brøndby Kommune', '65113015', 'KY_0153', '153'),
Municipality('Dragør Kommune', '12881517', 'KY_0155', '155'),
Municipality('Gentofte Kommune', '19438414', 'KY_0157', '157'),
Municipality('Gladsaxe Kommune', '62761113', 'KY_0159', '159'),
Municipality('Glostrup Kommune', '65120119', 'KY_0161', '161'),
Municipality('Herlev Kommune', '63640719', 'KY_0163', '163'),
Municipality('Albertslund Kommune', '66137112', 'KY_0165', '165'),
Municipality('Hvidovre Kommune', '55606617', 'KY_0167', '167'),
Municipality('Høje Taastrup Kommune', '19501817', 'KY_0169', '169'),
Municipality('Lyngby-Taarbæk Kommune', '11715311', 'KY_0173', '173'),
Municipality('Rødovre Kommune', '65307316', 'KY_0175', '175'),
Municipality('Ishøj Kommune', '11931316', 'KY_0183', '183'),
Municipality('Tårnby Kommune', '20310413', 'KY_0185', '185'),
Municipality('Vallensbæk Kommune', '19583910', 'KY_0187', '187'),
Municipality('Furesø Kommune', '29188327', 'KY_0190', '190'),
Municipality('Allerød Kommune', '60183112', 'KY_0201', '201'),
Municipality('Fredensborg Kommune', '29188335', 'KY_0210', '210'),
Municipality('Helsingør Kommune', '64502018', 'KY_0217', '217'),
Municipality('Hillerød Kommune', '29189366', 'KY_0219', '219'),
Municipality('Hørsholm Kommune', '70960516', 'KY_0223', '223'),
Municipality('Rudersdal Kommune', '29188378', 'KY_0230', '230'),
Municipality('Egedal Kommune', '29188386', 'KY_0240', '240'),
Municipality('Frederikssund Kommune', '29189129', 'KY_0250', '250'),
Municipality('Greve Kommune', '44023911', 'KY_0253', '253'),
Municipality('Køge Kommune', '29189374', 'KY_0259', '259'),
Municipality('Halsnæs Kommune', '29188416', 'KY_0260', '260'),
Municipality('Roskilde Kommune', '29189404', 'KY_0265', '265'),
Municipality('Solrød Kommune', '68534917', 'KY_0269', '269'),
Municipality('Gribskov Kommune', '29188440', 'KY_0270', '270'),
Municipality('Odsherred Kommune', '29188459', 'KY_0306', '306'),
Municipality('Holbæk Kommune', '29189447', 'KY_0316', '316'),
Municipality('Faxe Kommune', '29188475', 'KY_0320', '320'),
Municipality('Kalundborg Kommune', '29189595', 'KY_0326', '326'),
Municipality('Ringsted Kommune', '18957981', 'KY_0329', '329'),
Municipality('Slagelse Kommune', '29188505', 'KY_0330', '330'),
Municipality('Stevns Kommune', '29208654', 'KY_0336', '336'),
Municipality('Sorø Kommune', '29189994', 'KY_0340', '340'),
Municipality('Lejre Kommune', '29188548', 'KY_0350', '350'),
Municipality('Lolland Kommune', '29188572', 'KY_0360', '360'),
Municipality('Næstved Kommune', '29189625', 'KY_0370', '370'),
Municipality('Guldborgsund Kommune', '29188599', 'KY_0376', '376'),
Municipality('Vordingborg Kommune', '29189676', 'KY_0390', '390'),
Municipality('Bornholms Regionskommune', '26696348', 'KY_0400', '400'),
Municipality('Middelfart Kommune', '29189684', 'KY_0410', '410'),
Municipality('Assens Kommune', '29189692', 'KY_0420', '420'),
Municipality('Faaborg-Midtfyn Kommune', '29188645', 'KY_0430', '430'),
Municipality('Kerteminde Kommune', '29189706', 'KY_0440', '440'),
Municipality('Nyborg Kommune', '29189722', 'KY_0450', '450'),
]
#db_adapter.run_sql_file_multiple_statements()
db_adapter = DBAdapter(keepass, config.database, logger)
db_adapter.run_sql_file_export_to_file_multiple_schemas(municipalities=municipalities)
# db_adapter.run_sql_file_multiple_statements()

55
config.py → models/DatabaseConfig.py
View File

@ -1,15 +1,6 @@
import logging from models.DatabaseType import DatabaseType
from enum import Enum
import toml
from dataclasses import dataclass from dataclasses import dataclass
class DatabaseType(Enum):
PSQL = 1
ORCL = 2
SQLITE = 3
@dataclass @dataclass
class DatabaseConfig: class DatabaseConfig:
_host: str _host: str
@ -20,7 +11,7 @@ class DatabaseConfig:
def __init__(self, config: dict): def __init__(self, config: dict):
self._host = config["HOST"] self._host = config["HOST"]
type = config["DATABASE_TYPE"] type = config["DATABASE_TYPE"]
match type: match type:
case 'PSQL': case 'PSQL':
@ -69,45 +60,3 @@ class DatabaseConfig:
def port(self) -> str: def port(self) -> str:
return self._database_port return self._database_port
@dataclass
class KeePassConfig:
_path: str
_db_credentials_name: str
_db_credentials_group: str
def __init__(self, config: dict):
self._path: str = config["PATH"]
self._db_credentials_name: str = config["DB_CREDENTIALS_NAME"]
self._db_credentials_group: str = config["DB_CREDENTIALS_GROUP"]
@property
def path(self) -> str:
return self._path
@property
def db_credentials_name(self) -> str:
return self._db_credentials_name
@property
def db_credentials_group(self) -> str:
return self._db_credentials_group
@dataclass
class Config:
_config: dict
_kee_pass_config: KeePassConfig
_database_config: DatabaseConfig
def __init__(self):
with open('config.toml', 'r') as f:
self._config: dict = toml.load(f)
self._kee_pass_config: KeePassConfig = KeePassConfig(self._config['keepass'])
self._database_config: DatabaseConfig = DatabaseConfig(self._config['database'])
@property
def kee_pass(self) -> KeePassConfig:
return self._kee_pass_config
@property
def database(self) -> DatabaseConfig:
return self._database_config

7
models/DatabaseType.py Normal file
View File

@ -0,0 +1,7 @@
from enum import Enum
class DatabaseType(Enum):
PSQL = 1
ORCL = 2
SQLITE = 3

7
models/ExportType.py Normal file
View File

@ -0,0 +1,7 @@
from enum import Enum
class ExportType(Enum):
XML = 3
CSV = 1
EXCEL = 2

25
models/KeePassConfig.py Normal file
View File

@ -0,0 +1,25 @@
from dataclasses import dataclass
@dataclass
class KeePassConfig:
_path: str
_db_credentials_name: str
_db_credentials_group: str
def __init__(self, config: dict):
self._path: str = config["PATH"]
self._db_credentials_name: str = config["DB_CREDENTIALS_NAME"]
self._db_credentials_group: str = config["DB_CREDENTIALS_GROUP"]
@property
def path(self) -> str:
return self._path
@property
def db_credentials_name(self) -> str:
return self._db_credentials_name
@property
def db_credentials_group(self) -> str:
return self._db_credentials_group

10
models.py → models/Municipality.py
View File

@ -1,12 +1,6 @@
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum
class ExportType(Enum):
XML = 3
CSV = 1
EXCEL = 2
@dataclass @dataclass
class Municipality: class Municipality:
@ -30,4 +24,4 @@ class Municipality:
@property @property
def kommunekode(self): def kommunekode(self):
return self._kommunekode return self._kommunekode

0
models/__init__.py Normal file
View File

1
pyproject.toml
View File

@ -6,6 +6,7 @@ readme = "README.md"
requires-python = ">=3.13" requires-python = ">=3.13"
dependencies = [ dependencies = [
"cx-oracle>=8.3.0", "cx-oracle>=8.3.0",
"notebook>=7.3.2",
"openpyxl>=3.1.5", "openpyxl>=3.1.5",
"pandas>=2.2.3", "pandas>=2.2.3",
"pg8000>=1.31.2", "pg8000>=1.31.2",

3
query.sql
View File

@ -1 +1,2 @@
SELECT 1 FROM DUAL; SELECT 1
FROM DUAL;

1154
uv.lock generated
View File

File diff suppressed because it is too large Load Diff