From c6cfd52b8b63d0a10975813718043e85de3511b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 25 Sep 2022 13:27:08 +0100 Subject: [PATCH] Adds new Manager that will handle components Move from a direct access to DB by the parsers/categorizers to a middle layer, which will bring an easier way to have two input alternatives. This patch starts by instantiating the manager on the cli runnable and using it for the parser function. This patch also moves the queries to a different file, so that introducing new functions on the DB client becomes more manageable and clearer. Finally, the new manager will need converters to move from the code type Transaction to the DB types. This will eventually simplify the code data model by removing more of its method and leaving it a simple dataclass. Issue #14 --- pfbudget/cli/runnable.py | 37 ++++---- pfbudget/core/categories.py | 8 +- pfbudget/core/input/parsers.py | 4 +- pfbudget/core/manager.py | 20 +++++ pfbudget/db/{manager.py => client.py} | 120 ++++---------------------- pfbudget/db/schema.py | 85 ++++++++++++++++++ pfbudget/reporting/graph.py | 8 +- pfbudget/reporting/report.py | 6 +- pfbudget/utils/converters.py | 18 ++++ 9 files changed, 172 insertions(+), 134 deletions(-) create mode 100644 pfbudget/core/manager.py rename pfbudget/db/{manager.py => client.py} (68%) create mode 100644 pfbudget/db/schema.py create mode 100644 pfbudget/utils/converters.py diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 63f860d..378b54a 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -2,9 +2,9 @@ from pathlib import Path import argparse import re +from pfbudget.core.manager import Manager from pfbudget.core.categories import categorize_data -from pfbudget.core.input.parsers import parse_data -from pfbudget.db.client import DBManager +from pfbudget.db.client import DatabaseClient import pfbudget.reporting.graph import pfbudget.reporting.report import pfbudget.utils @@ -12,7 +12,7 @@ import pfbudget.utils from pfbudget.core.input.nordigen import Client -DEFAULT_DB = "data.db" +DEFAULT_DB = "stub.db" class PfBudgetInitialized(Exception): @@ -27,7 +27,8 @@ class DataFileMissing(Exception): pass -def argparser() -> argparse.ArgumentParser: +def argparser(manager: Manager) -> argparse.ArgumentParser: + help = argparse.ArgumentParser(add_help=False) help.add_argument( "-db", @@ -73,7 +74,7 @@ def argparser() -> argparse.ArgumentParser: parents=[help], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_init.set_defaults(func=lambda args: DBManager(args.database).init()) + p_init.set_defaults(func=lambda args: DatabaseClient(args.database).init()) """ Exporting @@ -84,7 +85,7 @@ def argparser() -> argparse.ArgumentParser: parents=[help], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_export.set_defaults(func=lambda args: DBManager(args.database).export()) + p_export.set_defaults(func=lambda args: DatabaseClient(args.database).export()) """ Parsing @@ -99,7 +100,7 @@ def argparser() -> argparse.ArgumentParser: p_parse.add_argument("--bank", nargs=1, type=str) p_parse.add_argument("--creditcard", nargs=1, type=str) p_parse.add_argument("--category", nargs=1, type=int) - p_parse.set_defaults(func=parse) + p_parse.set_defaults(func=lambda args: parse(manager, args)) """ Categorizing @@ -111,7 +112,7 @@ def argparser() -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_categorize.set_defaults( - func=lambda args: categorize_data(DBManager(args.database)) + func=lambda args: categorize_data(DatabaseClient(args.database)) ) """ @@ -206,19 +207,18 @@ def argparser() -> argparse.ArgumentParser: return parser -def parse(args): +def parse(manager: Manager, args): """Parses the contents of the path in args to the selected database. Args: args (dict): argparse variables """ - db = DBManager(args.database) for path in args.path: if (dir := Path(path)).is_dir(): for file in dir.iterdir(): - parse_data(db, file, vars(args)) + manager.parse(file, vars(args)) elif Path(path).is_file(): - parse_data(db, path, vars(args)) + manager.parse(path, vars(args)) else: raise FileNotFoundError @@ -231,11 +231,11 @@ def graph(args): """ start, end = pfbudget.utils.parse_args_period(args) if args.option == "monthly": - pfbudget.reporting.graph.monthly(DBManager(args.database), vars(args), start, end) + pfbudget.reporting.graph.monthly(DatabaseClient(args.database), vars(args), start, end) elif args.option == "discrete": - pfbudget.reporting.graph.discrete(DBManager(args.database), vars(args), start, end) + pfbudget.reporting.graph.discrete(DatabaseClient(args.database), vars(args), start, end) elif args.option == "networth": - pfbudget.reporting.graph.networth(DBManager(args.database), vars(args), start, end) + pfbudget.reporting.graph.networth(DatabaseClient(args.database), vars(args), start, end) def report(args): @@ -246,11 +246,12 @@ def report(args): """ start, end = pfbudget.utils.parse_args_period(args) if args.option == "net": - pfbudget.reporting.report.net(DBManager(args.database), start, end) + pfbudget.reporting.report.net(DatabaseClient(args.database), start, end) elif args.option == "detailed": - pfbudget.reporting.report.detailed(DBManager(args.database), start, end) + pfbudget.reporting.report.detailed(DatabaseClient(args.database), start, end) def run(): - args = argparser().parse_args() + manager = Manager(DEFAULT_DB) + args = argparser(manager).parse_args() args.func(args) diff --git a/pfbudget/core/categories.py b/pfbudget/core/categories.py index 4af5abc..3d29a86 100644 --- a/pfbudget/core/categories.py +++ b/pfbudget/core/categories.py @@ -8,7 +8,7 @@ import yaml if TYPE_CHECKING: - from pfbudget.db.manager import DBManager + from pfbudget.db.client import DatabaseClient from pfbudget.core.transactions import Transaction @@ -53,7 +53,7 @@ groups = { } -def categorize_data(db: DBManager): +def categorize_data(db: DatabaseClient): # 1st) Classifying null transactions, i.e. transfers between banks. # Will not overwrite previous categories @@ -96,7 +96,7 @@ def categorize_data(db: DBManager): break -def vacations(db: DBManager) -> None: +def vacations(db: DatabaseClient) -> None: try: date_fmt = categories["Travel"].date_fmt for start, end in categories["Travel"].vacations: @@ -134,7 +134,7 @@ def vacations(db: DBManager) -> None: logging.exception(e) -def nulls(db: DBManager) -> None: +def nulls(db: DatabaseClient) -> None: null = categories.get("Null", Options()) transactions = db.get_uncategorized_transactions() if not transactions: diff --git a/pfbudget/core/input/parsers.py b/pfbudget/core/input/parsers.py index f45d1c2..5bfaf1a 100644 --- a/pfbudget/core/input/parsers.py +++ b/pfbudget/core/input/parsers.py @@ -43,7 +43,7 @@ Options = namedtuple( ) -def parse_data(db: DBManager, filename: str, args: dict) -> None: +def parse_data(filename: str, args: dict) -> None: cfg: dict = yaml.safe_load(open("parsers.yaml")) assert ( "Banks" in cfg @@ -72,7 +72,7 @@ def parse_data(db: DBManager, filename: str, args: dict) -> None: else: transactions = Parser(filename, bank, options).parse() - db.insert_transactions(transactions) + return transactions class Parser: diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py new file mode 100644 index 0000000..6faab87 --- /dev/null +++ b/pfbudget/core/manager.py @@ -0,0 +1,20 @@ +from pfbudget.core.input.parsers import parse_data +from pfbudget.core.transactions import Transaction +from pfbudget.db.client import DatabaseClient +from pfbudget.utils.converters import convert + + +class Manager: + def __init__(self, db: str): + self.__db = DatabaseClient(db) + + def transactions() -> list[Transaction]: + pass + + def add_transactions(self, transactions: list[Transaction]): + converted = convert(transactions) + self.__db.insert_transactions(converted) + + def parse(self, filename: str, args: dict): + transactions = parse_data(filename, args) + self.add_transactions(transactions) diff --git a/pfbudget/db/manager.py b/pfbudget/db/client.py similarity index 68% rename from pfbudget/db/manager.py rename to pfbudget/db/client.py index ad6ad8a..d6084bb 100644 --- a/pfbudget/db/manager.py +++ b/pfbudget/db/client.py @@ -7,7 +7,8 @@ import logging.config import pathlib import sqlite3 -from ..core.transactions import Transaction +from pfbudget.core.transactions import Transaction +import pfbudget.db.schema as Q if not pathlib.Path("logs").is_dir(): @@ -19,94 +20,8 @@ sqlite3.register_adapter(Decimal, lambda d: float(d)) __DB_NAME = "data.db" -CREATE_TRANSACTIONS_TABLE = """ -CREATE TABLE IF NOT EXISTS "transactions" ( - "date" TEXT NOT NULL, - "description" TEXT, - "bank" TEXT NOT NULL, - "value" REAL NOT NULL, - "category" TEXT, - "original" TEXT, - "additional comments" TEXT -); -""" -CREATE_BACKUPS_TABLE = """ -CREATE TABLE IF NOT EXISTS backups ( - datetime TEXT NOT NULL, - file TEXT NOT NULL -) -""" - -CREATE_BANKS_TABLE = """ -CREATE TABLE banks ( - name TEXT NOT NULL PRIMARY KEY, - url TEXT -) -""" - -ADD_TRANSACTION = """ -INSERT INTO transactions (date, description, bank, value, category) values (?,?,?,?,?) -""" - -UPDATE_CATEGORY = """ -UPDATE transactions -SET category = (?) -WHERE date = (?) AND description = (?) AND bank = (?) AND value = (?) -""" - -DUPLICATED_TRANSACTIONS = """ -SELECT COUNT(*), date, description, bank, value -FROM transactions -GROUP BY date, description, bank, value -HAVING COUNT(*) > 1 -ORDER BY date ASC -""" - -SORTED_TRANSACTIONS = """ -SELECT * -FROM transactions -ORDER BY date ASC -""" - -SELECT_TRANSACTIONS_BETWEEN_DATES = """ -SELECT * -FROM transactions -WHERE date BETWEEN (?) AND (?) -ORDER BY date ASC -""" - -SELECT_TRANSACTIONS_BY_CATEGORY = """ -SELECT * -FROM transactions -WHERE category IS (?) -ORDER BY date ASC -""" - -SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY = """ -SELECT * -FROM transactions -WHERE date BETWEEN (?) AND (?) -AND category IS (?) -ORDER BY date ASC -""" - -SELECT_TRANSACTION_BY_PERIOD = """ -SELECT EXTRACT((?) FROM date) AS (?), date, description, bank, value -FROM transactions -ORDER BY date ASC -""" - -SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES = """ -SELECT * -FROM transactions -WHERE date BETWEEN (?) AND (?) -AND category NOT IN {} -ORDER BY date ASC -""" - - -class DBManager: +class DatabaseClient: """SQLite DB connection manager""" __EXPORT_DIR = "export" @@ -160,8 +75,8 @@ class DBManager: logging.info(f"Initializing {self.db} database") self.__create_tables( ( - ("transactions", CREATE_TRANSACTIONS_TABLE), - ("backups", CREATE_BACKUPS_TABLE), + ("transactions", Q.CREATE_TRANSACTIONS_TABLE), + ("backups", Q.CREATE_BACKUPS_TABLE), ) ) @@ -174,48 +89,47 @@ class DBManager: def insert_transaction(self, transaction: Transaction): logger.info(f"Adding {transaction} into {self.db}") - self.__execute(ADD_TRANSACTION, (transaction.to_list(),)) + self.__execute(Q.ADD_TRANSACTION, (transaction.to_list(),)) - def insert_transactions(self, transactions: list[Transaction]): + def insert_transactions(self, transactions: list[list]): logger.info(f"Adding {len(transactions)} into {self.db}") - transactions = [t.to_list() for t in transactions] - self.__executemany(ADD_TRANSACTION, transactions) + self.__executemany(Q.ADD_TRANSACTION, transactions) def update_category(self, transaction: Transaction): logger.info(f"Update {transaction} category") - self.__execute(UPDATE_CATEGORY, transaction.update_category()) + self.__execute(Q.UPDATE_CATEGORY, transaction.update_category()) def update_categories(self, transactions: list[Transaction]): logger.info(f"Update {len(transactions)} transactions' categories") self.__executemany( - UPDATE_CATEGORY, + Q.UPDATE_CATEGORY, [transaction.update_category() for transaction in transactions], ) def get_duplicated_transactions(self) -> list[Transaction] | None: logger.info("Get duplicated transactions") - transactions = self.__execute(DUPLICATED_TRANSACTIONS) + transactions = self.__execute(Q.DUPLICATED_TRANSACTIONS) if transactions: return [Transaction(t) for t in transactions] return None def get_sorted_transactions(self) -> list[Transaction] | None: logger.info("Get transactions sorted by date") - transactions = self.__execute(SORTED_TRANSACTIONS) + transactions = self.__execute(Q.SORTED_TRANSACTIONS) if transactions: return [Transaction(t) for t in transactions] return None def get_daterange(self, start: datetime, end: datetime) -> list[Transaction] | None: logger.info(f"Get transactions from {start} to {end}") - transactions = self.__execute(SELECT_TRANSACTIONS_BETWEEN_DATES, (start, end)) + transactions = self.__execute(Q.SELECT_TRANSACTIONS_BETWEEN_DATES, (start, end)) if transactions: return [Transaction(t) for t in transactions] return None def get_category(self, value: str) -> list[Transaction] | None: logger.info(f"Get transactions where category = {value}") - transactions = self.__execute(SELECT_TRANSACTIONS_BY_CATEGORY, (value,)) + transactions = self.__execute(Q.SELECT_TRANSACTIONS_BY_CATEGORY, (value,)) if transactions: return [Transaction(t) for t in transactions] return None @@ -227,7 +141,7 @@ class DBManager: f"Get transactions from {start} to {end} where category = {category}" ) transactions = self.__execute( - SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY, (start, end, category) + Q.SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY, (start, end, category) ) if transactions: return [Transaction(t) for t in transactions] @@ -235,7 +149,7 @@ class DBManager: def get_by_period(self, period: str) -> list[Transaction] | None: logger.info(f"Get transactions by {period}") - transactions = self.__execute(SELECT_TRANSACTION_BY_PERIOD, period) + transactions = self.__execute(Q.SELECT_TRANSACTION_BY_PERIOD, period) if transactions: return [Transaction(t) for t in transactions] return None @@ -252,7 +166,7 @@ class DBManager: self, start: datetime, end: datetime, *categories: str ) -> list[Transaction] | None: logger.info(f"Get transactions between {start} and {end} not in {categories}") - query = SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES.format( + query = Q.SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES.format( "(" + ", ".join("?" for _ in categories) + ")" ) transactions = self.__execute(query, (start, end, *categories)) diff --git a/pfbudget/db/schema.py b/pfbudget/db/schema.py new file mode 100644 index 0000000..80679b7 --- /dev/null +++ b/pfbudget/db/schema.py @@ -0,0 +1,85 @@ +CREATE_TRANSACTIONS_TABLE = """ +CREATE TABLE IF NOT EXISTS "transactions" ( + "date" TEXT NOT NULL, + "description" TEXT, + "bank" TEXT NOT NULL, + "value" REAL NOT NULL, + "category" TEXT, + "original" TEXT, + "additional comments" TEXT +); +""" + +CREATE_BACKUPS_TABLE = """ +CREATE TABLE IF NOT EXISTS backups ( + datetime TEXT NOT NULL, + file TEXT NOT NULL +) +""" + +CREATE_BANKS_TABLE = """ +CREATE TABLE banks ( + name TEXT NOT NULL PRIMARY KEY, + url TEXT +) +""" + +ADD_TRANSACTION = """ +INSERT INTO transactions (date, description, bank, value, category) values (?,?,?,?,?) +""" + +UPDATE_CATEGORY = """ +UPDATE transactions +SET category = (?) +WHERE date = (?) AND description = (?) AND bank = (?) AND value = (?) +""" + +DUPLICATED_TRANSACTIONS = """ +SELECT COUNT(*), date, description, bank, value +FROM transactions +GROUP BY date, description, bank, value +HAVING COUNT(*) > 1 +ORDER BY date ASC +""" + +SORTED_TRANSACTIONS = """ +SELECT * +FROM transactions +ORDER BY date ASC +""" + +SELECT_TRANSACTIONS_BETWEEN_DATES = """ +SELECT * +FROM transactions +WHERE date BETWEEN (?) AND (?) +ORDER BY date ASC +""" + +SELECT_TRANSACTIONS_BY_CATEGORY = """ +SELECT * +FROM transactions +WHERE category IS (?) +ORDER BY date ASC +""" + +SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY = """ +SELECT * +FROM transactions +WHERE date BETWEEN (?) AND (?) +AND category IS (?) +ORDER BY date ASC +""" + +SELECT_TRANSACTION_BY_PERIOD = """ +SELECT EXTRACT((?) FROM date) AS (?), date, description, bank, value +FROM transactions +ORDER BY date ASC +""" + +SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES = """ +SELECT * +FROM transactions +WHERE date BETWEEN (?) AND (?) +AND category NOT IN {} +ORDER BY date ASC +""" diff --git a/pfbudget/reporting/graph.py b/pfbudget/reporting/graph.py index 9e9ca1f..1491c89 100644 --- a/pfbudget/reporting/graph.py +++ b/pfbudget/reporting/graph.py @@ -9,14 +9,14 @@ import pfbudget.core.categories if TYPE_CHECKING: - from pfbudget.db.manager import DBManager + from pfbudget.db.client import DatabaseClient groups = pfbudget.core.categories.cfg["Groups"] def monthly( - db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max + db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max ): transactions = db.get_daterange(start, end) start, end = transactions[0].date, transactions[-1].date @@ -96,7 +96,7 @@ def monthly( def discrete( - db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max + db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max ): transactions = db.get_daterange(start, end) start, end = transactions[0].date, transactions[-1].date @@ -180,7 +180,7 @@ def discrete( def networth( - db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max + db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max ): transactions = db.get_daterange(start, end) start, end = transactions[0].date, transactions[-1].date diff --git a/pfbudget/reporting/report.py b/pfbudget/reporting/report.py index b181d6e..420b942 100644 --- a/pfbudget/reporting/report.py +++ b/pfbudget/reporting/report.py @@ -6,10 +6,10 @@ import datetime as dt import pfbudget.core.categories if TYPE_CHECKING: - from pfbudget.db.manager import DBManager + from pfbudget.db.client import DatabaseClient -def net(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max): +def net(db: DatabaseClient, start: dt.date = dt.date.min, end: dt.date = dt.date.max): transactions = db.get_daterange(start, end) start, end = transactions[0].date, transactions[-1].date @@ -63,7 +63,7 @@ def net(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max) print(f"Invested: {investments:.2f}€\n") -def detailed(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max): +def detailed(db: DatabaseClient, start: dt.date = dt.date.min, end: dt.date = dt.date.max): transactions = db.get_daterange(start, end) start, end = transactions[0].date, transactions[-1].date diff --git a/pfbudget/utils/converters.py b/pfbudget/utils/converters.py new file mode 100644 index 0000000..8d780cb --- /dev/null +++ b/pfbudget/utils/converters.py @@ -0,0 +1,18 @@ +from functools import singledispatch + +from pfbudget.core.transactions import Transaction + + +@singledispatch +def convert(t): + pass + + +@convert.register +def _(t: Transaction) -> list: + return (t.date, t.description, t.bank, t.value, t.category) + + +@convert.register +def _(transactions: list) -> list[list]: + return [convert(c) for c in transactions]