personal-finance/budget
2
1
Fork 0
Code Issues 11 Pull Requests Projects 6 Releases 2 Wiki Activity

Adds new Manager that will handle components

Browse Source 
Move from a direct access to DB by the parsers/categorizers to a middle
layer, which will bring an easier way to have two input alternatives.
This patch starts by instantiating the manager on the cli runnable and
using it for the parser function.

This patch also moves the queries to a different file, so that
introducing new functions on the DB client becomes more manageable and
clearer.

Finally, the new manager will need converters to move from the code type
Transaction to the DB types. This will eventually simplify the code data
model by removing more of its method and leaving it a simple dataclass.

Issue #14
This commit is contained in:
Luís Murta 2022-09-25 13:27:08 +01:00
parent 4b5773aa13
commit c6cfd52b8b
Signed by: satprog
GPG Key ID: 169EF1BBD7049F94
9 changed files with 172 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pfbudget/cli/runnable.py
View File

@ -2,9 +2,9 @@ from pathlib import Path
import argparse
import re
from pfbudget.core.manager import Manager
from pfbudget.core.categories import categorize_data
from pfbudget.core.input.parsers import parse_data
from pfbudget.db.client import DBManager
from pfbudget.db.client import DatabaseClient
import pfbudget.reporting.graph
import pfbudget.reporting.report
import pfbudget.utils
@ -12,7 +12,7 @@ import pfbudget.utils
from pfbudget.core.input.nordigen import Client
DEFAULT_DB = "data.db"
DEFAULT_DB = "stub.db"
class PfBudgetInitialized(Exception):
@ -27,7 +27,8 @@ class DataFileMissing(Exception):
pass
def argparser() -> argparse.ArgumentParser:
def argparser(manager: Manager) -> argparse.ArgumentParser:
help = argparse.ArgumentParser(add_help=False)
help.add_argument(
"-db",
@ -73,7 +74,7 @@ def argparser() -> argparse.ArgumentParser:
parents=[help],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p_init.set_defaults(func=lambda args: DBManager(args.database).init())
p_init.set_defaults(func=lambda args: DatabaseClient(args.database).init())
"""
Exporting
@ -84,7 +85,7 @@ def argparser() -> argparse.ArgumentParser:
parents=[help],
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p_export.set_defaults(func=lambda args: DBManager(args.database).export())
p_export.set_defaults(func=lambda args: DatabaseClient(args.database).export())
"""
Parsing
@ -99,7 +100,7 @@ def argparser() -> argparse.ArgumentParser:
p_parse.add_argument("--bank", nargs=1, type=str)
p_parse.add_argument("--creditcard", nargs=1, type=str)
p_parse.add_argument("--category", nargs=1, type=int)
p_parse.set_defaults(func=parse)
p_parse.set_defaults(func=lambda args: parse(manager, args))
"""
Categorizing
@ -111,7 +112,7 @@ def argparser() -> argparse.ArgumentParser:
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
p_categorize.set_defaults(
func=lambda args: categorize_data(DBManager(args.database))
func=lambda args: categorize_data(DatabaseClient(args.database))
)
"""
@ -206,19 +207,18 @@ def argparser() -> argparse.ArgumentParser:
return parser
def parse(args):
def parse(manager: Manager, args):
"""Parses the contents of the path in args to the selected database.
Args:
args (dict): argparse variables
"""
db = DBManager(args.database)
for path in args.path:
if (dir := Path(path)).is_dir():
for file in dir.iterdir():
parse_data(db, file, vars(args))
manager.parse(file, vars(args))
elif Path(path).is_file():
parse_data(db, path, vars(args))
manager.parse(path, vars(args))
else:
raise FileNotFoundError
@ -231,11 +231,11 @@ def graph(args):
"""
start, end = pfbudget.utils.parse_args_period(args)
if args.option == "monthly":
pfbudget.reporting.graph.monthly(DBManager(args.database), vars(args), start, end)
pfbudget.reporting.graph.monthly(DatabaseClient(args.database), vars(args), start, end)
elif args.option == "discrete":
pfbudget.reporting.graph.discrete(DBManager(args.database), vars(args), start, end)
pfbudget.reporting.graph.discrete(DatabaseClient(args.database), vars(args), start, end)
elif args.option == "networth":
pfbudget.reporting.graph.networth(DBManager(args.database), vars(args), start, end)
pfbudget.reporting.graph.networth(DatabaseClient(args.database), vars(args), start, end)
def report(args):
@ -246,11 +246,12 @@ def report(args):
"""
start, end = pfbudget.utils.parse_args_period(args)
if args.option == "net":
pfbudget.reporting.report.net(DBManager(args.database), start, end)
pfbudget.reporting.report.net(DatabaseClient(args.database), start, end)
elif args.option == "detailed":
pfbudget.reporting.report.detailed(DBManager(args.database), start, end)
pfbudget.reporting.report.detailed(DatabaseClient(args.database), start, end)
def run():
args = argparser().parse_args()
manager = Manager(DEFAULT_DB)
args = argparser(manager).parse_args()
args.func(args)

8
pfbudget/core/categories.py
View File

@ -8,7 +8,7 @@ import yaml
if TYPE_CHECKING:
from pfbudget.db.manager import DBManager
from pfbudget.db.client import DatabaseClient
from pfbudget.core.transactions import Transaction
@ -53,7 +53,7 @@ groups = {
}
def categorize_data(db: DBManager):
def categorize_data(db: DatabaseClient):
# 1st) Classifying null transactions, i.e. transfers between banks.
# Will not overwrite previous categories
@ -96,7 +96,7 @@ def categorize_data(db: DBManager):
break
def vacations(db: DBManager) -> None:
def vacations(db: DatabaseClient) -> None:
try:
date_fmt = categories["Travel"].date_fmt
for start, end in categories["Travel"].vacations:
@ -134,7 +134,7 @@ def vacations(db: DBManager) -> None:
logging.exception(e)
def nulls(db: DBManager) -> None:
def nulls(db: DatabaseClient) -> None:
null = categories.get("Null", Options())
transactions = db.get_uncategorized_transactions()
if not transactions:

4
pfbudget/core/input/parsers.py
View File

@ -43,7 +43,7 @@ Options = namedtuple(
)
def parse_data(db: DBManager, filename: str, args: dict) -> None:
def parse_data(filename: str, args: dict) -> None:
cfg: dict = yaml.safe_load(open("parsers.yaml"))
assert (
"Banks" in cfg
@ -72,7 +72,7 @@ def parse_data(db: DBManager, filename: str, args: dict) -> None:
else:
transactions = Parser(filename, bank, options).parse()
db.insert_transactions(transactions)
return transactions
class Parser:

20
pfbudget/core/manager.py Normal file
View File

@ -0,0 +1,20 @@
from pfbudget.core.input.parsers import parse_data
from pfbudget.core.transactions import Transaction
from pfbudget.db.client import DatabaseClient
from pfbudget.utils.converters import convert
class Manager:
def __init__(self, db: str):
self.__db = DatabaseClient(db)
def transactions() -> list[Transaction]:
pass
def add_transactions(self, transactions: list[Transaction]):
converted = convert(transactions)
self.__db.insert_transactions(converted)
def parse(self, filename: str, args: dict):
transactions = parse_data(filename, args)
self.add_transactions(transactions)

120
pfbudget/db/manager.py → pfbudget/db/client.py
View File

@ -7,7 +7,8 @@ import logging.config
import pathlib
import sqlite3
from ..core.transactions import Transaction
from pfbudget.core.transactions import Transaction
import pfbudget.db.schema as Q
if not pathlib.Path("logs").is_dir():
@ -19,94 +20,8 @@ sqlite3.register_adapter(Decimal, lambda d: float(d))
__DB_NAME = "data.db"
CREATE_TRANSACTIONS_TABLE = """
CREATE TABLE IF NOT EXISTS "transactions" (
"date" TEXT NOT NULL,
"description" TEXT,
"bank" TEXT NOT NULL,
"value" REAL NOT NULL,
"category" TEXT,
"original" TEXT,
"additional comments" TEXT
);
"""
CREATE_BACKUPS_TABLE = """
CREATE TABLE IF NOT EXISTS backups (
datetime TEXT NOT NULL,
file TEXT NOT NULL
)
"""
CREATE_BANKS_TABLE = """
CREATE TABLE banks (
name TEXT NOT NULL PRIMARY KEY,
url TEXT
)
"""
ADD_TRANSACTION = """
INSERT INTO transactions (date, description, bank, value, category) values (?,?,?,?,?)
"""
UPDATE_CATEGORY = """
UPDATE transactions
SET category = (?)
WHERE date = (?) AND description = (?) AND bank = (?) AND value = (?)
"""
DUPLICATED_TRANSACTIONS = """
SELECT COUNT(*), date, description, bank, value
FROM transactions
GROUP BY date, description, bank, value
HAVING COUNT(*) > 1
ORDER BY date ASC
"""
SORTED_TRANSACTIONS = """
SELECT *
FROM transactions
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BY_CATEGORY = """
SELECT *
FROM transactions
WHERE category IS (?)
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
AND category IS (?)
ORDER BY date ASC
"""
SELECT_TRANSACTION_BY_PERIOD = """
SELECT EXTRACT((?) FROM date) AS (?), date, description, bank, value
FROM transactions
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
AND category NOT IN {}
ORDER BY date ASC
"""
class DBManager:
class DatabaseClient:
"""SQLite DB connection manager"""
__EXPORT_DIR = "export"
@ -160,8 +75,8 @@ class DBManager:
logging.info(f"Initializing {self.db} database")
self.__create_tables(
(
("transactions", CREATE_TRANSACTIONS_TABLE),
("backups", CREATE_BACKUPS_TABLE),
("transactions", Q.CREATE_TRANSACTIONS_TABLE),
("backups", Q.CREATE_BACKUPS_TABLE),
)
)
@ -174,48 +89,47 @@ class DBManager:
def insert_transaction(self, transaction: Transaction):
logger.info(f"Adding {transaction} into {self.db}")
self.__execute(ADD_TRANSACTION, (transaction.to_list(),))
self.__execute(Q.ADD_TRANSACTION, (transaction.to_list(),))
def insert_transactions(self, transactions: list[Transaction]):
def insert_transactions(self, transactions: list[list]):
logger.info(f"Adding {len(transactions)} into {self.db}")
transactions = [t.to_list() for t in transactions]
self.__executemany(ADD_TRANSACTION, transactions)
self.__executemany(Q.ADD_TRANSACTION, transactions)
def update_category(self, transaction: Transaction):
logger.info(f"Update {transaction} category")
self.__execute(UPDATE_CATEGORY, transaction.update_category())
self.__execute(Q.UPDATE_CATEGORY, transaction.update_category())
def update_categories(self, transactions: list[Transaction]):
logger.info(f"Update {len(transactions)} transactions' categories")
self.__executemany(
UPDATE_CATEGORY,
Q.UPDATE_CATEGORY,
[transaction.update_category() for transaction in transactions],
)
def get_duplicated_transactions(self) -> list[Transaction] | None:
logger.info("Get duplicated transactions")
transactions = self.__execute(DUPLICATED_TRANSACTIONS)
transactions = self.__execute(Q.DUPLICATED_TRANSACTIONS)
if transactions:
return [Transaction(t) for t in transactions]
return None
def get_sorted_transactions(self) -> list[Transaction] | None:
logger.info("Get transactions sorted by date")
transactions = self.__execute(SORTED_TRANSACTIONS)
transactions = self.__execute(Q.SORTED_TRANSACTIONS)
if transactions:
return [Transaction(t) for t in transactions]
return None
def get_daterange(self, start: datetime, end: datetime) -> list[Transaction] | None:
logger.info(f"Get transactions from {start} to {end}")
transactions = self.__execute(SELECT_TRANSACTIONS_BETWEEN_DATES, (start, end))
transactions = self.__execute(Q.SELECT_TRANSACTIONS_BETWEEN_DATES, (start, end))
if transactions:
return [Transaction(t) for t in transactions]
return None
def get_category(self, value: str) -> list[Transaction] | None:
logger.info(f"Get transactions where category = {value}")
transactions = self.__execute(SELECT_TRANSACTIONS_BY_CATEGORY, (value,))
transactions = self.__execute(Q.SELECT_TRANSACTIONS_BY_CATEGORY, (value,))
if transactions:
return [Transaction(t) for t in transactions]
return None
@ -227,7 +141,7 @@ class DBManager:
f"Get transactions from {start} to {end} where category = {category}"
)
transactions = self.__execute(
SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY, (start, end, category)
Q.SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY, (start, end, category)
)
if transactions:
return [Transaction(t) for t in transactions]
@ -235,7 +149,7 @@ class DBManager:
def get_by_period(self, period: str) -> list[Transaction] | None:
logger.info(f"Get transactions by {period}")
transactions = self.__execute(SELECT_TRANSACTION_BY_PERIOD, period)
transactions = self.__execute(Q.SELECT_TRANSACTION_BY_PERIOD, period)
if transactions:
return [Transaction(t) for t in transactions]
return None
@ -252,7 +166,7 @@ class DBManager:
self, start: datetime, end: datetime, *categories: str
) -> list[Transaction] | None:
logger.info(f"Get transactions between {start} and {end} not in {categories}")
query = SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES.format(
query = Q.SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES.format(
"(" + ", ".join("?" for _ in categories) + ")"
)
transactions = self.__execute(query, (start, end, *categories))

85
pfbudget/db/schema.py Normal file
View File

@ -0,0 +1,85 @@
CREATE_TRANSACTIONS_TABLE = """
CREATE TABLE IF NOT EXISTS "transactions" (
"date" TEXT NOT NULL,
"description" TEXT,
"bank" TEXT NOT NULL,
"value" REAL NOT NULL,
"category" TEXT,
"original" TEXT,
"additional comments" TEXT
);
"""
CREATE_BACKUPS_TABLE = """
CREATE TABLE IF NOT EXISTS backups (
datetime TEXT NOT NULL,
file TEXT NOT NULL
)
"""
CREATE_BANKS_TABLE = """
CREATE TABLE banks (
name TEXT NOT NULL PRIMARY KEY,
url TEXT
)
"""
ADD_TRANSACTION = """
INSERT INTO transactions (date, description, bank, value, category) values (?,?,?,?,?)
"""
UPDATE_CATEGORY = """
UPDATE transactions
SET category = (?)
WHERE date = (?) AND description = (?) AND bank = (?) AND value = (?)
"""
DUPLICATED_TRANSACTIONS = """
SELECT COUNT(*), date, description, bank, value
FROM transactions
GROUP BY date, description, bank, value
HAVING COUNT(*) > 1
ORDER BY date ASC
"""
SORTED_TRANSACTIONS = """
SELECT *
FROM transactions
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BY_CATEGORY = """
SELECT *
FROM transactions
WHERE category IS (?)
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES_WITH_CATEGORY = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
AND category IS (?)
ORDER BY date ASC
"""
SELECT_TRANSACTION_BY_PERIOD = """
SELECT EXTRACT((?) FROM date) AS (?), date, description, bank, value
FROM transactions
ORDER BY date ASC
"""
SELECT_TRANSACTIONS_BETWEEN_DATES_WITHOUT_CATEGORIES = """
SELECT *
FROM transactions
WHERE date BETWEEN (?) AND (?)
AND category NOT IN {}
ORDER BY date ASC
"""

8
pfbudget/reporting/graph.py
View File

@ -9,14 +9,14 @@ import pfbudget.core.categories
if TYPE_CHECKING:
from pfbudget.db.manager import DBManager
from pfbudget.db.client import DatabaseClient
groups = pfbudget.core.categories.cfg["Groups"]
def monthly(
db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
):
transactions = db.get_daterange(start, end)
start, end = transactions[0].date, transactions[-1].date
@ -96,7 +96,7 @@ def monthly(
def discrete(
db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
):
transactions = db.get_daterange(start, end)
start, end = transactions[0].date, transactions[-1].date
@ -180,7 +180,7 @@ def discrete(
def networth(
db: DBManager, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
db: DatabaseClient, args: dict, start: dt.date = dt.date.min, end: dt.date = dt.date.max
):
transactions = db.get_daterange(start, end)
start, end = transactions[0].date, transactions[-1].date

6
pfbudget/reporting/report.py
View File

@ -6,10 +6,10 @@ import datetime as dt
import pfbudget.core.categories
if TYPE_CHECKING:
from pfbudget.db.manager import DBManager
from pfbudget.db.client import DatabaseClient
def net(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max):
def net(db: DatabaseClient, start: dt.date = dt.date.min, end: dt.date = dt.date.max):
transactions = db.get_daterange(start, end)
start, end = transactions[0].date, transactions[-1].date
@ -63,7 +63,7 @@ def net(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max)
print(f"Invested: {investments:.2f}€\n")
def detailed(db: DBManager, start: dt.date = dt.date.min, end: dt.date = dt.date.max):
def detailed(db: DatabaseClient, start: dt.date = dt.date.min, end: dt.date = dt.date.max):
transactions = db.get_daterange(start, end)
start, end = transactions[0].date, transactions[-1].date

18
pfbudget/utils/converters.py Normal file
View File

@ -0,0 +1,18 @@
from functools import singledispatch
from pfbudget.core.transactions import Transaction
@singledispatch
def convert(t):
pass
@convert.register
def _(t: Transaction) -> list:
return (t.date, t.description, t.bank, t.value, t.category)
@convert.register
def _(transactions: list) -> list[list]:
return [convert(c) for c in transactions]