From 395576d73fcdbd820858a1033e3f4b94ff500ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 25 Nov 2022 18:25:11 +0000 Subject: [PATCH 01/66] Move SQLite client to separate file --- pfbudget/cli/runnable.py | 2 +- pfbudget/core/categories.py | 2 +- pfbudget/core/manager.py | 2 +- pfbudget/db/{client.py => sqlite.py} | 0 pfbudget/reporting/graph.py | 2 +- pfbudget/reporting/report.py | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename pfbudget/db/{client.py => sqlite.py} (100%) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 9d89d42..6778110 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -6,7 +6,7 @@ from pfbudget.core.categories import categorize_data from pfbudget.core.manager import Manager from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput -from pfbudget.db.client import DatabaseClient +from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph import pfbudget.reporting.report import pfbudget.utils diff --git a/pfbudget/core/categories.py b/pfbudget/core/categories.py index 23aa46a..82c2653 100644 --- a/pfbudget/core/categories.py +++ b/pfbudget/core/categories.py @@ -9,7 +9,7 @@ import yaml if TYPE_CHECKING: from pfbudget.common.types import Transaction - from pfbudget.db.client import DatabaseClient + from pfbudget.db.sqlite import DatabaseClient Options = namedtuple( diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 77d0dc8..d1b799d 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,7 +1,7 @@ from pfbudget.input.input import Input from pfbudget.input.parsers import parse_data from pfbudget.common.types import Bank, Banks, Transaction, Transactions -from pfbudget.db.client import DatabaseClient +from pfbudget.db.sqlite import DatabaseClient from pfbudget.utils import convert diff --git a/pfbudget/db/client.py b/pfbudget/db/sqlite.py similarity index 100% rename from pfbudget/db/client.py rename to pfbudget/db/sqlite.py diff --git a/pfbudget/reporting/graph.py b/pfbudget/reporting/graph.py index 1491c89..99666ef 100644 --- a/pfbudget/reporting/graph.py +++ b/pfbudget/reporting/graph.py @@ -9,7 +9,7 @@ import pfbudget.core.categories if TYPE_CHECKING: - from pfbudget.db.client import DatabaseClient + from pfbudget.db.sqlite import DatabaseClient groups = pfbudget.core.categories.cfg["Groups"] diff --git a/pfbudget/reporting/report.py b/pfbudget/reporting/report.py index 420b942..bacf9e9 100644 --- a/pfbudget/reporting/report.py +++ b/pfbudget/reporting/report.py @@ -6,7 +6,7 @@ import datetime as dt import pfbudget.core.categories if TYPE_CHECKING: - from pfbudget.db.client import DatabaseClient + from pfbudget.db.sqlite import DatabaseClient def net(db: DatabaseClient, start: dt.date = dt.date.min, end: dt.date = dt.date.max): From 0d22b02b3f36c348dc56feb2638131eda18b1ebc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 1 Dec 2022 00:57:32 +0000 Subject: [PATCH 02/66] Adds alembic tool for DB versioning Alembic is a lightweight database migration tool for usage with the SQLAlchemy Database Toolkit for Python. https://alembic.sqlalchemy.org/en/latest/index.html Adds first version of DB schema. --- alembic.ini | 105 ++++++++++++++++++++++++ alembic/README | 1 + alembic/env.py | 88 ++++++++++++++++++++ alembic/script.py.mako | 24 ++++++ alembic/versions/50ff1fbb8a00_init.py | 111 ++++++++++++++++++++++++++ pfbudget/db/model.py | 99 +++++++++++++++++++++++ 6 files changed, 428 insertions(+) create mode 100644 alembic.ini create mode 100644 alembic/README create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/50ff1fbb8a00_init.py create mode 100644 pfbudget/db/model.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..09a6086 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,105 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +timezone = UTC + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = postgresql://pf-budget:muster-neutron-omega@database.lan/pf-budget + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +hooks = black +black.type = console_scripts +black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..4b46b18 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,88 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +from pfbudget.db.model import Base +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def include_name(name, type_, parent_names): + if type_ == "schema": + return name in ["transactions"] + else: + return True + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + include_name=include_name, + include_schemas=True, + compare_type=True, + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..55df286 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/50ff1fbb8a00_init.py b/alembic/versions/50ff1fbb8a00_init.py new file mode 100644 index 0000000..4817884 --- /dev/null +++ b/alembic/versions/50ff1fbb8a00_init.py @@ -0,0 +1,111 @@ +"""Init + +Revision ID: 50ff1fbb8a00 +Revises: +Create Date: 2022-12-03 11:49:30.450115+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "50ff1fbb8a00" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "banks", + sa.Column("name", sa.String(), nullable=False), + sa.Column("BIC", sa.String(length=8), nullable=False), + sa.Column( + "type", + sa.Enum( + "checking", + "savings", + "investment", + "VISA", + "MASTERCARD", + name="accounttype", + schema="transactions", + inherit_schema=True, + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("BIC", "type", name=op.f("pk_banks")), + sa.UniqueConstraint("name", name=op.f("uq_banks_name")), + schema="transactions", + ) + op.create_table( + "nordigen", + sa.Column("name", sa.Text(), nullable=False), + sa.Column("bank_id", sa.String(), nullable=True), + sa.Column("requisition_id", sa.String(), nullable=True), + sa.Column("invert", sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint( + ["name"], ["transactions.banks.name"], name=op.f("fk_nordigen_name_banks") + ), + sa.PrimaryKeyConstraint("name", name=op.f("pk_nordigen")), + schema="transactions", + ) + op.create_table( + "originals", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("date", sa.Date(), nullable=False), + sa.Column("description", sa.String(), nullable=True), + sa.Column("bank", sa.Text(), nullable=False), + sa.Column("amount", sa.Numeric(precision=16, scale=2), nullable=False), + sa.ForeignKeyConstraint( + ["bank"], ["transactions.banks.name"], name=op.f("fk_originals_bank_banks") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_originals")), + schema="transactions", + ) + op.create_table( + "categorized", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("category", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.originals.id"], + name=op.f("fk_categorized_id_originals"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_categorized")), + schema="transactions", + ) + op.create_table( + "notes", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("note", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], ["transactions.originals.id"], name=op.f("fk_notes_id_originals") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_notes")), + schema="transactions", + ) + op.create_table( + "tags", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("tag", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], ["transactions.originals.id"], name=op.f("fk_tags_id_originals") + ), + sa.PrimaryKeyConstraint("id", "tag", name=op.f("pk_tags")), + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("tags", schema="transactions") + op.drop_table("notes", schema="transactions") + op.drop_table("categorized", schema="transactions") + op.drop_table("originals", schema="transactions") + op.drop_table("nordigen", schema="transactions") + op.drop_table("banks", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py new file mode 100644 index 0000000..492e877 --- /dev/null +++ b/pfbudget/db/model.py @@ -0,0 +1,99 @@ +from sqlalchemy import ( + BigInteger, + Enum, + ForeignKey, + MetaData, + Numeric, + String, + Text, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, MappedAsDataclass + +from decimal import Decimal +from typing import Annotated, Optional +import datetime as dt +import enum + + +class Base(MappedAsDataclass, DeclarativeBase): + __table_args__ = {"schema": "transactions"} + metadata = MetaData( + naming_convention={ + "ix": "ix_%(column_0_label)s", + "uq": "uq_%(table_name)s_%(column_0_name)s", + "ck": "ck_%(table_name)s_`%(constraint_name)s`", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", + } + ) + + +class AccountType(enum.Enum): + checking = enum.auto() + savings = enum.auto() + investment = enum.auto() + VISA = enum.auto() + MASTERCARD = enum.auto() + + +accounttype = Annotated[ + AccountType, + mapped_column(Enum(AccountType, inherit_schema=True)), +] + + +class Bank(Base): + __tablename__ = "banks" + + name: Mapped[str] = mapped_column(unique=True) + BIC: Mapped[str] = mapped_column(String(8), primary_key=True) + type: Mapped[accounttype] = mapped_column(primary_key=True) + + +bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] + +idpk = Annotated[int, mapped_column(BigInteger, primary_key=True)] +money = Annotated[Decimal, mapped_column(Numeric(16, 2), nullable=False)] + + +class Original(Base): + __tablename__ = "originals" + + id: Mapped[idpk] = mapped_column(autoincrement=True) + date: Mapped[dt.date] + description: Mapped[Optional[str]] + bank: Mapped[bankfk] + amount: Mapped[money] + + +idfk = Annotated[int, mapped_column(BigInteger, ForeignKey(Original.id))] + + +class Categorized(Base): + __tablename__ = "categorized" + + id: Mapped[idfk] = mapped_column(primary_key=True) + category: Mapped[str] + + +class Note(Base): + __tablename__ = "notes" + + id: Mapped[idfk] = mapped_column(primary_key=True) + note: Mapped[str] + + +class Nordigen(Base): + __tablename__ = "nordigen" + + name: Mapped[bankfk] = mapped_column(primary_key=True) + bank_id: Mapped[Optional[str]] + requisition_id: Mapped[Optional[str]] + invert: Mapped[Optional[bool]] + + +class Tags(Base): + __tablename__ = "tags" + + id: Mapped[idfk] = mapped_column(primary_key=True) + tag: Mapped[str] = mapped_column(primary_key=True) From 246c948d76971558b53cff8544c022db5dc06891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 3 Dec 2022 12:54:22 +0000 Subject: [PATCH 03/66] All meal card to account type enum --- .../versions/d3534f493239_add_meal_card.py | 36 +++++++++++++++++++ pfbudget/db/model.py | 1 + 2 files changed, 37 insertions(+) create mode 100644 alembic/versions/d3534f493239_add_meal_card.py diff --git a/alembic/versions/d3534f493239_add_meal_card.py b/alembic/versions/d3534f493239_add_meal_card.py new file mode 100644 index 0000000..3f25956 --- /dev/null +++ b/alembic/versions/d3534f493239_add_meal_card.py @@ -0,0 +1,36 @@ +"""Add meal card + +Revision ID: d3534f493239 +Revises: 50ff1fbb8a00 +Create Date: 2022-12-03 12:18:33.519666+00:00 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "d3534f493239" +down_revision = "50ff1fbb8a00" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TYPE transactions.accounttype ADD VALUE 'mealcard' BEFORE 'VISA'") + + +def downgrade() -> None: + op.execute( + """CREATE TYPE transactions.accounttype_new + AS ENUM ('checking', 'savings', 'investment', 'VISA', 'MASTERCARD') + """ + ) + op.execute("UPDATE transactions.banks SET type = DEFAULT WHERE type = 'mealcard'") + op.execute( + """ALTER TABLE transactions.banks + ALTER COLUMN type TYPE transactions.accounttype_new + USING type::text::transactions.accounttype_new + """ + ) + op.execute("DROP TYPE transactions.accounttype") + op.execute("ALTER TYPE transactions.accounttype_new RENAME TO accounttype") diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 492e877..2701bf8 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -32,6 +32,7 @@ class AccountType(enum.Enum): checking = enum.auto() savings = enum.auto() investment = enum.auto() + mealcard = enum.auto() VISA = enum.auto() MASTERCARD = enum.auto() From 91514f71b1f6a547dc1a5806b3f8dc590b369c68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 3 Dec 2022 16:45:02 +0000 Subject: [PATCH 04/66] Adds relationships and CASCADE on deletes --- .gitignore | 1 + .../287fe9e6682a_add_relationships.py | 109 ++++++++++++++++++ pfbudget/db/model.py | 32 ++++- 3 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 alembic/versions/287fe9e6682a_add_relationships.py diff --git a/.gitignore b/.gitignore index 8acbd12..5328e2c 100644 --- a/.gitignore +++ b/.gitignore @@ -153,4 +153,5 @@ dmypy.json ### Default user directories export/ +tmp/ .pfbudget diff --git a/alembic/versions/287fe9e6682a_add_relationships.py b/alembic/versions/287fe9e6682a_add_relationships.py new file mode 100644 index 0000000..13a669b --- /dev/null +++ b/alembic/versions/287fe9e6682a_add_relationships.py @@ -0,0 +1,109 @@ +"""Add relationships + +Revision ID: 287fe9e6682a +Revises: d3534f493239 +Create Date: 2022-12-03 16:43:39.633382+00:00 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "287fe9e6682a" +down_revision = "d3534f493239" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "fk_categorized_id_originals", + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_categorized_id_originals"), + "categorized", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + op.drop_constraint( + "fk_notes_id_originals", "notes", schema="transactions", type_="foreignkey" + ) + op.create_foreign_key( + op.f("fk_notes_id_originals"), + "notes", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + op.drop_constraint( + "fk_tags_id_originals", "tags", schema="transactions", type_="foreignkey" + ) + op.create_foreign_key( + op.f("fk_tags_id_originals"), + "tags", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_tags_id_originals"), "tags", schema="transactions", type_="foreignkey" + ) + op.create_foreign_key( + "fk_tags_id_originals", + "tags", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ) + op.drop_constraint( + op.f("fk_notes_id_originals"), + "notes", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_notes_id_originals", + "notes", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ) + op.drop_constraint( + op.f("fk_categorized_id_originals"), + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_categorized_id_originals", + "categorized", + "originals", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ) + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 2701bf8..385dabd 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from sqlalchemy import ( BigInteger, Enum, @@ -7,7 +9,13 @@ from sqlalchemy import ( String, Text, ) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, MappedAsDataclass +from sqlalchemy.orm import ( + DeclarativeBase, + Mapped, + mapped_column, + MappedAsDataclass, + relationship, +) from decimal import Decimal from typing import Annotated, Optional @@ -66,16 +74,26 @@ class Original(Base): bank: Mapped[bankfk] amount: Mapped[money] - -idfk = Annotated[int, mapped_column(BigInteger, ForeignKey(Original.id))] + category: Mapped[Category] = relationship(back_populates="original") + note: Mapped[Note] = relationship(back_populates="original") + tags: Mapped[set[Tag]] = relationship( + back_populates="original", cascade="all, delete-orphan", passive_deletes=True + ) -class Categorized(Base): +idfk = Annotated[ + int, mapped_column(BigInteger, ForeignKey(Original.id, ondelete="CASCADE")) +] + + +class Category(Base): __tablename__ = "categorized" id: Mapped[idfk] = mapped_column(primary_key=True) category: Mapped[str] + original: Mapped[Original] = relationship(back_populates="category") + class Note(Base): __tablename__ = "notes" @@ -83,6 +101,8 @@ class Note(Base): id: Mapped[idfk] = mapped_column(primary_key=True) note: Mapped[str] + original: Mapped[Original] = relationship(back_populates="note") + class Nordigen(Base): __tablename__ = "nordigen" @@ -93,8 +113,10 @@ class Nordigen(Base): invert: Mapped[Optional[bool]] -class Tags(Base): +class Tag(Base): __tablename__ = "tags" id: Mapped[idfk] = mapped_column(primary_key=True) tag: Mapped[str] = mapped_column(primary_key=True) + + original: Mapped[Original] = relationship(back_populates="tags") From e379d7799551cafa4ad933aeabcc38420e4b7efd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 3 Dec 2022 23:55:37 +0000 Subject: [PATCH 05/66] Nordigen -> PostgreSQL path completed Can now download from all banks registered on the banks/nordigen table and write to the PostgresSQL DB (or any DB, since we're now agnostic). Commented out most of the managers functions until the integration with the new DB client is complete. Set Optional relationships. Remove the DB types as dataclasses, it only increased the verbosity of the types w/o much going for it. Change the name Original to Transaction, since the type is the placeholder for the rest of the transaction information. --- pfbudget/core/manager.py | 64 +++++++++++++------------- pfbudget/db/client.py | 57 +++++++++++++++++++++++ pfbudget/db/model.py | 46 ++++++++++++------- pfbudget/input/nordigen.py | 7 +-- pfbudget/utils/converters.py | 87 ++++++++++++++++++------------------ 5 files changed, 169 insertions(+), 92 deletions(-) create mode 100644 pfbudget/db/client.py diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index d1b799d..b04e410 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,47 +1,51 @@ from pfbudget.input.input import Input from pfbudget.input.parsers import parse_data -from pfbudget.common.types import Bank, Banks, Transaction, Transactions -from pfbudget.db.sqlite import DatabaseClient +from pfbudget.db.client import DbClient from pfbudget.utils import convert class Manager: - def __init__(self, db: str): - self.__db = db + def __init__(self, url: str): + self._db = DbClient(url) - def init(self): - client = DatabaseClient(self.__db) - client.init() + # def init(self): + # client = DatabaseClient(self.__db) + # client.init() - def register(self, args: dict): - bank = Bank(args["bank"][0], "", args["requisition"][0], args["invert"]) - client = DatabaseClient(self.__db) - client.register_bank(convert(bank)) + # def register(self, args: dict): + # bank = Bank(args["bank"][0], "", args["requisition"][0], args["invert"]) + # client = DatabaseClient(self.__db) + # client.register_bank(convert(bank)) - def unregister(self, args: dict): - client = DatabaseClient(self.__db) - client.unregister_bank(args["bank"][0]) + # def unregister(self, args: dict): + # client = DatabaseClient(self.__db) + # client.unregister_bank(args["bank"][0]) def parser(self, parser: Input): transactions = parser.parse() - self.add_transactions(transactions) + print(transactions) + # self.add_transactions(transactions) - def parse(self, filename: str, args: dict): - transactions = parse_data(filename, args) - self.add_transactions(transactions) + # def parse(self, filename: str, args: dict): + # transactions = parse_data(filename, args) + # self.add_transactions(transactions) - def transactions() -> list[Transaction]: - pass + # def transactions() -> list[Transaction]: + # pass - def add_transactions(self, transactions: Transactions): - client = DatabaseClient(self.__db) - client.insert_transactions([convert(t) for t in transactions]) + def add_transactions(self, transactions): + with self.db.session() as session: + session.add(transactions) + session.commit() - def get_bank_by(self, key: str, value: str) -> Bank: - client = DatabaseClient(self.__db) - bank = client.get_bank(key, value) - return convert(bank) + # def get_bank_by(self, key: str, value: str) -> Bank: + # client = DatabaseClient(self.__db) + # bank = client.get_bank(key, value) + # return convert(bank) - def get_banks(self) -> Banks: - client = DatabaseClient(self.__db) - return [convert(bank) for bank in client.get_banks()] + def get_banks(self): + return self.db.get_nordigen_banks() + + @property + def db(self): + return self._db diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py new file mode 100644 index 0000000..3da414c --- /dev/null +++ b/pfbudget/db/client.py @@ -0,0 +1,57 @@ +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, joinedload, selectinload + +from pfbudget.db.model import Bank, Transaction + +# import logging + +# logging.basicConfig() +# logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) + + +class DbClient: + """ + General database client using sqlalchemy + """ + + __sessions: list[Session] + + def __init__(self, url: str) -> None: + self._engine = create_engine(url) + + def get_transactions(self): + """¿Non-optimized? get_transactions, will load the entire Transaction""" + with Session(self.engine) as session: + stmt = select(Transaction).options( + joinedload("*"), selectinload(Transaction.tags) + ) + return session.scalars(stmt).all() + + def get_uncategorized(self): + with Session(self.engine) as session: + stmt = select(Transaction).where(~Transaction.category.has()) + return session.scalars(stmt).all() + + def get_categorized(self): + with Session(self.engine) as session: + stmt = select(Transaction).where(Transaction.category.has()) + return session.scalars(stmt).all() + + def insert_transactions(self, input: list[Transaction]): + with Session(self.engine) as session: + session.add_all(input) + session.commit() + + def get_banks(self): + with Session(self.engine) as session: + stmt = select(Bank) + return session.scalars(stmt).all() + + def get_nordigen_banks(self): + with Session(self.engine) as session: + stmt = select(Bank).where(Bank.nordigen.has()) + return session.scalars(stmt).all() + + @property + def engine(self): + return self._engine diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 385dabd..16837c5 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -9,13 +9,7 @@ from sqlalchemy import ( String, Text, ) -from sqlalchemy.orm import ( - DeclarativeBase, - Mapped, - mapped_column, - MappedAsDataclass, - relationship, -) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from decimal import Decimal from typing import Annotated, Optional @@ -23,7 +17,7 @@ import datetime as dt import enum -class Base(MappedAsDataclass, DeclarativeBase): +class Base(DeclarativeBase): __table_args__ = {"schema": "transactions"} metadata = MetaData( naming_convention={ @@ -58,6 +52,13 @@ class Bank(Base): BIC: Mapped[str] = mapped_column(String(8), primary_key=True) type: Mapped[accounttype] = mapped_column(primary_key=True) + nordigen: Mapped[Optional[Nordigen]] = relationship( + back_populates="bank", lazy="joined" + ) + + def __repr__(self) -> str: + return f"Bank(name={self.name}, BIC={self.BIC}, type={self.type}, nordigen={self.nordigen})" + bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] @@ -65,7 +66,7 @@ idpk = Annotated[int, mapped_column(BigInteger, primary_key=True)] money = Annotated[Decimal, mapped_column(Numeric(16, 2), nullable=False)] -class Original(Base): +class Transaction(Base): __tablename__ = "originals" id: Mapped[idpk] = mapped_column(autoincrement=True) @@ -74,15 +75,20 @@ class Original(Base): bank: Mapped[bankfk] amount: Mapped[money] - category: Mapped[Category] = relationship(back_populates="original") - note: Mapped[Note] = relationship(back_populates="original") - tags: Mapped[set[Tag]] = relationship( + category: Mapped[Optional[Category]] = relationship( + back_populates="original", lazy="joined" + ) + note: Mapped[Optional[Note]] = relationship(back_populates="original") + tags: Mapped[Optional[set[Tag]]] = relationship( back_populates="original", cascade="all, delete-orphan", passive_deletes=True ) + def __repr__(self) -> str: + return f"Transaction(date={self.date}, description={self.description}, bank={self.bank}, amount={self.amount}, category={self.category})" + idfk = Annotated[ - int, mapped_column(BigInteger, ForeignKey(Original.id, ondelete="CASCADE")) + int, mapped_column(BigInteger, ForeignKey(Transaction.id, ondelete="CASCADE")) ] @@ -92,7 +98,10 @@ class Category(Base): id: Mapped[idfk] = mapped_column(primary_key=True) category: Mapped[str] - original: Mapped[Original] = relationship(back_populates="category") + original: Mapped[Transaction] = relationship(back_populates="category") + + def __repr__(self) -> str: + return f"Category({self.category})" class Note(Base): @@ -101,7 +110,7 @@ class Note(Base): id: Mapped[idfk] = mapped_column(primary_key=True) note: Mapped[str] - original: Mapped[Original] = relationship(back_populates="note") + original: Mapped[Transaction] = relationship(back_populates="note") class Nordigen(Base): @@ -112,6 +121,11 @@ class Nordigen(Base): requisition_id: Mapped[Optional[str]] invert: Mapped[Optional[bool]] + bank: Mapped[Bank] = relationship(back_populates="nordigen") + + def __repr__(self) -> str: + return f"(bank_id={self.bank_id}, requisition_id={self.requisition_id}, invert={self.invert})" + class Tag(Base): __tablename__ = "tags" @@ -119,4 +133,4 @@ class Tag(Base): id: Mapped[idfk] = mapped_column(primary_key=True) tag: Mapped[str] = mapped_column(primary_key=True) - original: Mapped[Original] = relationship(back_populates="tags") + original: Mapped[Transaction] = relationship(back_populates="tags") diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index fa1a128..e43f878 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -9,7 +9,8 @@ import os import webbrowser from .input import Input -from pfbudget.common.types import NoBankSelected, Transactions +from pfbudget.common.types import NoBankSelected +from pfbudget.db.model import Transaction from pfbudget.utils import convert load_dotenv() @@ -43,7 +44,7 @@ class NordigenInput(Input): self.__from = start self.__to = end - def parse(self) -> Transactions: + def parse(self) -> list[Transaction]: transactions = [] if not self.__banks: raise NoBankSelected @@ -51,7 +52,7 @@ class NordigenInput(Input): for bank in self.__banks: print(f"Downloading from {bank}...") requisition = self.client.requisition.get_requisition_by_id( - bank.requisition_id + bank.nordigen.requisition_id ) for acc in requisition["accounts"]: diff --git a/pfbudget/utils/converters.py b/pfbudget/utils/converters.py index b3f27df..287ea1b 100644 --- a/pfbudget/utils/converters.py +++ b/pfbudget/utils/converters.py @@ -1,7 +1,8 @@ -from datetime import timedelta +from datetime import date, timedelta from functools import singledispatch -from pfbudget.common.types import Bank, Transaction, TransactionError +from pfbudget.common.types import TransactionError +from pfbudget.db.model import Bank, Transaction from pfbudget.db.schema import DbBank, DbTransaction from .utils import parse_decimal @@ -12,60 +13,60 @@ def convert(t): pass -@convert.register -def _(t: Transaction) -> DbTransaction: - return DbTransaction( - t.date, - t.description, - t.bank, - t.value, - t.category, - t.original, - t.additional_comment, - ) +# @convert.register +# def _(t: Transaction) -> DbTransaction: +# return DbTransaction( +# t.date, +# t.description, +# t.bank, +# t.value, +# t.category, +# t.original, +# t.additional_comment, +# ) -@convert.register -def _(db: DbTransaction) -> Transaction: - try: - return Transaction(db) - except TransactionError: - print(f"{db} is in the wrong format") +# @convert.register +# def _(db: DbTransaction) -> Transaction: +# try: +# return Transaction(db) +# except TransactionError: +# print(f"{db} is in the wrong format") -@convert.register -def _(db: DbBank, key: str = "") -> Bank: - bank = Bank(db.name, db.bic, db.requisition_id, db.invert, db.offset, key=key) - if not bank.invert: - bank.invert = False - if not bank.offset: - bank.offset = 0 - return bank +# @convert.register +# def _(db: DbBank, key: str = "") -> Bank: +# bank = Bank(db.name, db.bic, db.requisition_id, db.invert, db.offset, key=key) +# if not bank.invert: +# bank.invert = False +# if not bank.offset: +# bank.offset = 0 +# return bank -@convert.register -def _(bank: Bank) -> DbBank: - bank = DbBank( - bank.name, bank.bic, "", "", bank.requisition_id, bank.invert, bank.offset - ) - if not bank.invert: - bank.invert = False - if not bank.offset: - bank.offset = 0 - return bank +# @convert.register +# def _(bank: Bank) -> DbBank: +# bank = DbBank( +# bank.name, bank.bic, "", "", bank.requisition_id, bank.invert, bank.offset +# ) +# if not bank.invert: +# bank.invert = False +# if not bank.offset: +# bank.offset = 0 +# return bank @convert.register def _(json: dict, bank: Bank) -> Transaction: - i = -1 if bank.invert else 1 + i = -1 if bank.nordigen.invert else 1 try: transaction = Transaction( - json["bookingDate"], - json["remittanceInformationUnstructured"], - bank.name, - i * parse_decimal(json["transactionAmount"]["amount"]), + date=date.fromisoformat(json["bookingDate"]), + description=json["remittanceInformationUnstructured"], + bank=bank.name, + amount=i * parse_decimal(json["transactionAmount"]["amount"]), ) - transaction.date += timedelta(days=bank.offset) + # transaction.date += timedelta(days=bank.offset) return transaction except TransactionError: From b8142f4f9917126e2e6e216731c947b5b26b5781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 4 Dec 2022 11:19:21 +0000 Subject: [PATCH 06/66] Add available categories table --- ...91f1be11_available_categories_and_rules.py | 49 +++++++++++++++++++ pfbudget/db/model.py | 31 ++++++++++-- 2 files changed, 76 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/2d0891f1be11_available_categories_and_rules.py diff --git a/alembic/versions/2d0891f1be11_available_categories_and_rules.py b/alembic/versions/2d0891f1be11_available_categories_and_rules.py new file mode 100644 index 0000000..2aa1168 --- /dev/null +++ b/alembic/versions/2d0891f1be11_available_categories_and_rules.py @@ -0,0 +1,49 @@ +"""Available categories and rules + +Revision ID: 2d0891f1be11 +Revises: 287fe9e6682a +Create Date: 2022-12-04 11:15:22.758487+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "2d0891f1be11" +down_revision = "287fe9e6682a" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "categories_available", + sa.Column("name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_categories_available")), + schema="transactions", + ) + op.create_table( + "categories_rules", + sa.Column("name", sa.String(), nullable=False), + sa.Column("rule", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["name"], + ["transactions.categories_available.name"], + name=op.f("fk_categories_rules_name_categories_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("name", "rule", name=op.f("pk_categories_rules")), + schema="transactions", + ) + op.alter_column("categorized", "category", new_column_name="name", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column("categorized", "name", new_column_name="category", schema="transactions") + op.drop_table("categories_rules", schema="transactions") + op.drop_table("categories_available", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 16837c5..daed50d 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -75,7 +75,7 @@ class Transaction(Base): bank: Mapped[bankfk] amount: Mapped[money] - category: Mapped[Optional[Category]] = relationship( + category: Mapped[Optional[TransactionCategory]] = relationship( back_populates="original", lazy="joined" ) note: Mapped[Optional[Note]] = relationship(back_populates="original") @@ -92,16 +92,17 @@ idfk = Annotated[ ] -class Category(Base): +class TransactionCategory(Base): __tablename__ = "categorized" id: Mapped[idfk] = mapped_column(primary_key=True) - category: Mapped[str] + name: Mapped[str] original: Mapped[Transaction] = relationship(back_populates="category") + category: Mapped[AvailableCategory] = relationship(back_populates="category") def __repr__(self) -> str: - return f"Category({self.category})" + return f"Category({self.name})" class Note(Base): @@ -134,3 +135,25 @@ class Tag(Base): tag: Mapped[str] = mapped_column(primary_key=True) original: Mapped[Transaction] = relationship(back_populates="tags") + + +class AvailableCategory(Base): + __tablename__ = "categories_available" + + name: Mapped[str] = mapped_column(primary_key=True) + + rules: Mapped[Optional[set[CategoryRule]]] = relationship( + back_populates="original", cascade="all, delete-orphan", passive_deletes=True + ) + category: Mapped[TransactionCategory] = relationship(back_populates="category") + + +class CategoryRule(Base): + __tablename__ = "categories_rules" + + name: Mapped[str] = mapped_column( + ForeignKey(AvailableCategory.name, ondelete="CASCADE"), primary_key=True + ) + rule: Mapped[str] = mapped_column(primary_key=True) + + category: Mapped[AvailableCategory] = relationship(back_populates="rules") From 78e545589d722319b32d20de8611639c61e30c14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 4 Dec 2022 15:14:24 +0000 Subject: [PATCH 07/66] Adds category groups --- ...37f6e_category_groups_and_relationships.py | 69 +++++++++++++++++++ pfbudget/db/model.py | 37 ++++++---- 2 files changed, 91 insertions(+), 15 deletions(-) create mode 100644 alembic/versions/83f4c9837f6e_category_groups_and_relationships.py diff --git a/alembic/versions/83f4c9837f6e_category_groups_and_relationships.py b/alembic/versions/83f4c9837f6e_category_groups_and_relationships.py new file mode 100644 index 0000000..04dd4f7 --- /dev/null +++ b/alembic/versions/83f4c9837f6e_category_groups_and_relationships.py @@ -0,0 +1,69 @@ +"""Category groups and relationships + +Revision ID: 83f4c9837f6e +Revises: 2d0891f1be11 +Create Date: 2022-12-04 15:10:51.924875+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "83f4c9837f6e" +down_revision = "2d0891f1be11" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "categories_groups", + sa.Column("name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_categories_groups")), + schema="transactions", + ) + op.add_column( + "categories_available", + sa.Column("group", sa.String(), nullable=True), + schema="transactions", + ) + op.create_foreign_key( + op.f("fk_categories_available_group_categories_groups"), + "categories_available", + "categories_groups", + ["group"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + op.create_foreign_key( + op.f("fk_categorized_name_categories_available"), + "categorized", + "categories_available", + ["name"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_categorized_name_categories_available"), + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_categories_available_group_categories_groups"), + "categories_available", + schema="transactions", + type_="foreignkey", + ) + op.drop_column("categories_available", "group", schema="transactions") + op.drop_table("categories_groups", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index daed50d..f985e8a 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -92,14 +92,32 @@ idfk = Annotated[ ] +class CategoryGroup(Base): + __tablename__ = "categories_groups" + + name: Mapped[str] = mapped_column(primary_key=True) + + +class Category(Base): + __tablename__ = "categories_available" + + name: Mapped[str] = mapped_column(primary_key=True) + group: Mapped[Optional[str]] = mapped_column(ForeignKey(CategoryGroup.name)) + + rules: Mapped[Optional[set[CategoryRule]]] = relationship( + back_populates="category", cascade="all, delete-orphan", passive_deletes=True + ) + categorygroup: Mapped[Optional[CategoryGroup]] = relationship() + + class TransactionCategory(Base): __tablename__ = "categorized" id: Mapped[idfk] = mapped_column(primary_key=True) - name: Mapped[str] + name: Mapped[str] = mapped_column(ForeignKey(Category.name)) original: Mapped[Transaction] = relationship(back_populates="category") - category: Mapped[AvailableCategory] = relationship(back_populates="category") + category: Mapped[Category] = relationship() def __repr__(self) -> str: return f"Category({self.name})" @@ -137,23 +155,12 @@ class Tag(Base): original: Mapped[Transaction] = relationship(back_populates="tags") -class AvailableCategory(Base): - __tablename__ = "categories_available" - - name: Mapped[str] = mapped_column(primary_key=True) - - rules: Mapped[Optional[set[CategoryRule]]] = relationship( - back_populates="original", cascade="all, delete-orphan", passive_deletes=True - ) - category: Mapped[TransactionCategory] = relationship(back_populates="category") - - class CategoryRule(Base): __tablename__ = "categories_rules" name: Mapped[str] = mapped_column( - ForeignKey(AvailableCategory.name, ondelete="CASCADE"), primary_key=True + ForeignKey(Category.name, ondelete="CASCADE"), primary_key=True ) rule: Mapped[str] = mapped_column(primary_key=True) - category: Mapped[AvailableCategory] = relationship(back_populates="rules") + category: Mapped[Category] = relationship(back_populates="rules") From be67612f678e110034231319a6f14fd3410a6cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 4 Dec 2022 16:09:54 +0000 Subject: [PATCH 08/66] Introduces categorizer that works on ORM classes Categorizer will work directly on ORM classes, which will cleanup the code, since changes will automatically be persisted when change the objects. Adds wrapper session class inside the DbClient for the manager to use. The manager will have to have some DB session knowledge, which adds some unfortunate coupling. Removes some unnecessary relations between tables that were added by mistake. category CLI option now uses the manager. --- pfbudget/cli/runnable.py | 2 +- pfbudget/core/categorizer.py | 43 ++++++++++++++++++++++++++++++++++++ pfbudget/core/manager.py | 7 ++++++ pfbudget/db/client.py | 29 +++++++++++++++++++++++- pfbudget/db/model.py | 6 +---- 5 files changed, 80 insertions(+), 7 deletions(-) create mode 100644 pfbudget/core/categorizer.py diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 6778110..b980a1c 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -112,7 +112,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_categorize.set_defaults( - func=lambda args: categorize_data(DatabaseClient(args.database)) + func=lambda args: manager.categorize(vars(args)) ) """ diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py new file mode 100644 index 0000000..f15d349 --- /dev/null +++ b/pfbudget/core/categorizer.py @@ -0,0 +1,43 @@ +from pfbudget.db.model import Transaction, TransactionCategory + +from datetime import timedelta + + +class Categorizer: + options = {} + + def __init__(self, args: dict): + self.options["null_days"] = 4 + + def categorize(self, transactions: list[Transaction]): + """Overarching categorization tool + + Receives a list of transactions (by ref) and updates their category + + Args: + transactions (list[Transaction]): uncategorized transactions + """ + + self._nullify(transactions) + + def _nullify(self, transactions: list[Transaction]): + matching = [] + for transaction in transactions: + for cancel in ( + cancel + for cancel in transactions + if ( + transaction.date - timedelta(days=self.options["null_days"]) + <= cancel.date + <= transaction.date + timedelta(days=self.options["null_days"]) + and transaction not in matching + and cancel not in matching + and cancel != transaction + and cancel.bank != transaction.bank + and cancel.amount == -transaction.amount + ) + ): + transaction.category = TransactionCategory(name="null") + cancel.category = TransactionCategory(name="null") + matching.extend([transaction, cancel]) + break diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index b04e410..071324a 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,6 +1,7 @@ from pfbudget.input.input import Input from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient +from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -38,6 +39,12 @@ class Manager: session.add(transactions) session.commit() + def categorize(self, args: dict): + with self.db.session() as session: + uncategorized = session.uncategorized() + Categorizer().categorize(uncategorized) + session.commit() + # def get_bank_by(self, key: str, value: str) -> Bank: # client = DatabaseClient(self.__db) # bank = client.get_bank(key, value) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 3da414c..0b4d1c5 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,7 +1,7 @@ from sqlalchemy import create_engine, select from sqlalchemy.orm import Session, joinedload, selectinload -from pfbudget.db.model import Bank, Transaction +from pfbudget.db.model import Bank, Category, Transaction # import logging @@ -55,3 +55,30 @@ class DbClient: @property def engine(self): return self._engine + + class ClientSession: + def __init__(self, engine): + self.__engine = engine + + def __enter__(self): + self.__session = Session(self.__engine) + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self.__session.close() + + def commit(self): + self.__session.commit() + + def add(self, transactions: list[Transaction]): + self.__session.add_all(transactions) + + def addcategory(self, category: Category): + self.__session.add(category) + + def uncategorized(self) -> list[Transaction]: + stmt = select(Transaction).where(~Transaction.category.has()) + return self.__session.scalars(stmt).all() + + def session(self): + return self.ClientSession(self.engine) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index f985e8a..fb900ce 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -105,9 +105,8 @@ class Category(Base): group: Mapped[Optional[str]] = mapped_column(ForeignKey(CategoryGroup.name)) rules: Mapped[Optional[set[CategoryRule]]] = relationship( - back_populates="category", cascade="all, delete-orphan", passive_deletes=True + cascade="all, delete-orphan", passive_deletes=True ) - categorygroup: Mapped[Optional[CategoryGroup]] = relationship() class TransactionCategory(Base): @@ -117,7 +116,6 @@ class TransactionCategory(Base): name: Mapped[str] = mapped_column(ForeignKey(Category.name)) original: Mapped[Transaction] = relationship(back_populates="category") - category: Mapped[Category] = relationship() def __repr__(self) -> str: return f"Category({self.name})" @@ -162,5 +160,3 @@ class CategoryRule(Base): ForeignKey(Category.name, ondelete="CASCADE"), primary_key=True ) rule: Mapped[str] = mapped_column(primary_key=True) - - category: Mapped[Category] = relationship(back_populates="rules") From 882a77d24c24e8614b6938c2618c97a074448de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 4 Dec 2022 17:45:05 +0000 Subject: [PATCH 09/66] [Refactor] CLI argparser passes options to Manager Move all BL to the manager. The ArgParser now only parses the CLI arguments and creates a command which contains which command was run. In turn, this information is passed to the manager, which will run the appropriate business logic. This will make it easier to add new options, separating the parsing of the CLI options from the implementation of the logic. It also simplifies any future effort in adding a different input (e.g. GUI). Warning: some function were commented out, this is only a tracer bullet. --- main.py | 5 +- pfbudget/__init__.py | 1 + pfbudget/cli/runnable.py | 96 +++++++++++++++++------------------- pfbudget/common/types.py | 11 +++++ pfbudget/core/categorizer.py | 6 ++- pfbudget/core/manager.py | 53 +++++++++++++++++++- pfbudget/utils/utils.py | 23 ++++----- 7 files changed, 128 insertions(+), 67 deletions(-) diff --git a/main.py b/main.py index d25b52a..c5f8983 100644 --- a/main.py +++ b/main.py @@ -1,4 +1,5 @@ -from pfbudget import run +from pfbudget import Manager, run if __name__ == "__main__": - run() + command, args = run() + Manager(command).start(args) diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index 80ed96e..8b5a5fa 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -3,5 +3,6 @@ __author__ = "Luís Murta" __version__ = "0.1" from pfbudget.core.categories import categorize_data +from pfbudget.core.manager import Manager from pfbudget.cli.runnable import run from pfbudget.input.parsers import parse_data diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index b980a1c..d5b1a0e 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -2,8 +2,8 @@ from pathlib import Path import argparse import re +from pfbudget.common.types import Command from pfbudget.core.categories import categorize_data -from pfbudget.core.manager import Manager from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput from pfbudget.db.sqlite import DatabaseClient @@ -27,7 +27,7 @@ class DataFileMissing(Exception): pass -def argparser(manager: Manager) -> argparse.ArgumentParser: +def argparser() -> argparse.ArgumentParser: help = argparse.ArgumentParser(add_help=False) help.add_argument( @@ -74,7 +74,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: parents=[help], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_init.set_defaults(func=lambda args: manager.init()) + p_init.set_defaults(command=Command.Init) """ Exporting @@ -100,7 +100,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: p_parse.add_argument("--bank", nargs=1, type=str) p_parse.add_argument("--creditcard", nargs=1, type=str) p_parse.add_argument("--category", nargs=1, type=int) - p_parse.set_defaults(func=lambda args: parse(manager, args)) + p_parse.set_defaults(command=Command.Parse) """ Categorizing @@ -111,9 +111,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: parents=[help], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_categorize.set_defaults( - func=lambda args: manager.categorize(vars(args)) - ) + p_categorize.set_defaults(command=Command.Categorize) """ Graph @@ -168,7 +166,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: "--requisition", type=str, nargs=1, help="requisition option help" ) p_register.add_argument("--invert", action="store_true") - p_register.set_defaults(func=lambda args: manager.register(vars(args))) + p_register.set_defaults(command=Command.Register) """ Unregister bank @@ -180,7 +178,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_register.add_argument("bank", type=str, nargs=1, help="bank option help") - p_register.set_defaults(func=lambda args: manager.unregister(vars(args))) + p_register.set_defaults(command=Command.Unregister) """ Nordigen API @@ -191,7 +189,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: parents=[help], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_nordigen_access.set_defaults(func=lambda args: NordigenInput(manager).token()) + p_nordigen_access.set_defaults(command=Command.Token) """ (Re)new bank requisition ID @@ -204,11 +202,7 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: ) p_nordigen_access.add_argument("name", nargs=1, type=str) p_nordigen_access.add_argument("country", nargs=1, type=str) - p_nordigen_access.set_defaults( - func=lambda args: NordigenInput(manager).requisition( - args.name[0], args.country[0] - ) - ) + p_nordigen_access.set_defaults(command=Command.Renew) """ Downloading through Nordigen API @@ -222,40 +216,40 @@ def argparser(manager: Manager) -> argparse.ArgumentParser: p_nordigen_download.add_argument("--id", nargs="+", type=str) p_nordigen_download.add_argument("--name", nargs="+", type=str) p_nordigen_download.add_argument("--all", action="store_true") - p_nordigen_download.set_defaults(func=lambda args: download(manager, args)) + p_nordigen_download.set_defaults(command=Command.Download) - """ - List available banks on Nordigen API - """ - p_nordigen_list = subparsers.add_parser( - "list", - description="Lists banks in {country}", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_nordigen_list.add_argument("country", nargs=1, type=str) - p_nordigen_list.set_defaults(func=lambda args: nordigen_banks(manager, args)) + # """ + # List available banks on Nordigen API + # """ + # p_nordigen_list = subparsers.add_parser( + # "list", + # description="Lists banks in {country}", + # parents=[help], + # formatter_class=argparse.ArgumentDefaultsHelpFormatter, + # ) + # p_nordigen_list.add_argument("country", nargs=1, type=str) + # p_nordigen_list.set_defaults(func=lambda args: nordigen_banks(manager, args)) - """ - Nordigen JSONs - """ - p_nordigen_json = subparsers.add_parser( - "json", - description="", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_nordigen_json.add_argument("json", nargs=1, type=str) - p_nordigen_json.add_argument("bank", nargs=1, type=str) - p_nordigen_json.add_argument("--invert", action=argparse.BooleanOptionalAction) - p_nordigen_json.set_defaults( - func=lambda args: manager.parser(JsonParser(vars(args))) - ) + # """ + # Nordigen JSONs + # """ + # p_nordigen_json = subparsers.add_parser( + # "json", + # description="", + # parents=[help], + # formatter_class=argparse.ArgumentDefaultsHelpFormatter, + # ) + # p_nordigen_json.add_argument("json", nargs=1, type=str) + # p_nordigen_json.add_argument("bank", nargs=1, type=str) + # p_nordigen_json.add_argument("--invert", action=argparse.BooleanOptionalAction) + # p_nordigen_json.set_defaults( + # func=lambda args: manager.parser(JsonParser(vars(args))) + # ) return parser -def parse(manager: Manager, args): +def parse(manager, args): """Parses the contents of the path in args to the selected database. Args: @@ -305,17 +299,17 @@ def report(args): pfbudget.reporting.report.detailed(DatabaseClient(args.database), start, end) -def nordigen_banks(manager: Manager, args): - input = NordigenInput(manager) - input.list(vars(args)["country"][0]) +# def nordigen_banks(manager: Manager, args): +# input = NordigenInput(manager) +# input.list(vars(args)["country"][0]) -def download(manager: Manager, args): +def download(manager, args: dict): start, end = pfbudget.utils.parse_args_period(args) - manager.parser(NordigenInput(manager, vars(args), start, end)) + manager.parser(NordigenInput(manager, args, start, end)) def run(): - manager = Manager(DEFAULT_DB) - args = argparser(manager).parse_args() - args.func(args) + args = vars(argparser().parse_args()) + assert "command" in args, "No command selected" + return args["command"], args diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 54c4fe3..c7602c5 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -4,6 +4,17 @@ from decimal import Decimal, InvalidOperation from enum import Enum, auto +class Command(Enum): + Init = auto() + Parse = auto() + Download = auto() + Categorize = auto() + Register = auto() + Unregister = auto() + Token = auto() + Renew = auto() + + class TransactionError(Exception): pass diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index f15d349..7b4f8e2 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -6,7 +6,7 @@ from datetime import timedelta class Categorizer: options = {} - def __init__(self, args: dict): + def __init__(self): self.options["null_days"] = 4 def categorize(self, transactions: list[Transaction]): @@ -21,6 +21,7 @@ class Categorizer: self._nullify(transactions) def _nullify(self, transactions: list[Transaction]): + count = 0 matching = [] for transaction in transactions: for cancel in ( @@ -40,4 +41,7 @@ class Categorizer: transaction.category = TransactionCategory(name="null") cancel.category = TransactionCategory(name="null") matching.extend([transaction, cancel]) + count += 2 break + + print(f"Nullified {count} transactions") diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 071324a..a56342e 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,13 +1,62 @@ from pfbudget.input.input import Input +from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient +from pfbudget.common.types import Command from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert +from pfbudget.cli.runnable import download, parse + class Manager: - def __init__(self, url: str): - self._db = DbClient(url) + def __init__(self, command: Command): + self.__command = command + match (command): + case Command.Init: + pass + case Command.Parse: + pass + case Command.Download: + pass + case Command.Categorize: + pass + case Command.Register: + pass + case Command.Unregister: + pass + case Command.Token: + pass + case Command.Renew: + pass + + def start(self, args): + match (self.__command): + case Command.Init: + pass + case Command.Parse: + # TODO this is a monstrosity, remove when possible + self._db = DbClient(args["database"]) + parse(self, args) + case Command.Download: + # TODO this is a monstrosity, remove when possible + self._db = DbClient(args["database"]) + download(self, args) + case Command.Categorize: + self._db = DbClient(args["database"]) + self.categorize(args) + case Command.Register: + # self._db = DbClient(args["database"]) + # self.register(args) + pass + case Command.Unregister: + # self._db = DbClient(args["database"]) + # self.unregister(args) + pass + case Command.Token: + NordigenClient(self).token() + case Command.Renew: + NordigenClient(self).requisition(args["name"], args["country"]) # def init(self): # client = DatabaseClient(self.__db) diff --git a/pfbudget/utils/utils.py b/pfbudget/utils/utils.py index 7e692f6..6cacab6 100644 --- a/pfbudget/utils/utils.py +++ b/pfbudget/utils/utils.py @@ -59,21 +59,22 @@ def find_credit_institution(fn, banks, creditcards): return bank, cc -def parse_args_period(args): +def parse_args_period(args: dict): start, end = date.min, date.max - if args.start: - start = datetime.strptime(args.start[0], "%Y/%m/%d").date() + print(args) + if args["start"]: + start = datetime.strptime(args["start"][0], "%Y/%m/%d").date() - if args.end: - end = datetime.strptime(args.end[0], "%Y/%m/%d").date() + if args["end"]: + end = datetime.strptime(args["end"][0], "%Y/%m/%d").date() - if args.interval: - start = datetime.strptime(args.interval[0], "%Y/%m/%d").date() - end = datetime.strptime(args.interval[1], "%Y/%m/%d").date() + if args["interval"]: + start = datetime.strptime(args["interval"][0], "%Y/%m/%d").date() + end = datetime.strptime(args["interval"][1], "%Y/%m/%d").date() - if args.year: - start = datetime.strptime(args.year[0], "%Y").date() - end = datetime.strptime(str(int(args.year[0]) + 1), "%Y").date() - timedelta( + if args["year"]: + start = datetime.strptime(args["year"][0], "%Y").date() + end = datetime.strptime(str(int(args["year"][0]) + 1), "%Y").date() - timedelta( days=1 ) From 9d33df78a897f82124048c42ea5c72371268f6a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 00:09:28 +0000 Subject: [PATCH 10/66] Adds command line option to add/remove categories Implements the argument parser, the manager logic and the DB client methods. Encapsulates the DbClient connection under the _db attribute on the manager. Adds verbose option to enable ORM increased logging. --- main.py | 2 +- pfbudget/cli/runnable.py | 55 ++++++++++++++++++++++- pfbudget/common/types.py | 9 ++++ pfbudget/core/manager.py | 95 +++++++++++++++++++++++++++++++--------- pfbudget/db/client.py | 34 +++++++++++--- pfbudget/db/model.py | 5 +++ 6 files changed, 172 insertions(+), 28 deletions(-) diff --git a/main.py b/main.py index c5f8983..5ed5689 100644 --- a/main.py +++ b/main.py @@ -2,4 +2,4 @@ from pfbudget import Manager, run if __name__ == "__main__": command, args = run() - Manager(command).start(args) + Manager(command, args).start() diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index d5b1a0e..7e89ef7 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -2,7 +2,7 @@ from pathlib import Path import argparse import re -from pfbudget.common.types import Command +from pfbudget.common.types import Command, Operation from pfbudget.core.categories import categorize_data from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput @@ -40,6 +40,9 @@ def argparser() -> argparse.ArgumentParser: help.add_argument( "-q", "--quiet", action="store_true", help="reduces the amount of verbose" ) + help.add_argument( + "-v", "--verbose", action="store_true", help="increases the amount of verbose" + ) period = argparse.ArgumentParser(add_help=False).add_mutually_exclusive_group() period.add_argument( @@ -246,6 +249,56 @@ def argparser() -> argparse.ArgumentParser: # func=lambda args: manager.parser(JsonParser(vars(args))) # ) + # Add category + p_categories = subparsers.add_parser( + "category", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_commands = p_categories.add_subparsers(dest="command", required=True) + p_categories_add = p_categories_commands.add_parser( + "add", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_add.add_argument("category", nargs="+", type=str) + p_categories_add.add_argument("--group", nargs="?", type=str) + p_categories_add.set_defaults(command=Command.Category, op=Operation.Add) + + p_categories_remove = p_categories_commands.add_parser( + "remove", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_remove.add_argument("category", nargs="+", type=str) + p_categories_remove.add_argument("--group", nargs="?", type=str) + p_categories_remove.set_defaults(command=Command.Category, op=Operation.Remove) + + p_categories_addgroup = p_categories_commands.add_parser( + "addgroup", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_addgroup.add_argument("group", nargs="+", type=str) + p_categories_addgroup.set_defaults(command=Command.Category, op=Operation.AddGroup) + + p_categories_removegroup = p_categories_commands.add_parser( + "removegroup", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_removegroup.add_argument("group", nargs="+", type=str) + p_categories_removegroup.set_defaults(command=Command.Category, op=Operation.RemoveGroup) + + p_categories_updategroup = p_categories_commands.add_parser( + "updategroup", + parents=[help], + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + p_categories_updategroup.add_argument("category", nargs="+", type=str) + p_categories_updategroup.add_argument("--group", nargs=1, type=str) + p_categories_updategroup.set_defaults(command=Command.Category, op=Operation.UpdateGroup) + return parser diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index c7602c5..829552f 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -13,6 +13,15 @@ class Command(Enum): Unregister = auto() Token = auto() Renew = auto() + Category = auto() + + +class Operation(Enum): + Add = auto() + Remove = auto() + AddGroup = auto() + RemoveGroup = auto() + UpdateGroup = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index a56342e..b964ba8 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -2,7 +2,8 @@ from pfbudget.input.input import Input from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient -from pfbudget.common.types import Command +from pfbudget.db.model import Category, CategoryGroup +from pfbudget.common.types import Command, Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -10,8 +11,9 @@ from pfbudget.cli.runnable import download, parse class Manager: - def __init__(self, command: Command): + def __init__(self, command: Command, args: dict): self.__command = command + self._args = args match (command): case Command.Init: pass @@ -29,22 +31,24 @@ class Manager: pass case Command.Renew: pass + case Command.Category: + pass - def start(self, args): + assert "database" in args, "ArgParser didn't include db" + self._db = args["database"] + + def start(self): match (self.__command): case Command.Init: pass case Command.Parse: # TODO this is a monstrosity, remove when possible - self._db = DbClient(args["database"]) - parse(self, args) + parse(self, self.args) case Command.Download: # TODO this is a monstrosity, remove when possible - self._db = DbClient(args["database"]) - download(self, args) + download(self, self.args) case Command.Categorize: - self._db = DbClient(args["database"]) - self.categorize(args) + self.categorize(self.args) case Command.Register: # self._db = DbClient(args["database"]) # self.register(args) @@ -55,29 +59,72 @@ class Manager: pass case Command.Token: NordigenClient(self).token() + case Command.Renew: - NordigenClient(self).requisition(args["name"], args["country"]) + NordigenClient(self).requisition( + self.args["name"], self.args["country"] + ) + + case Command.Category: + assert "op" in self.args, "category operation not defined" + + with self.db.session() as session: + match self.args["op"]: + case Operation.Add: + for category in self.args["category"]: + session.addcategory( + Category(name=category, group=self.args["group"]) + ) + + case Operation.Remove: + session.removecategory( + [ + Category(name=category) + for category in self.args["category"] + ] + ) + + case Operation.UpdateGroup: + session.updategroup( + [ + Category(name=category) + for category in self.args["category"] + ], + self.args["group"][0], + ) + + case Operation.AddGroup: + for group in self.args["group"]: + session.addcategorygroup(CategoryGroup(name=group)) + + case Operation.RemoveGroup: + session.removecategorygroup( + [ + CategoryGroup(name=group) + for group in self.args["group"] + ] + ) # def init(self): # client = DatabaseClient(self.__db) # client.init() - # def register(self, args: dict): - # bank = Bank(args["bank"][0], "", args["requisition"][0], args["invert"]) + # def register(self): + # bank = Bank(self.args["bank"][0], "", self.args["requisition"][0], self.args["invert"]) # client = DatabaseClient(self.__db) # client.register_bank(convert(bank)) - # def unregister(self, args: dict): + # def unregister(self): # client = DatabaseClient(self.__db) - # client.unregister_bank(args["bank"][0]) + # client.unregister_bank(self.args["bank"][0]) def parser(self, parser: Input): transactions = parser.parse() print(transactions) # self.add_transactions(transactions) - # def parse(self, filename: str, args: dict): - # transactions = parse_data(filename, args) + # def parse(self, filename: str): + # transactions = parse_data(filename, self.args) # self.add_transactions(transactions) # def transactions() -> list[Transaction]: @@ -86,13 +133,11 @@ class Manager: def add_transactions(self, transactions): with self.db.session() as session: session.add(transactions) - session.commit() - def categorize(self, args: dict): + def categorize(self): with self.db.session() as session: uncategorized = session.uncategorized() Categorizer().categorize(uncategorized) - session.commit() # def get_bank_by(self, key: str, value: str) -> Bank: # client = DatabaseClient(self.__db) @@ -103,5 +148,13 @@ class Manager: return self.db.get_nordigen_banks() @property - def db(self): - return self._db + def db(self) -> DbClient: + return DbClient(self._db, self.args["verbose"]) + + @db.setter + def db(self, url: str): + self._db = url + + @property + def args(self) -> dict: + return self._args diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 0b4d1c5..88bc213 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,7 +1,7 @@ -from sqlalchemy import create_engine, select +from sqlalchemy import create_engine, delete, select, update from sqlalchemy.orm import Session, joinedload, selectinload -from pfbudget.db.model import Bank, Category, Transaction +from pfbudget.db.model import Bank, Category, CategoryGroup, Transaction # import logging @@ -16,8 +16,8 @@ class DbClient: __sessions: list[Session] - def __init__(self, url: str) -> None: - self._engine = create_engine(url) + def __init__(self, url: str, echo=False) -> None: + self._engine = create_engine(url, echo=echo) def get_transactions(self): """¿Non-optimized? get_transactions, will load the entire Transaction""" @@ -65,6 +65,7 @@ class DbClient: return self def __exit__(self, exc_type, exc_value, exc_tb): + self.commit() self.__session.close() def commit(self): @@ -76,9 +77,32 @@ class DbClient: def addcategory(self, category: Category): self.__session.add(category) + def removecategory(self, categories: list[Category]): + stmt = delete(Category).where( + Category.name.in_([cat.name for cat in categories]) + ) + self.__session.execute(stmt) + + def updategroup(self, categories: list[Category], group: CategoryGroup): + stmt = ( + update(Category) + .where(Category.name.in_([cat.name for cat in categories])) + .values(group=group) + ) + self.__session.execute(stmt) + + def addcategorygroup(self, group: CategoryGroup): + self.__session.add(group) + + def removecategorygroup(self, groups: list[CategoryGroup]): + stmt = delete(CategoryGroup).where( + CategoryGroup.name.in_([grp.name for grp in groups]) + ) + self.__session.execute(stmt) + def uncategorized(self) -> list[Transaction]: stmt = select(Transaction).where(~Transaction.category.has()) return self.__session.scalars(stmt).all() - def session(self): + def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index fb900ce..be1ceaa 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -108,6 +108,11 @@ class Category(Base): cascade="all, delete-orphan", passive_deletes=True ) + def __repr__(self) -> str: + return ( + f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)})" + ) + class TransactionCategory(Base): __tablename__ = "categorized" From d409038072104f8bb83c5e9c5d053e046233b517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 00:32:06 +0000 Subject: [PATCH 11/66] Adds category selector column This column indicates where has the category came from. --- .../6863dda76ea2_category_selector.py | 54 +++++++++++++++++++ pfbudget/db/model.py | 31 +++++++++++ 2 files changed, 85 insertions(+) create mode 100644 alembic/versions/6863dda76ea2_category_selector.py diff --git a/alembic/versions/6863dda76ea2_category_selector.py b/alembic/versions/6863dda76ea2_category_selector.py new file mode 100644 index 0000000..5ccac48 --- /dev/null +++ b/alembic/versions/6863dda76ea2_category_selector.py @@ -0,0 +1,54 @@ +"""Category selector + +Revision ID: 6863dda76ea2 +Revises: 83f4c9837f6e +Create Date: 2022-12-08 00:56:59.032641+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "6863dda76ea2" +down_revision = "83f4c9837f6e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "categories_selector", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column( + "selector", + sa.Enum( + "unknown", + "nullifier", + "vacations", + "rules", + "algorithm", + "manual", + name="selector", + schema="transactions", + inherit_schema=True, + ), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.categorized.id"], + name=op.f("fk_categories_selector_id_categorized"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_categories_selector")), + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("categories_selector", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index be1ceaa..6bc0ef9 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -121,11 +121,18 @@ class TransactionCategory(Base): name: Mapped[str] = mapped_column(ForeignKey(Category.name)) original: Mapped[Transaction] = relationship(back_populates="category") + selector: Mapped[CategorySelector] = relationship(back_populates="category") def __repr__(self) -> str: return f"Category({self.name})" +catfk = Annotated[ + int, + mapped_column(BigInteger, ForeignKey(TransactionCategory.id, ondelete="CASCADE")), +] + + class Note(Base): __tablename__ = "notes" @@ -165,3 +172,27 @@ class CategoryRule(Base): ForeignKey(Category.name, ondelete="CASCADE"), primary_key=True ) rule: Mapped[str] = mapped_column(primary_key=True) + + +class Selector(enum.Enum): + unknown = enum.auto() + nullifier = enum.auto() + vacations = enum.auto() + rules = enum.auto() + algorithm = enum.auto() + manual = enum.auto() + + +categoryselector = Annotated[ + Selector, + mapped_column(Enum(Selector, inherit_schema=True), default=Selector.unknown), +] + + +class CategorySelector(Base): + __tablename__ = "categories_selector" + + id: Mapped[catfk] = mapped_column(primary_key=True) + selector: Mapped[categoryselector] + + category: Mapped[TransactionCategory] = relationship(back_populates="selector") From d11bc6df1d953cbe8be010d260f14634481eeaad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 13:31:31 +0000 Subject: [PATCH 12/66] Adds category recurring possibility --- .../d18cbd50f7c6_category_schedule.py | 53 +++++++++++++++++++ pfbudget/db/model.py | 37 ++++++++++--- 2 files changed, 83 insertions(+), 7 deletions(-) create mode 100644 alembic/versions/d18cbd50f7c6_category_schedule.py diff --git a/alembic/versions/d18cbd50f7c6_category_schedule.py b/alembic/versions/d18cbd50f7c6_category_schedule.py new file mode 100644 index 0000000..c6da5ab --- /dev/null +++ b/alembic/versions/d18cbd50f7c6_category_schedule.py @@ -0,0 +1,53 @@ +"""Category schedule + +Revision ID: d18cbd50f7c6 +Revises: 6863dda76ea2 +Create Date: 2022-12-08 13:30:29.048811+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "d18cbd50f7c6" +down_revision = "6863dda76ea2" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "categories_schedules", + sa.Column("name", sa.String(), nullable=False), + sa.Column("recurring", sa.Boolean(), nullable=False), + sa.Column( + "period", + sa.Enum( + "daily", + "monthly", + "yearly", + name="period", + schema="transactions", + inherit_schema=True, + ), + nullable=True, + ), + sa.Column("period_multiplier", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["name"], + ["transactions.categories_available.name"], + name=op.f("fk_categories_schedules_name_categories_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("name", name=op.f("pk_categories_schedules")), + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("categories_schedules", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 6bc0ef9..a852768 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -107,6 +107,7 @@ class Category(Base): rules: Mapped[Optional[set[CategoryRule]]] = relationship( cascade="all, delete-orphan", passive_deletes=True ) + schedule: Mapped[CategorySchedule] = relationship() def __repr__(self) -> str: return ( @@ -114,6 +115,12 @@ class Category(Base): ) +catfk = Annotated[ + str, + mapped_column(ForeignKey(Category.name, ondelete="CASCADE")), +] + + class TransactionCategory(Base): __tablename__ = "categorized" @@ -127,12 +134,6 @@ class TransactionCategory(Base): return f"Category({self.name})" -catfk = Annotated[ - int, - mapped_column(BigInteger, ForeignKey(TransactionCategory.id, ondelete="CASCADE")), -] - - class Note(Base): __tablename__ = "notes" @@ -192,7 +193,29 @@ categoryselector = Annotated[ class CategorySelector(Base): __tablename__ = "categories_selector" - id: Mapped[catfk] = mapped_column(primary_key=True) + id: Mapped[int] = mapped_column( + BigInteger, + ForeignKey(TransactionCategory.id, ondelete="CASCADE"), + primary_key=True, + ) selector: Mapped[categoryselector] category: Mapped[TransactionCategory] = relationship(back_populates="selector") + + +class Period(enum.Enum): + daily = enum.auto() + monthly = enum.auto() + yearly = enum.auto() + + +scheduleperiod = Annotated[Selector, mapped_column(Enum(Period, inherit_schema=True))] + + +class CategorySchedule(Base): + __tablename__ = "categories_schedules" + + name: Mapped[catfk] = mapped_column(primary_key=True) + recurring: Mapped[bool] + period: Mapped[Optional[scheduleperiod]] + period_multiplier: Mapped[Optional[int]] From 72974c90aa43feb8d0809ee526af2cf490b92aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 16:15:53 +0000 Subject: [PATCH 13/66] Operations refactor Unify the manager operations under one enum. Start cleaning up the argparser structure to easily add new operations. --- pfbudget/cli/runnable.py | 141 ++++++++++++++++++--------------------- pfbudget/common/types.py | 16 ++--- pfbudget/core/manager.py | 105 +++++++++++------------------ 3 files changed, 109 insertions(+), 153 deletions(-) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 7e89ef7..3891241 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -2,7 +2,7 @@ from pathlib import Path import argparse import re -from pfbudget.common.types import Command, Operation +from pfbudget.common.types import Operation from pfbudget.core.categories import categorize_data from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput @@ -29,18 +29,18 @@ class DataFileMissing(Exception): def argparser() -> argparse.ArgumentParser: - help = argparse.ArgumentParser(add_help=False) - help.add_argument( + universal = argparse.ArgumentParser(add_help=False) + universal.add_argument( "-db", "--database", nargs="?", help="select current database", default=DEFAULT_DB, ) - help.add_argument( + universal.add_argument( "-q", "--quiet", action="store_true", help="reduces the amount of verbose" ) - help.add_argument( + universal.add_argument( "-v", "--verbose", action="store_true", help="increases the amount of verbose" ) @@ -54,7 +54,7 @@ def argparser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( description="does cool finance stuff", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) parser.add_argument( @@ -66,7 +66,7 @@ def argparser() -> argparse.ArgumentParser: ).group(1), ) - subparsers = parser.add_subparsers(dest="command", required=True) + subparsers = parser.add_subparsers(required=True) """ Init @@ -74,10 +74,10 @@ def argparser() -> argparse.ArgumentParser: p_init = subparsers.add_parser( "init", description="Initializes the SQLite3 database", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_init.set_defaults(command=Command.Init) + p_init.set_defaults(command=Operation.Init) """ Exporting @@ -85,7 +85,7 @@ def argparser() -> argparse.ArgumentParser: p_export = subparsers.add_parser( "export", description="Exports the selected database to a .csv file", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_export.set_defaults(func=lambda args: DatabaseClient(args.database).export()) @@ -96,14 +96,14 @@ def argparser() -> argparse.ArgumentParser: p_parse = subparsers.add_parser( "parse", description="Parses and adds the requested transactions into the selected database", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_parse.add_argument("path", nargs="+", type=str) p_parse.add_argument("--bank", nargs=1, type=str) p_parse.add_argument("--creditcard", nargs=1, type=str) p_parse.add_argument("--category", nargs=1, type=int) - p_parse.set_defaults(command=Command.Parse) + p_parse.set_defaults(command=Operation.Parse) """ Categorizing @@ -111,10 +111,10 @@ def argparser() -> argparse.ArgumentParser: p_categorize = subparsers.add_parser( "categorize", description="Categorizes the transactions in the selected database", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_categorize.set_defaults(command=Command.Categorize) + p_categorize.set_defaults(command=Operation.Categorize) """ Graph @@ -122,7 +122,7 @@ def argparser() -> argparse.ArgumentParser: p_graph = subparsers.add_parser( "graph", description="Graph of the transactions", - parents=[help, period], + parents=[universal, period], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_graph.add_argument( @@ -142,7 +142,7 @@ def argparser() -> argparse.ArgumentParser: p_report = subparsers.add_parser( "report", description="Prints report of transaction groups", - parents=[help, period], + parents=[universal, period], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_report.add_argument( @@ -161,7 +161,7 @@ def argparser() -> argparse.ArgumentParser: p_register = subparsers.add_parser( "register", description="Register a bank", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_register.add_argument("bank", type=str, nargs=1, help="bank option help") @@ -169,7 +169,7 @@ def argparser() -> argparse.ArgumentParser: "--requisition", type=str, nargs=1, help="requisition option help" ) p_register.add_argument("--invert", action="store_true") - p_register.set_defaults(command=Command.Register) + p_register.set_defaults(command=Operation.Register) """ Unregister bank @@ -177,11 +177,11 @@ def argparser() -> argparse.ArgumentParser: p_register = subparsers.add_parser( "unregister", description="Unregister a bank", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_register.add_argument("bank", type=str, nargs=1, help="bank option help") - p_register.set_defaults(command=Command.Unregister) + p_register.set_defaults(command=Operation.Unregister) """ Nordigen API @@ -189,10 +189,10 @@ def argparser() -> argparse.ArgumentParser: p_nordigen_access = subparsers.add_parser( "token", description="Get new access token", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_nordigen_access.set_defaults(command=Command.Token) + p_nordigen_access.set_defaults(command=Operation.Token) """ (Re)new bank requisition ID @@ -200,12 +200,12 @@ def argparser() -> argparse.ArgumentParser: p_nordigen_access = subparsers.add_parser( "renew", description="(Re)new the Bank requisition ID", - parents=[help], + parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_nordigen_access.add_argument("name", nargs=1, type=str) p_nordigen_access.add_argument("country", nargs=1, type=str) - p_nordigen_access.set_defaults(command=Command.Renew) + p_nordigen_access.set_defaults(command=Operation.Renew) """ Downloading through Nordigen API @@ -213,13 +213,13 @@ def argparser() -> argparse.ArgumentParser: p_nordigen_download = subparsers.add_parser( "download", description="Downloads transactions using Nordigen API", - parents=[help, period], + parents=[universal, period], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) p_nordigen_download.add_argument("--id", nargs="+", type=str) p_nordigen_download.add_argument("--name", nargs="+", type=str) p_nordigen_download.add_argument("--all", action="store_true") - p_nordigen_download.set_defaults(command=Command.Download) + p_nordigen_download.set_defaults(command=Operation.Download) # """ # List available banks on Nordigen API @@ -249,55 +249,9 @@ def argparser() -> argparse.ArgumentParser: # func=lambda args: manager.parser(JsonParser(vars(args))) # ) - # Add category - p_categories = subparsers.add_parser( - "category", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_commands = p_categories.add_subparsers(dest="command", required=True) - p_categories_add = p_categories_commands.add_parser( - "add", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_add.add_argument("category", nargs="+", type=str) - p_categories_add.add_argument("--group", nargs="?", type=str) - p_categories_add.set_defaults(command=Command.Category, op=Operation.Add) - - p_categories_remove = p_categories_commands.add_parser( - "remove", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_remove.add_argument("category", nargs="+", type=str) - p_categories_remove.add_argument("--group", nargs="?", type=str) - p_categories_remove.set_defaults(command=Command.Category, op=Operation.Remove) - - p_categories_addgroup = p_categories_commands.add_parser( - "addgroup", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_addgroup.add_argument("group", nargs="+", type=str) - p_categories_addgroup.set_defaults(command=Command.Category, op=Operation.AddGroup) - - p_categories_removegroup = p_categories_commands.add_parser( - "removegroup", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_removegroup.add_argument("group", nargs="+", type=str) - p_categories_removegroup.set_defaults(command=Command.Category, op=Operation.RemoveGroup) - - p_categories_updategroup = p_categories_commands.add_parser( - "updategroup", - parents=[help], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_categories_updategroup.add_argument("category", nargs="+", type=str) - p_categories_updategroup.add_argument("--group", nargs=1, type=str) - p_categories_updategroup.set_defaults(command=Command.Category, op=Operation.UpdateGroup) + # Categories + category_parser = subparsers.add_parser("category", parents=[universal]) + category(category_parser, universal) return parser @@ -362,7 +316,40 @@ def download(manager, args: dict): manager.parser(NordigenInput(manager, args, start, end)) +def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add", parents=[universal]) + add.set_defaults(op=Operation.CategoryAdd) + add.add_argument("category", nargs="+", type=str) + add.add_argument("--group", nargs="?", type=str) + + remove = commands.add_parser("remove", parents=[universal]) + remove.set_defaults(op=Operation.CategoryRemove) + remove.add_argument("category", nargs="+", type=str) + + update = commands.add_parser("update", parents=[universal]) + update.set_defaults(op=Operation.CategoryUpdate) + update.add_argument("category", nargs="+", type=str) + update.add_argument("--group", nargs="?", type=str) + + group = commands.add_parser("group", parents=[universal]) + category_group(group, universal) + + +def category_group(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add", parents=[universal]) + add.set_defaults(op=Operation.GroupAdd) + add.add_argument("group", nargs="+", type=str) + + remove = commands.add_parser("remove", parents=[universal]) + remove.set_defaults(op=Operation.GroupRemove) + remove.add_argument("group", nargs="+", type=str) + + def run(): args = vars(argparser().parse_args()) - assert "command" in args, "No command selected" - return args["command"], args + assert "op" in args, "No operation selected" + return args["op"], args diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 829552f..5439d7b 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -4,7 +4,7 @@ from decimal import Decimal, InvalidOperation from enum import Enum, auto -class Command(Enum): +class Operation(Enum): Init = auto() Parse = auto() Download = auto() @@ -13,15 +13,11 @@ class Command(Enum): Unregister = auto() Token = auto() Renew = auto() - Category = auto() - - -class Operation(Enum): - Add = auto() - Remove = auto() - AddGroup = auto() - RemoveGroup = auto() - UpdateGroup = auto() + CategoryAdd = auto() + CategoryUpdate = auto() + CategoryRemove = auto() + GroupAdd = auto() + GroupRemove = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index b964ba8..4f9ea11 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -3,7 +3,7 @@ from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient from pfbudget.db.model import Category, CategoryGroup -from pfbudget.common.types import Command, Operation +from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -11,99 +11,72 @@ from pfbudget.cli.runnable import download, parse class Manager: - def __init__(self, command: Command, args: dict): - self.__command = command + def __init__(self, op: Operation, args: dict): + self._operation = op self._args = args - match (command): - case Command.Init: - pass - case Command.Parse: - pass - case Command.Download: - pass - case Command.Categorize: - pass - case Command.Register: - pass - case Command.Unregister: - pass - case Command.Token: - pass - case Command.Renew: - pass - case Command.Category: - pass assert "database" in args, "ArgParser didn't include db" self._db = args["database"] def start(self): - match (self.__command): - case Command.Init: + match (self._operation): + case Operation.Init: pass - case Command.Parse: + case Operation.Parse: # TODO this is a monstrosity, remove when possible parse(self, self.args) - case Command.Download: + case Operation.Download: # TODO this is a monstrosity, remove when possible download(self, self.args) - case Command.Categorize: - self.categorize(self.args) - case Command.Register: + case Operation.Categorize: + self.categorize() + + case Operation.Register: # self._db = DbClient(args["database"]) # self.register(args) pass - case Command.Unregister: + case Operation.Unregister: # self._db = DbClient(args["database"]) # self.unregister(args) pass - case Command.Token: + case Operation.Token: NordigenClient(self).token() - case Command.Renew: + case Operation.Renew: NordigenClient(self).requisition( self.args["name"], self.args["country"] ) - case Command.Category: - assert "op" in self.args, "category operation not defined" - + case Operation.CategoryAdd: with self.db.session() as session: - match self.args["op"]: - case Operation.Add: - for category in self.args["category"]: - session.addcategory( - Category(name=category, group=self.args["group"]) - ) + for category in self.args["category"]: + session.addcategory( + Category(name=category, group=self.args["group"]) + ) - case Operation.Remove: - session.removecategory( - [ - Category(name=category) - for category in self.args["category"] - ] - ) + case Operation.CategoryUpdate: + with self.db.session() as session: + session.updategroup( + [Category(name=category) for category in self.args["category"]], + self.args["group"][0], + ) - case Operation.UpdateGroup: - session.updategroup( - [ - Category(name=category) - for category in self.args["category"] - ], - self.args["group"][0], - ) + case Operation.CategoryRemove: + with self.db.session() as session: + session.removecategory( + [Category(name=category) for category in self.args["category"]] + ) - case Operation.AddGroup: - for group in self.args["group"]: - session.addcategorygroup(CategoryGroup(name=group)) + case Operation.GroupAdd: + with self.db.session() as session: + for group in self.args["group"]: + session.addcategorygroup(CategoryGroup(name=group)) - case Operation.RemoveGroup: - session.removecategorygroup( - [ - CategoryGroup(name=group) - for group in self.args["group"] - ] - ) + case Operation.GroupRemove: + with self.db.session() as session: + session.removecategorygroup( + [CategoryGroup(name=group) for group in self.args["group"]] + ) # def init(self): # client = DatabaseClient(self.__db) From 6f97b883fb11c77643eddde3e5425811746629f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 16:44:07 +0000 Subject: [PATCH 14/66] Add weekly option on the schedule period --- .../versions/e77395969585_weekly_period.py | 37 +++++++++++++++++++ pfbudget/db/model.py | 7 ++-- 2 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/e77395969585_weekly_period.py diff --git a/alembic/versions/e77395969585_weekly_period.py b/alembic/versions/e77395969585_weekly_period.py new file mode 100644 index 0000000..bd42cb0 --- /dev/null +++ b/alembic/versions/e77395969585_weekly_period.py @@ -0,0 +1,37 @@ +"""Weekly period + +Revision ID: e77395969585 +Revises: d18cbd50f7c6 +Create Date: 2022-12-08 16:35:27.506504+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "e77395969585" +down_revision = "d18cbd50f7c6" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TYPE transactions.period ADD VALUE 'weekly' AFTER 'daily'") + + +def downgrade() -> None: + op.execute( + """CREATE TYPE transactions.period_new + AS ENUM ('daily', 'monthly', 'yearly') + """ + ) + op.execute("UPDATE transactions.categories_schedules SET period = DEFAULT WHERE period = 'weekly'") + op.execute( + """ALTER TABLE transactions.categories_schedules + ALTER COLUMN period TYPE transactions.period_new + USING period::text::transactions.period_new + """ + ) + op.execute("DROP TYPE transactions.period") + op.execute("ALTER TYPE transactions.period_new RENAME TO period") diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index a852768..d5704e5 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -204,9 +204,10 @@ class CategorySelector(Base): class Period(enum.Enum): - daily = enum.auto() - monthly = enum.auto() - yearly = enum.auto() + daily = "daily" + weekly = "weekly" + monthly = "monthly" + yearly = "yearly" scheduleperiod = Annotated[Selector, mapped_column(Enum(Period, inherit_schema=True))] From f09b328c66a285e968ded96570d5f49b6f39e8a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 20:11:33 +0000 Subject: [PATCH 15/66] Package now run as python3 -m pfbudget Moves in line in the normal way python packages are run. https://docs.python.org/3/library/__main__.html --- main.py | 5 ----- pfbudget/__init__.py | 4 ++-- pfbudget/__main__.py | 8 ++++++-- 3 files changed, 8 insertions(+), 9 deletions(-) delete mode 100644 main.py diff --git a/main.py b/main.py deleted file mode 100644 index 5ed5689..0000000 --- a/main.py +++ /dev/null @@ -1,5 +0,0 @@ -from pfbudget import Manager, run - -if __name__ == "__main__": - command, args = run() - Manager(command, args).start() diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index 8b5a5fa..2a836a1 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -1,8 +1,8 @@ -__all__ = ["run", "parse_data", "categorize_data"] +__all__ = ["argparser", "Manager", "parse_data", "categorize_data"] __author__ = "Luís Murta" __version__ = "0.1" from pfbudget.core.categories import categorize_data from pfbudget.core.manager import Manager -from pfbudget.cli.runnable import run +from pfbudget.cli.runnable import argparser from pfbudget.input.parsers import parse_data diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index eba7182..a5fa7de 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,4 +1,8 @@ -from pfbudget.cli.runnable import run +import pfbudget + if __name__ == "__main__": - run() + argparser = pfbudget.argparser() + args = vars(argparser.parse_args()) + assert "op" in args, "No operation selected" + pfbudget.Manager(args["op"], args).start() From 63d6a2aab71b9f473248f82bab459402303d88c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 8 Dec 2022 20:15:30 +0000 Subject: [PATCH 16/66] Finishes category schedule implementation --- pfbudget/cli/runnable.py | 7 +++++++ pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 17 ++++++++++++++++- pfbudget/db/client.py | 35 ++++++++++++++++++++++++++++++++++- pfbudget/db/model.py | 15 +++++++++++---- 5 files changed, 69 insertions(+), 6 deletions(-) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 3891241..766baea 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -4,6 +4,7 @@ import re from pfbudget.common.types import Operation from pfbudget.core.categories import categorize_data +from pfbudget.db.model import Period from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput from pfbudget.db.sqlite import DatabaseClient @@ -333,6 +334,12 @@ def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser update.add_argument("category", nargs="+", type=str) update.add_argument("--group", nargs="?", type=str) + schedule = commands.add_parser("schedule", parents=[universal]) + schedule.set_defaults(op=Operation.CategorySchedule) + schedule.add_argument("category", nargs="+", type=str) + schedule.add_argument("period", nargs=1, choices=[e.value for e in Period]) + schedule.add_argument("--frequency", nargs=1, default=[1], type=int) + group = commands.add_parser("group", parents=[universal]) category_group(group, universal) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 5439d7b..2c30b9c 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -16,6 +16,7 @@ class Operation(Enum): CategoryAdd = auto() CategoryUpdate = auto() CategoryRemove = auto() + CategorySchedule = auto() GroupAdd = auto() GroupRemove = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 4f9ea11..9dd2911 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -2,7 +2,7 @@ from pfbudget.input.input import Input from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient -from pfbudget.db.model import Category, CategoryGroup +from pfbudget.db.model import Category, CategoryGroup, CategorySchedule from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -67,6 +67,21 @@ class Manager: [Category(name=category) for category in self.args["category"]] ) + case Operation.CategorySchedule: + assert ( + "period" in self.args and "frequency" in self.args + ), "Schedule not well defined" + + with self.db.session() as session: + session.updateschedules( + [Category(name=category) for category in self.args["category"]], + CategorySchedule( + recurring=True, + period=self.args["period"][0], + period_multiplier=self.args["frequency"][0], + ), + ) + case Operation.GroupAdd: with self.db.session() as session: for group in self.args["group"]: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 88bc213..d2da511 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,7 +1,16 @@ +from copy import deepcopy +from dataclasses import asdict from sqlalchemy import create_engine, delete, select, update +from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session, joinedload, selectinload -from pfbudget.db.model import Bank, Category, CategoryGroup, Transaction +from pfbudget.db.model import ( + Bank, + Category, + CategoryGroup, + CategorySchedule, + Transaction, +) # import logging @@ -91,6 +100,30 @@ class DbClient: ) self.__session.execute(stmt) + def updateschedules( + self, categories: list[Category], schedule: CategorySchedule + ): + stmt = insert(CategorySchedule).values( + [ + dict( + name=cat.name, + recurring=schedule.recurring, + period=schedule.period, + period_multiplier=schedule.period_multiplier, + ) + for cat in categories + ] + ) + stmt = stmt.on_conflict_do_update( + index_elements=[CategorySchedule.name], + set_=dict( + recurring=stmt.excluded.recurring, + period=stmt.excluded.period, + period_multiplier=stmt.excluded.period_multiplier, + ), + ) + self.__session.execute(stmt) + def addcategorygroup(self, group: CategoryGroup): self.__session.add(group) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index d5704e5..17470b7 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -107,12 +107,10 @@ class Category(Base): rules: Mapped[Optional[set[CategoryRule]]] = relationship( cascade="all, delete-orphan", passive_deletes=True ) - schedule: Mapped[CategorySchedule] = relationship() + schedule: Mapped[CategorySchedule] = relationship(back_populates="category") def __repr__(self) -> str: - return ( - f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)})" - ) + return f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)}, schedule={self.schedule})" catfk = Annotated[ @@ -220,3 +218,12 @@ class CategorySchedule(Base): recurring: Mapped[bool] period: Mapped[Optional[scheduleperiod]] period_multiplier: Mapped[Optional[int]] + + category: Mapped[Category] = relationship(back_populates="schedule") + + def __repr__(self) -> str: + return ( + f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier})" + if self.recurring + else f"{self.name} has no Schedule" + ) From 471331ffc9534d2b65b28f7f8ae14844b0804238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 9 Dec 2022 19:38:12 +0000 Subject: [PATCH 17/66] Categories rules for rule-based categorization This patch extends the categories_rules with a rule for each field of a transaction. It also changes the ORM classes to behave as dataclasses again. --- .../8b5d5fbc8211_transaction_based_rules.py | 92 +++++++++++++++++++ pfbudget/cli/runnable.py | 9 ++ pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 12 +-- pfbudget/db/client.py | 4 + pfbudget/db/model.py | 44 ++++++--- 6 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 alembic/versions/8b5d5fbc8211_transaction_based_rules.py diff --git a/alembic/versions/8b5d5fbc8211_transaction_based_rules.py b/alembic/versions/8b5d5fbc8211_transaction_based_rules.py new file mode 100644 index 0000000..4874fff --- /dev/null +++ b/alembic/versions/8b5d5fbc8211_transaction_based_rules.py @@ -0,0 +1,92 @@ +"""Transaction based rules + +Revision ID: 8b5d5fbc8211 +Revises: e77395969585 +Create Date: 2022-12-08 21:05:41.378466+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "8b5d5fbc8211" +down_revision = "e77395969585" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.drop_constraint("pk_categories_rules", "categories_rules", schema="transactions") + op.execute(sa.schema.CreateSequence(sa.schema.Sequence("categories_rules_id_seq", schema="transactions"))) + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "categories_rules", + sa.Column( + "id", + sa.BigInteger(), + autoincrement=True, + nullable=False, + server_default=sa.text( + "nextval('transactions.categories_rules_id_seq'::regclass)" + ), + ), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("date", sa.String(), nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("description", sa.String(), nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("bank", sa.String(), nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("min_amount", sa.Float(), nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("max_amount", sa.Float(), nullable=True), + schema="transactions", + ) + op.drop_column("categories_rules", "rule", schema="transactions") + # ### end Alembic commands ### + op.create_primary_key( + "pk_categories_rules", + "categories_rules", + ["id"], + schema="transactions", + ) + + +def downgrade() -> None: + op.drop_constraint("pk_categories_rules", "categories_rules", schema="transactions") + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "categories_rules", + sa.Column("rule", sa.String(), autoincrement=False, nullable=False), + schema="transactions", + ) + op.drop_column("categories_rules", "max_amount", schema="transactions") + op.drop_column("categories_rules", "min_amount", schema="transactions") + op.drop_column("categories_rules", "bank", schema="transactions") + op.drop_column("categories_rules", "description", schema="transactions") + op.drop_column("categories_rules", "date", schema="transactions") + op.drop_column("categories_rules", "id", schema="transactions") + # ### end Alembic commands ### + op.execute(sa.schema.DropSequence(sa.schema.Sequence("categories_rules_id_seq", schema="transactions"))) + op.create_primary_key( + "pk_categories_rules", + "categories_rules", + ["name", "rule"], + schema="transactions", + ) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 766baea..49f6f77 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -340,6 +340,15 @@ def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser schedule.add_argument("period", nargs=1, choices=[e.value for e in Period]) schedule.add_argument("--frequency", nargs=1, default=[1], type=int) + rule = commands.add_parser("rule", parents=[universal]) + rule.set_defaults(op=Operation.CategoryRule) + rule.add_argument("category", nargs="+", type=str) + rule.add_argument("--date", nargs=1, type=str) + rule.add_argument("--description", nargs=1, type=str) + rule.add_argument("--bank", nargs=1, type=str) + rule.add_argument("--min", nargs=1, type=float) + rule.add_argument("--max", nargs=1, type=float) + group = commands.add_parser("group", parents=[universal]) category_group(group, universal) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 2c30b9c..dd8f431 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -17,6 +17,7 @@ class Operation(Enum): CategoryUpdate = auto() CategoryRemove = auto() CategorySchedule = auto() + CategoryRule = auto() GroupAdd = auto() GroupRemove = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 9dd2911..fb0d54f 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -2,7 +2,7 @@ from pfbudget.input.input import Input from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient -from pfbudget.db.model import Category, CategoryGroup, CategorySchedule +from pfbudget.db.model import Category, CategoryGroup, CategoryRule, CategorySchedule from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -72,15 +72,9 @@ class Manager: "period" in self.args and "frequency" in self.args ), "Schedule not well defined" + case Operation.CategoryRule: with self.db.session() as session: - session.updateschedules( - [Category(name=category) for category in self.args["category"]], - CategorySchedule( - recurring=True, - period=self.args["period"][0], - period_multiplier=self.args["frequency"][0], - ), - ) + session.addrules(params) case Operation.GroupAdd: with self.db.session() as session: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index d2da511..9e44a93 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -8,6 +8,7 @@ from pfbudget.db.model import ( Bank, Category, CategoryGroup, + CategoryRule, CategorySchedule, Transaction, ) @@ -124,6 +125,9 @@ class DbClient: ) self.__session.execute(stmt) + def addrules(self, rules: list[CategoryRule]): + self.__session.add_all(rules) + def addcategorygroup(self, group: CategoryGroup): self.__session.add(group) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 17470b7..be0c5dc 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -9,7 +9,13 @@ from sqlalchemy import ( String, Text, ) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +from sqlalchemy.orm import ( + DeclarativeBase, + Mapped, + mapped_column, + MappedAsDataclass, + relationship, +) from decimal import Decimal from typing import Annotated, Optional @@ -17,7 +23,7 @@ import datetime as dt import enum -class Base(DeclarativeBase): +class Base(MappedAsDataclass, DeclarativeBase): __table_args__ = {"schema": "transactions"} metadata = MetaData( naming_convention={ @@ -69,18 +75,21 @@ money = Annotated[Decimal, mapped_column(Numeric(16, 2), nullable=False)] class Transaction(Base): __tablename__ = "originals" - id: Mapped[idpk] = mapped_column(autoincrement=True) + id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) date: Mapped[dt.date] description: Mapped[Optional[str]] bank: Mapped[bankfk] amount: Mapped[money] category: Mapped[Optional[TransactionCategory]] = relationship( - back_populates="original", lazy="joined" + back_populates="original", lazy="joined", default=None ) - note: Mapped[Optional[Note]] = relationship(back_populates="original") + note: Mapped[Optional[Note]] = relationship(back_populates="original", default=None) tags: Mapped[Optional[set[Tag]]] = relationship( - back_populates="original", cascade="all, delete-orphan", passive_deletes=True + back_populates="original", + cascade="all, delete-orphan", + passive_deletes=True, + default=None, ) def __repr__(self) -> str: @@ -102,12 +111,16 @@ class Category(Base): __tablename__ = "categories_available" name: Mapped[str] = mapped_column(primary_key=True) - group: Mapped[Optional[str]] = mapped_column(ForeignKey(CategoryGroup.name)) + group: Mapped[Optional[str]] = mapped_column( + ForeignKey(CategoryGroup.name), default=None + ) rules: Mapped[Optional[set[CategoryRule]]] = relationship( - cascade="all, delete-orphan", passive_deletes=True + cascade="all, delete-orphan", passive_deletes=True, default=None + ) + schedule: Mapped[CategorySchedule] = relationship( + back_populates="category", default=None ) - schedule: Mapped[CategorySchedule] = relationship(back_populates="category") def __repr__(self) -> str: return f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)}, schedule={self.schedule})" @@ -122,7 +135,7 @@ catfk = Annotated[ class TransactionCategory(Base): __tablename__ = "categorized" - id: Mapped[idfk] = mapped_column(primary_key=True) + id: Mapped[idfk] = mapped_column(primary_key=True, init=False) name: Mapped[str] = mapped_column(ForeignKey(Category.name)) original: Mapped[Transaction] = relationship(back_populates="category") @@ -167,10 +180,13 @@ class Tag(Base): class CategoryRule(Base): __tablename__ = "categories_rules" - name: Mapped[str] = mapped_column( - ForeignKey(Category.name, ondelete="CASCADE"), primary_key=True - ) - rule: Mapped[str] = mapped_column(primary_key=True) + id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) + name: Mapped[catfk] = mapped_column() + date: Mapped[Optional[str]] = mapped_column() + description: Mapped[Optional[str]] = mapped_column() + bank: Mapped[Optional[str]] = mapped_column() + min_amount: Mapped[Optional[float]] = mapped_column() + max_amount: Mapped[Optional[float]] = mapped_column() class Selector(enum.Enum): From f721e6a910bd40301dc06d7c43a8b1e57942bd1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 9 Dec 2022 19:43:09 +0000 Subject: [PATCH 18/66] [Refactor] Decouple CLI arguments from Manager The Manager doesn't need to know that it was called from the CLI, so it now is initialized with the database and performs an action, based on the operation it receives and its parameters. The work isn't finished, some Manager actions are still based on the CLI arguments. The CLI logic and creation of parameters to pass to the manager have been moved to the __main__.py file, which brings it to line to the program being called as a package from the command line. --- pfbudget/__init__.py | 3 ++ pfbudget/__main__.py | 72 ++++++++++++++++++++++++++++++++++++++-- pfbudget/core/manager.py | 30 ++++++----------- pfbudget/db/client.py | 22 +++--------- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index 2a836a1..abbb667 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -2,7 +2,10 @@ __all__ = ["argparser", "Manager", "parse_data", "categorize_data"] __author__ = "Luís Murta" __version__ = "0.1" +from pfbudget.common.types import Operation from pfbudget.core.categories import categorize_data from pfbudget.core.manager import Manager from pfbudget.cli.runnable import argparser from pfbudget.input.parsers import parse_data + +import pfbudget.db.model as types diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index a5fa7de..023331e 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -4,5 +4,73 @@ import pfbudget if __name__ == "__main__": argparser = pfbudget.argparser() args = vars(argparser.parse_args()) - assert "op" in args, "No operation selected" - pfbudget.Manager(args["op"], args).start() + + assert "op" in args, "No pfbudget.Operation selected" + op: pfbudget.Operation = args["op"] + + assert "database" in args, "No database selected" + db = args["database"] + + params = None + match (op): + case pfbudget.Operation.CategoryAdd: + assert args.keys() >= {"category", "group"}, "argparser ill defined" + params = [ + pfbudget.types.Category(cat, args["group"][0]) + for cat in args["category"] + ] + + case pfbudget.Operation.CategoryUpdate: + assert args.keys() >= {"category", "group"}, "argparser ill defined" + params = [pfbudget.types.Category(cat) for cat in args["category"]] + params.append(args["group"][0]) + + case pfbudget.Operation.CategoryRemove: + assert "category" in args, "argparser ill defined" + params = [pfbudget.types.Category(cat) for cat in args["category"]] + + case pfbudget.Operation.CategorySchedule: + assert args.keys() >= { + "category", + "period", + "frequency", + }, "argparser ill defined" + + params = [ + pfbudget.types.CategorySchedule( + cat, True, args["period"][0], args["frequency"][0] + ) + for cat in args["category"] + ] + + case pfbudget.Operation.CategoryRule: + assert args.keys() >= { + "category", + "date", + "description", + "bank", + "min", + "max", + }, "argparser ill defined" + + params = [ + pfbudget.types.CategoryRule( + cat, + args["date"][0] if args["date"] else None, + args["description"][0] if args["description"] else None, + args["bank"][0] if args["bank"] else None, + args["min"][0] if args["min"] else None, + args["max"][0] if args["max"] else None, + ) + for cat in args["category"] + ] + + case pfbudget.Operation.GroupAdd: + assert "group" in args, "argparser ill defined" + params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] + + case pfbudget.Operation.GroupRemove: + assert "group" in args, "argparser ill defined" + params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] + + pfbudget.Manager(db, args).action(op, params) diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index fb0d54f..2227ccf 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -11,15 +11,14 @@ from pfbudget.cli.runnable import download, parse class Manager: - def __init__(self, op: Operation, args: dict): - self._operation = op + def __init__(self, db: str, args: dict): self._args = args + print(args) - assert "database" in args, "ArgParser didn't include db" - self._db = args["database"] + self._db = db - def start(self): - match (self._operation): + def action(self, op: Operation, params: list): + match (op): case Operation.Init: pass case Operation.Parse: @@ -49,28 +48,19 @@ class Manager: case Operation.CategoryAdd: with self.db.session() as session: - for category in self.args["category"]: - session.addcategory( - Category(name=category, group=self.args["group"]) - ) + session.addcategories(params) case Operation.CategoryUpdate: with self.db.session() as session: - session.updategroup( - [Category(name=category) for category in self.args["category"]], - self.args["group"][0], - ) + session.updategroup(*params) case Operation.CategoryRemove: with self.db.session() as session: - session.removecategory( - [Category(name=category) for category in self.args["category"]] - ) + session.removecategories(params) case Operation.CategorySchedule: - assert ( - "period" in self.args and "frequency" in self.args - ), "Schedule not well defined" + with self.db.session() as session: + session.updateschedules(params) case Operation.CategoryRule: with self.db.session() as session: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 9e44a93..28c2b39 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -84,10 +84,10 @@ class DbClient: def add(self, transactions: list[Transaction]): self.__session.add_all(transactions) - def addcategory(self, category: Category): - self.__session.add(category) + def addcategories(self, category: list[Category]): + self.__session.add_all(category) - def removecategory(self, categories: list[Category]): + def removecategories(self, categories: list[Category]): stmt = delete(Category).where( Category.name.in_([cat.name for cat in categories]) ) @@ -101,20 +101,8 @@ class DbClient: ) self.__session.execute(stmt) - def updateschedules( - self, categories: list[Category], schedule: CategorySchedule - ): - stmt = insert(CategorySchedule).values( - [ - dict( - name=cat.name, - recurring=schedule.recurring, - period=schedule.period, - period_multiplier=schedule.period_multiplier, - ) - for cat in categories - ] - ) + def updateschedules(self, schedules: list[CategorySchedule]): + stmt = insert(CategorySchedule).values([asdict(s) for s in schedules]) stmt = stmt.on_conflict_do_update( index_elements=[CategorySchedule.name], set_=dict( From fed007ff89c39ac1e5a87e8134861771fa03a34c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 10 Dec 2022 00:20:41 +0000 Subject: [PATCH 19/66] Changes recurring categories boolean to int to indicate the number of expected transactions per time period. --- ...7ef9c_amount_of_transactions_per_period.py | 38 +++++++++++++++++++ pfbudget/db/model.py | 8 +--- 2 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/83603bb7ef9c_amount_of_transactions_per_period.py diff --git a/alembic/versions/83603bb7ef9c_amount_of_transactions_per_period.py b/alembic/versions/83603bb7ef9c_amount_of_transactions_per_period.py new file mode 100644 index 0000000..887b452 --- /dev/null +++ b/alembic/versions/83603bb7ef9c_amount_of_transactions_per_period.py @@ -0,0 +1,38 @@ +"""Amount of transaction per period + +Revision ID: 83603bb7ef9c +Revises: 8b5d5fbc8211 +Create Date: 2022-12-09 23:12:15.644758+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "83603bb7ef9c" +down_revision = "8b5d5fbc8211" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "categories_schedules", + sa.Column("amount", sa.Integer(), nullable=True), + schema="transactions", + ) + op.drop_column("categories_schedules", "recurring", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "categories_schedules", + sa.Column("recurring", sa.BOOLEAN(), autoincrement=False, nullable=False), + schema="transactions", + ) + op.drop_column("categories_schedules", "amount", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index be0c5dc..3eab450 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -231,15 +231,11 @@ class CategorySchedule(Base): __tablename__ = "categories_schedules" name: Mapped[catfk] = mapped_column(primary_key=True) - recurring: Mapped[bool] period: Mapped[Optional[scheduleperiod]] period_multiplier: Mapped[Optional[int]] + amount: Mapped[Optional[int]] category: Mapped[Category] = relationship(back_populates="schedule") def __repr__(self) -> str: - return ( - f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier})" - if self.recurring - else f"{self.name} has no Schedule" - ) + return f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier}, amount={self.amount})" From d321481e29a3c98e148021d81ac0d735bf93fe07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 10 Dec 2022 00:22:33 +0000 Subject: [PATCH 20/66] Rule based categorizing Uses the rules defined for each category to classify each transaction. Fixes the categorize command, which was broken from previous refactors. Swaps str type on the categories_rules date to date. --- .../7adf89ec8d14_category_rule_date_format.py | 43 ++++++++++++++++++ pfbudget/cli/runnable.py | 8 ++-- pfbudget/core/categorizer.py | 44 +++++++++++++++++-- pfbudget/core/manager.py | 10 ++--- pfbudget/db/client.py | 4 ++ pfbudget/db/model.py | 15 +++---- 6 files changed, 102 insertions(+), 22 deletions(-) create mode 100644 alembic/versions/7adf89ec8d14_category_rule_date_format.py diff --git a/alembic/versions/7adf89ec8d14_category_rule_date_format.py b/alembic/versions/7adf89ec8d14_category_rule_date_format.py new file mode 100644 index 0000000..176734c --- /dev/null +++ b/alembic/versions/7adf89ec8d14_category_rule_date_format.py @@ -0,0 +1,43 @@ +"""Category rule date format + +Revision ID: 7adf89ec8d14 +Revises: 83603bb7ef9c +Create Date: 2022-12-10 00:08:47.535765+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "7adf89ec8d14" +down_revision = "83603bb7ef9c" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "date", + existing_type=sa.VARCHAR(), + type_=sa.Date(), + existing_nullable=True, + schema="transactions", + postgresql_using="date::date" + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "date", + existing_type=sa.Date(), + type_=sa.VARCHAR(), + existing_nullable=True, + schema="transactions", + ) + # ### end Alembic commands ### diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 49f6f77..9222e6d 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -1,5 +1,6 @@ from pathlib import Path import argparse +import datetime as dt import re from pfbudget.common.types import Operation @@ -109,13 +110,12 @@ def argparser() -> argparse.ArgumentParser: """ Categorizing """ - p_categorize = subparsers.add_parser( + categorize = subparsers.add_parser( "categorize", description="Categorizes the transactions in the selected database", parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - p_categorize.set_defaults(command=Operation.Categorize) + categorize.set_defaults(op=Operation.Categorize) """ Graph @@ -343,7 +343,7 @@ def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser rule = commands.add_parser("rule", parents=[universal]) rule.set_defaults(op=Operation.CategoryRule) rule.add_argument("category", nargs="+", type=str) - rule.add_argument("--date", nargs=1, type=str) + rule.add_argument("--date", nargs=1, type=dt.date.fromisoformat) rule.add_argument("--description", nargs=1, type=str) rule.add_argument("--bank", nargs=1, type=str) rule.add_argument("--min", nargs=1, type=float) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 7b4f8e2..793ba59 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -1,4 +1,10 @@ -from pfbudget.db.model import Transaction, TransactionCategory +from pfbudget.db.model import ( + Category, + CategorySelector, + Selector, + Transaction, + TransactionCategory, +) from datetime import timedelta @@ -9,7 +15,7 @@ class Categorizer: def __init__(self): self.options["null_days"] = 4 - def categorize(self, transactions: list[Transaction]): + def categorize(self, transactions: list[Transaction], categories: list[Category]): """Overarching categorization tool Receives a list of transactions (by ref) and updates their category @@ -19,6 +25,7 @@ class Categorizer: """ self._nullify(transactions) + self._rules(transactions, categories) def _nullify(self, transactions: list[Transaction]): count = 0 @@ -38,10 +45,39 @@ class Categorizer: and cancel.amount == -transaction.amount ) ): - transaction.category = TransactionCategory(name="null") - cancel.category = TransactionCategory(name="null") + transaction.category = TransactionCategory( + name="null", selector=CategorySelector(Selector.nullifier) + ) + cancel.category = TransactionCategory( + name="null", selector=CategorySelector(Selector.nullifier) + ) matching.extend([transaction, cancel]) count += 2 break print(f"Nullified {count} transactions") + + def _rules(self, transactions: list[Transaction], categories: list[Category]): + for category in [c for c in categories if c.rules]: + for rule in category.rules: + for transaction in [t for t in transactions if not t.category]: + if rule.date: + if rule.date < transaction.date: + continue + if rule.description: + if rule.description not in transaction.description: + continue + if rule.bank: + if rule.bank != transaction.bank: + continue + if rule.min_amount: + if rule.min_amount > transaction.amount: + continue + if rule.max_amount: + if rule.max_amount <= transaction.amount: + continue + + # passed all conditions, assign category + transaction.category = TransactionCategory( + category.name, CategorySelector(Selector.rules) + ) diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 2227ccf..d744af8 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -28,7 +28,10 @@ class Manager: # TODO this is a monstrosity, remove when possible download(self, self.args) case Operation.Categorize: - self.categorize() + with self.db.session() as session: + uncategorized = session.uncategorized() + categories = session.categories() + Categorizer().categorize(uncategorized, categories) case Operation.Register: # self._db = DbClient(args["database"]) @@ -106,11 +109,6 @@ class Manager: with self.db.session() as session: session.add(transactions) - def categorize(self): - with self.db.session() as session: - uncategorized = session.uncategorized() - Categorizer().categorize(uncategorized) - # def get_bank_by(self, key: str, value: str) -> Bank: # client = DatabaseClient(self.__db) # bank = client.get_bank(key, value) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 28c2b39..5bdd558 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -129,5 +129,9 @@ class DbClient: stmt = select(Transaction).where(~Transaction.category.has()) return self.__session.scalars(stmt).all() + def categories(self) -> list[Category]: + stmt = select(Category) + return self.__session.scalars(stmt).all() + def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 3eab450..b70ec0e 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -81,9 +81,7 @@ class Transaction(Base): bank: Mapped[bankfk] amount: Mapped[money] - category: Mapped[Optional[TransactionCategory]] = relationship( - back_populates="original", lazy="joined", default=None - ) + category: Mapped[Optional[TransactionCategory]] = relationship() note: Mapped[Optional[Note]] = relationship(back_populates="original", default=None) tags: Mapped[Optional[set[Tag]]] = relationship( back_populates="original", @@ -138,8 +136,7 @@ class TransactionCategory(Base): id: Mapped[idfk] = mapped_column(primary_key=True, init=False) name: Mapped[str] = mapped_column(ForeignKey(Category.name)) - original: Mapped[Transaction] = relationship(back_populates="category") - selector: Mapped[CategorySelector] = relationship(back_populates="category") + selector: Mapped[CategorySelector] = relationship() def __repr__(self) -> str: return f"Category({self.name})" @@ -182,12 +179,15 @@ class CategoryRule(Base): id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) name: Mapped[catfk] = mapped_column() - date: Mapped[Optional[str]] = mapped_column() + date: Mapped[Optional[dt.date]] = mapped_column() description: Mapped[Optional[str]] = mapped_column() bank: Mapped[Optional[str]] = mapped_column() min_amount: Mapped[Optional[float]] = mapped_column() max_amount: Mapped[Optional[float]] = mapped_column() + def __hash__(self): + return hash(self.id) + class Selector(enum.Enum): unknown = enum.auto() @@ -211,11 +211,10 @@ class CategorySelector(Base): BigInteger, ForeignKey(TransactionCategory.id, ondelete="CASCADE"), primary_key=True, + init=False, ) selector: Mapped[categoryselector] - category: Mapped[TransactionCategory] = relationship(back_populates="selector") - class Period(enum.Enum): daily = "daily" From 72a8995fe6664a6584afc5bd9fe02c454584a8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 10 Dec 2022 17:51:23 +0000 Subject: [PATCH 21/66] Adds regex rule and remove rule option Categorization rules can now search using a regex pattern. --- alembic/versions/0ce89e987770_regex_rule.py | 32 +++++++++++++++++++++ pfbudget/__main__.py | 7 ++++- pfbudget/cli/runnable.py | 26 ++++++++++++----- pfbudget/common/types.py | 3 +- pfbudget/core/categorizer.py | 7 ++++- pfbudget/core/manager.py | 15 ++++++---- pfbudget/db/client.py | 12 ++++++-- pfbudget/db/model.py | 13 +++++---- 8 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 alembic/versions/0ce89e987770_regex_rule.py diff --git a/alembic/versions/0ce89e987770_regex_rule.py b/alembic/versions/0ce89e987770_regex_rule.py new file mode 100644 index 0000000..ddc47d5 --- /dev/null +++ b/alembic/versions/0ce89e987770_regex_rule.py @@ -0,0 +1,32 @@ +"""Regex rule + +Revision ID: 0ce89e987770 +Revises: 7adf89ec8d14 +Create Date: 2022-12-10 14:00:49.418494+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "0ce89e987770" +down_revision = "7adf89ec8d14" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "categories_rules", + sa.Column("regex", sa.String(), nullable=True), + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("categories_rules", "regex", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 023331e..4238e1e 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -43,7 +43,7 @@ if __name__ == "__main__": for cat in args["category"] ] - case pfbudget.Operation.CategoryRule: + case pfbudget.Operation.RuleAdd: assert args.keys() >= { "category", "date", @@ -58,6 +58,7 @@ if __name__ == "__main__": cat, args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, + args["regex"][0] if args["regex"] else None, args["bank"][0] if args["bank"] else None, args["min"][0] if args["min"] else None, args["max"][0] if args["max"] else None, @@ -65,6 +66,10 @@ if __name__ == "__main__": for cat in args["category"] ] + case pfbudget.Operation.RuleRemove: + assert args.keys() >= {"id"}, "argparser ill defined" + params = args["id"] + case pfbudget.Operation.GroupAdd: assert "group" in args, "argparser ill defined" params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 9222e6d..cdf2a94 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -341,13 +341,7 @@ def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser schedule.add_argument("--frequency", nargs=1, default=[1], type=int) rule = commands.add_parser("rule", parents=[universal]) - rule.set_defaults(op=Operation.CategoryRule) - rule.add_argument("category", nargs="+", type=str) - rule.add_argument("--date", nargs=1, type=dt.date.fromisoformat) - rule.add_argument("--description", nargs=1, type=str) - rule.add_argument("--bank", nargs=1, type=str) - rule.add_argument("--min", nargs=1, type=float) - rule.add_argument("--max", nargs=1, type=float) + category_rule(rule, universal) group = commands.add_parser("group", parents=[universal]) category_group(group, universal) @@ -365,6 +359,24 @@ def category_group(parser: argparse.ArgumentParser, universal: argparse.Argument remove.add_argument("group", nargs="+", type=str) +def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add", parents=[universal]) + add.set_defaults(op=Operation.RuleAdd) + add.add_argument("category", nargs="+", type=str) + add.add_argument("--date", nargs=1, type=dt.date.fromisoformat) + add.add_argument("--description", nargs=1, type=str) + add.add_argument("--regex", nargs=1, type=str) + add.add_argument("--bank", nargs=1, type=str) + add.add_argument("--min", nargs=1, type=float) + add.add_argument("--max", nargs=1, type=float) + + remove = commands.add_parser("remove", parents=[universal]) + remove.set_defaults(op=Operation.RuleRemove) + remove.add_argument("id", nargs="+", type=int) + + def run(): args = vars(argparser().parse_args()) assert "op" in args, "No operation selected" diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index dd8f431..b090bbb 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -17,7 +17,8 @@ class Operation(Enum): CategoryUpdate = auto() CategoryRemove = auto() CategorySchedule = auto() - CategoryRule = auto() + RuleAdd = auto() + RuleRemove = auto() GroupAdd = auto() GroupRemove = auto() diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 793ba59..15ac74d 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -7,6 +7,7 @@ from pfbudget.db.model import ( ) from datetime import timedelta +import re class Categorizer: @@ -64,9 +65,13 @@ class Categorizer: if rule.date: if rule.date < transaction.date: continue - if rule.description: + if rule.description and transaction.description: if rule.description not in transaction.description: continue + if rule.regex and transaction.description: + p = re.compile(rule.regex, re.IGNORECASE) + if not p.search(transaction.description): + continue if rule.bank: if rule.bank != transaction.bank: continue diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index d744af8..b4e6344 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -65,20 +65,23 @@ class Manager: with self.db.session() as session: session.updateschedules(params) - case Operation.CategoryRule: + case Operation.RuleAdd: with self.db.session() as session: session.addrules(params) + case Operation.RuleRemove: + assert all(isinstance(param, int) for param in params) + with self.db.session() as session: + session.removerules(params) + case Operation.GroupAdd: with self.db.session() as session: - for group in self.args["group"]: - session.addcategorygroup(CategoryGroup(name=group)) + session.addgroups(CategoryGroup(params)) case Operation.GroupRemove: + assert all(isinstance(param, CategoryGroup) for param in params) with self.db.session() as session: - session.removecategorygroup( - [CategoryGroup(name=group) for group in self.args["group"]] - ) + session.removegroups(params) # def init(self): # client = DatabaseClient(self.__db) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 5bdd558..67dbd19 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -116,10 +116,16 @@ class DbClient: def addrules(self, rules: list[CategoryRule]): self.__session.add_all(rules) - def addcategorygroup(self, group: CategoryGroup): - self.__session.add(group) + def removerules(self, ids: list[int]): + stmt = delete(CategoryRule).where( + CategoryRule.id.in_(ids) + ) + self.__session.execute(stmt) - def removecategorygroup(self, groups: list[CategoryGroup]): + def addgroups(self, groups: list[CategoryGroup]): + self.__session.add_all(groups) + + def removegroups(self, groups: list[CategoryGroup]): stmt = delete(CategoryGroup).where( CategoryGroup.name.in_([grp.name for grp in groups]) ) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index b70ec0e..e27521b 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -178,12 +178,13 @@ class CategoryRule(Base): __tablename__ = "categories_rules" id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) - name: Mapped[catfk] = mapped_column() - date: Mapped[Optional[dt.date]] = mapped_column() - description: Mapped[Optional[str]] = mapped_column() - bank: Mapped[Optional[str]] = mapped_column() - min_amount: Mapped[Optional[float]] = mapped_column() - max_amount: Mapped[Optional[float]] = mapped_column() + name: Mapped[catfk] + date: Mapped[Optional[dt.date]] + description: Mapped[Optional[str]] + regex: Mapped[Optional[str]] + bank: Mapped[Optional[str]] + min_amount: Mapped[Optional[float]] + max_amount: Mapped[Optional[float]] def __hash__(self): return hash(self.id) From 17d8d5d81347b41220b771ae71b411ccc621c85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 10 Dec 2022 18:58:03 +0000 Subject: [PATCH 22/66] [Fix] Changes rule's money type to Decimal Also fixes comparison w/ max amount. --- .../e36e6321568e_rules_min_max_money.py | 58 +++++++++++++++++++ pfbudget/cli/runnable.py | 5 +- pfbudget/core/categorizer.py | 2 +- pfbudget/db/model.py | 6 +- 4 files changed, 65 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/e36e6321568e_rules_min_max_money.py diff --git a/alembic/versions/e36e6321568e_rules_min_max_money.py b/alembic/versions/e36e6321568e_rules_min_max_money.py new file mode 100644 index 0000000..e69c89e --- /dev/null +++ b/alembic/versions/e36e6321568e_rules_min_max_money.py @@ -0,0 +1,58 @@ +"""Rules min/max money + +Revision ID: e36e6321568e +Revises: 0ce89e987770 +Create Date: 2022-12-10 18:55:07.149010+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "e36e6321568e" +down_revision = "0ce89e987770" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "min_amount", + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Numeric(precision=16, scale=2), + existing_nullable=True, + schema="transactions", + ) + op.alter_column( + "categories_rules", + "max_amount", + existing_type=sa.DOUBLE_PRECISION(precision=53), + type_=sa.Numeric(precision=16, scale=2), + existing_nullable=True, + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "max_amount", + existing_type=sa.Numeric(precision=16, scale=2), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_nullable=True, + schema="transactions", + ) + op.alter_column( + "categories_rules", + "min_amount", + existing_type=sa.Numeric(precision=16, scale=2), + type_=sa.DOUBLE_PRECISION(precision=53), + existing_nullable=True, + schema="transactions", + ) + # ### end Alembic commands ### diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index cdf2a94..ebe2504 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -1,6 +1,7 @@ from pathlib import Path import argparse import datetime as dt +import decimal import re from pfbudget.common.types import Operation @@ -369,8 +370,8 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP add.add_argument("--description", nargs=1, type=str) add.add_argument("--regex", nargs=1, type=str) add.add_argument("--bank", nargs=1, type=str) - add.add_argument("--min", nargs=1, type=float) - add.add_argument("--max", nargs=1, type=float) + add.add_argument("--min", nargs=1, type=decimal.Decimal) + add.add_argument("--max", nargs=1, type=decimal.Decimal) remove = commands.add_parser("remove", parents=[universal]) remove.set_defaults(op=Operation.RuleRemove) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 15ac74d..674c1aa 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -79,7 +79,7 @@ class Categorizer: if rule.min_amount > transaction.amount: continue if rule.max_amount: - if rule.max_amount <= transaction.amount: + if rule.max_amount < transaction.amount: continue # passed all conditions, assign category diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index e27521b..0a76846 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -69,7 +69,7 @@ class Bank(Base): bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] idpk = Annotated[int, mapped_column(BigInteger, primary_key=True)] -money = Annotated[Decimal, mapped_column(Numeric(16, 2), nullable=False)] +money = Annotated[Decimal, mapped_column(Numeric(16, 2))] class Transaction(Base): @@ -183,8 +183,8 @@ class CategoryRule(Base): description: Mapped[Optional[str]] regex: Mapped[Optional[str]] bank: Mapped[Optional[str]] - min_amount: Mapped[Optional[float]] - max_amount: Mapped[Optional[float]] + min_amount: Mapped[Optional[money]] + max_amount: Mapped[Optional[money]] def __hash__(self): return hash(self.id) From 7986fe8e5b480894edf0c1a5166d70ff9eb3edd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 17 Dec 2022 20:59:40 +0000 Subject: [PATCH 23/66] [Fix] argparse group option not an array --- pfbudget/__main__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 4238e1e..3785629 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -16,14 +16,13 @@ if __name__ == "__main__": case pfbudget.Operation.CategoryAdd: assert args.keys() >= {"category", "group"}, "argparser ill defined" params = [ - pfbudget.types.Category(cat, args["group"][0]) - for cat in args["category"] + pfbudget.types.Category(cat, args["group"]) for cat in args["category"] ] case pfbudget.Operation.CategoryUpdate: assert args.keys() >= {"category", "group"}, "argparser ill defined" params = [pfbudget.types.Category(cat) for cat in args["category"]] - params.append(args["group"][0]) + params.append(args["group"]) case pfbudget.Operation.CategoryRemove: assert "category" in args, "argparser ill defined" From a2b2f2c1d1136d009738995ee36989b1735f4de3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 17 Dec 2022 21:02:05 +0000 Subject: [PATCH 24/66] Adds modify rule operation --- pfbudget/__main__.py | 21 +++++++++++++++++++++ pfbudget/cli/runnable.py | 12 ++++++++++++ pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 5 +++++ pfbudget/db/client.py | 7 ++++--- pfbudget/db/model.py | 2 +- 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 3785629..1348ca9 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -69,6 +69,27 @@ if __name__ == "__main__": assert args.keys() >= {"id"}, "argparser ill defined" params = args["id"] + case pfbudget.Operation.RuleModify: + assert args.keys() >= { + "id", + "category", + "date", + "description", + "bank", + "min", + "max", + "remove", + }, "argparser ill defined" + + nargs_1 = ["category", "date", "description", "regex", "bank", "min", "max"] + params = [] + for id in args["id"]: + param = {"id": id} + param |= {k: v[0] for k, v in args.items() if k in nargs_1 and args[k]} + param |= {k: None for k in args["remove"] if k in nargs_1} + + params.append(param) + case pfbudget.Operation.GroupAdd: assert "group" in args, "argparser ill defined" params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index ebe2504..3a44494 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -377,6 +377,18 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP remove.set_defaults(op=Operation.RuleRemove) remove.add_argument("id", nargs="+", type=int) + modify = commands.add_parser("modify", parents=[universal]) + modify.set_defaults(op=Operation.RuleModify) + modify.add_argument("id", nargs="+", type=int) + modify.add_argument("--category", nargs=1, type=str) + modify.add_argument("--date", nargs=1, type=dt.date.fromisoformat) + modify.add_argument("--description", nargs=1, type=str) + modify.add_argument("--regex", nargs=1, type=str) + modify.add_argument("--bank", nargs=1, type=str) + modify.add_argument("--min", nargs=1, type=decimal.Decimal) + modify.add_argument("--max", nargs=1, type=decimal.Decimal) + modify.add_argument("--remove", nargs="*", default=[], type=str) + def run(): args = vars(argparser().parse_args()) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index b090bbb..c8f39d4 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -19,6 +19,7 @@ class Operation(Enum): CategorySchedule = auto() RuleAdd = auto() RuleRemove = auto() + RuleModify = auto() GroupAdd = auto() GroupRemove = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index b4e6344..2597ea1 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -74,6 +74,11 @@ class Manager: with self.db.session() as session: session.removerules(params) + case Operation.RuleModify: + assert all(isinstance(param, dict) for param in params) + with self.db.session() as session: + session.updaterules(params) + case Operation.GroupAdd: with self.db.session() as session: session.addgroups(CategoryGroup(params)) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 67dbd19..175fa67 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -117,11 +117,12 @@ class DbClient: self.__session.add_all(rules) def removerules(self, ids: list[int]): - stmt = delete(CategoryRule).where( - CategoryRule.id.in_(ids) - ) + stmt = delete(CategoryRule).where(CategoryRule.id.in_(ids)) self.__session.execute(stmt) + def updaterules(self, rules: list[dict]): + self.__session.execute(update(CategoryRule), rules) + def addgroups(self, groups: list[CategoryGroup]): self.__session.add_all(groups) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 0a76846..4384cd6 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -114,7 +114,7 @@ class Category(Base): ) rules: Mapped[Optional[set[CategoryRule]]] = relationship( - cascade="all, delete-orphan", passive_deletes=True, default=None + cascade="all, delete-orphan", passive_deletes=True, default_factory=set ) schedule: Mapped[CategorySchedule] = relationship( back_populates="category", default=None From 6c33a94a5f3aede21b4a5e62d102396dd301830a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 18 Dec 2022 00:28:07 +0000 Subject: [PATCH 25/66] Change min_amount/max_amount columns for min/max Simplifies the action creation. --- alembic/versions/753c0bfb2062_rule_min_max.py | 49 +++++++++++++++++++ pfbudget/core/categorizer.py | 8 +-- pfbudget/db/model.py | 4 +- 3 files changed, 55 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/753c0bfb2062_rule_min_max.py diff --git a/alembic/versions/753c0bfb2062_rule_min_max.py b/alembic/versions/753c0bfb2062_rule_min_max.py new file mode 100644 index 0000000..5e32010 --- /dev/null +++ b/alembic/versions/753c0bfb2062_rule_min_max.py @@ -0,0 +1,49 @@ +"""Rule min/max + +Revision ID: 753c0bfb2062 +Revises: e36e6321568e +Create Date: 2022-12-18 00:24:03.861461+00:00 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "753c0bfb2062" +down_revision = "e36e6321568e" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "min_amount", + new_column_name="min", + schema="transactions", + ) + op.alter_column( + "categories_rules", + "max_amount", + new_column_name="max", + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "categories_rules", + "min", + new_column_name="min_amount", + schema="transactions", + ) + op.alter_column( + "categories_rules", + "max", + new_column_name="max_amount", + schema="transactions", + ) + # ### end Alembic commands ### diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 674c1aa..4348986 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -75,11 +75,11 @@ class Categorizer: if rule.bank: if rule.bank != transaction.bank: continue - if rule.min_amount: - if rule.min_amount > transaction.amount: + if rule.min: + if rule.min > transaction.amount: continue - if rule.max_amount: - if rule.max_amount < transaction.amount: + if rule.max: + if rule.max < transaction.amount: continue # passed all conditions, assign category diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 4384cd6..b506783 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -183,8 +183,8 @@ class CategoryRule(Base): description: Mapped[Optional[str]] regex: Mapped[Optional[str]] bank: Mapped[Optional[str]] - min_amount: Mapped[Optional[money]] - max_amount: Mapped[Optional[money]] + min: Mapped[Optional[money]] + max: Mapped[Optional[money]] def __hash__(self): return hash(self.id) From 058d0cc05d5d626d2ff30f30390b2fd31e232074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 18 Dec 2022 23:38:45 +0000 Subject: [PATCH 26/66] Adds Tags rules Creates an available tags table to cross reference to. Clarifies some logic in the DbClient, since all adds/removes are the same. --- alembic/versions/9028b0f3b985_tag_rules.py | 68 ++++++++++++++++++++++ pfbudget/__main__.py | 50 +++++++++++++++- pfbudget/cli/runnable.py | 62 +++++++++++++++----- pfbudget/common/types.py | 5 ++ pfbudget/core/manager.py | 36 ++++++++---- pfbudget/db/client.py | 36 +++++------- pfbudget/db/model.py | 35 +++++++++-- 7 files changed, 240 insertions(+), 52 deletions(-) create mode 100644 alembic/versions/9028b0f3b985_tag_rules.py diff --git a/alembic/versions/9028b0f3b985_tag_rules.py b/alembic/versions/9028b0f3b985_tag_rules.py new file mode 100644 index 0000000..ae8cf2c --- /dev/null +++ b/alembic/versions/9028b0f3b985_tag_rules.py @@ -0,0 +1,68 @@ +"""Tag rules + +Revision ID: 9028b0f3b985 +Revises: 753c0bfb2062 +Create Date: 2022-12-18 22:53:13.334046+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "9028b0f3b985" +down_revision = "753c0bfb2062" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "tags_available", + sa.Column("name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_tags_available")), + schema="transactions", + ) + op.create_table( + "tag_rules", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("tag", sa.String(), nullable=False), + sa.Column("date", sa.Date(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("regex", sa.String(), nullable=True), + sa.Column("bank", sa.String(), nullable=True), + sa.Column("min", sa.Numeric(precision=16, scale=2), nullable=True), + sa.Column("max", sa.Numeric(precision=16, scale=2), nullable=True), + sa.ForeignKeyConstraint( + ["tag"], + ["transactions.tags_available.name"], + name=op.f("fk_tag_rules_tag_tags_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_tag_rules")), + schema="transactions", + ) + op.create_foreign_key( + op.f("fk_tags_tag_tags_available"), + "tags", + "tags_available", + ["tag"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_tags_tag_tags_available"), + "tags", + schema="transactions", + type_="foreignkey", + ) + op.drop_table("tag_rules", schema="transactions") + op.drop_table("tags_available", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 1348ca9..e5c0303 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -65,7 +65,7 @@ if __name__ == "__main__": for cat in args["category"] ] - case pfbudget.Operation.RuleRemove: + case pfbudget.Operation.RuleRemove | pfbudget.Operation.TagRuleRemove: assert args.keys() >= {"id"}, "argparser ill defined" params = args["id"] @@ -90,6 +90,54 @@ if __name__ == "__main__": params.append(param) + case pfbudget.Operation.TagAdd: + assert args.keys() >= {"tag"}, "argparser ill defined" + params = [pfbudget.types.Tag(tag) for tag in args["tag"]] + + case pfbudget.Operation.TagRuleAdd: + assert args.keys() >= { + "tag", + "date", + "description", + "bank", + "min", + "max", + }, "argparser ill defined" + + params = [ + pfbudget.types.TagRule( + tag, + args["date"][0] if args["date"] else None, + args["description"][0] if args["description"] else None, + args["regex"][0] if args["regex"] else None, + args["bank"][0] if args["bank"] else None, + args["min"][0] if args["min"] else None, + args["max"][0] if args["max"] else None, + ) + for tag in args["tag"] + ] + + case pfbudget.Operation.TagRuleModify: + assert args.keys() >= { + "id", + "tag", + "date", + "description", + "bank", + "min", + "max", + "remove", + }, "argparser ill defined" + + nargs_1 = ["tag", "date", "description", "regex", "bank", "min", "max"] + params = [] + for id in args["id"]: + param = {"id": id} + param |= {k: v[0] for k, v in args.items() if k in nargs_1 and args[k]} + param |= {k: None for k in args["remove"] if k in nargs_1} + + params.append(param) + case pfbudget.Operation.GroupAdd: assert "group" in args, "argparser ill defined" params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 3a44494..cad96cc 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -255,6 +255,9 @@ def argparser() -> argparse.ArgumentParser: category_parser = subparsers.add_parser("category", parents=[universal]) category(category_parser, universal) + # Tag + tags(subparsers.add_parser("tag", parents=[universal]), universal) + return parser @@ -366,12 +369,7 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP add = commands.add_parser("add", parents=[universal]) add.set_defaults(op=Operation.RuleAdd) add.add_argument("category", nargs="+", type=str) - add.add_argument("--date", nargs=1, type=dt.date.fromisoformat) - add.add_argument("--description", nargs=1, type=str) - add.add_argument("--regex", nargs=1, type=str) - add.add_argument("--bank", nargs=1, type=str) - add.add_argument("--min", nargs=1, type=decimal.Decimal) - add.add_argument("--max", nargs=1, type=decimal.Decimal) + rules(add) remove = commands.add_parser("remove", parents=[universal]) remove.set_defaults(op=Operation.RuleRemove) @@ -381,13 +379,51 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP modify.set_defaults(op=Operation.RuleModify) modify.add_argument("id", nargs="+", type=int) modify.add_argument("--category", nargs=1, type=str) - modify.add_argument("--date", nargs=1, type=dt.date.fromisoformat) - modify.add_argument("--description", nargs=1, type=str) - modify.add_argument("--regex", nargs=1, type=str) - modify.add_argument("--bank", nargs=1, type=str) - modify.add_argument("--min", nargs=1, type=decimal.Decimal) - modify.add_argument("--max", nargs=1, type=decimal.Decimal) - modify.add_argument("--remove", nargs="*", default=[], type=str) + rules(modify) + + +def tags(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): + + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add", parents=[universal]) + add.set_defaults(op=Operation.TagAdd) + add.add_argument("tag", nargs="+", type=str) + + remove = commands.add_parser("remove", parents=[universal]) + remove.set_defaults(op=Operation.TagRemove) + remove.add_argument("tag", nargs="+", type=str) + + rule = commands.add_parser("rule", parents=[universal]) + tag_rule(rule, universal) + + +def tag_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add", parents=[universal]) + add.set_defaults(op=Operation.TagRuleAdd) + add.add_argument("tag", nargs="+", type=str) + rules(add) + + remove = commands.add_parser("remove", parents=[universal]) + remove.set_defaults(op=Operation.TagRuleRemove) + remove.add_argument("id", nargs="+", type=int) + + modify = commands.add_parser("modify", parents=[universal]) + modify.set_defaults(op=Operation.TagRuleModify) + modify.add_argument("id", nargs="+", type=int) + modify.add_argument("--tag", nargs=1, type=str) + rules(modify) + + +def rules(parser: argparse.ArgumentParser): + parser.add_argument("--date", nargs=1, type=dt.date.fromisoformat) + parser.add_argument("--description", nargs=1, type=str) + parser.add_argument("--regex", nargs=1, type=str) + parser.add_argument("--bank", nargs=1, type=str) + parser.add_argument("--min", nargs=1, type=decimal.Decimal) + parser.add_argument("--max", nargs=1, type=decimal.Decimal) def run(): diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index c8f39d4..8a6c0e4 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -22,6 +22,11 @@ class Operation(Enum): RuleModify = auto() GroupAdd = auto() GroupRemove = auto() + TagAdd = auto() + TagRemove = auto() + TagRuleAdd = auto() + TagRuleRemove = auto() + TagRuleModify = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 2597ea1..bf6d4bf 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -2,7 +2,14 @@ from pfbudget.input.input import Input from pfbudget.input.nordigen import NordigenClient from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient -from pfbudget.db.model import Category, CategoryGroup, CategoryRule, CategorySchedule +from pfbudget.db.model import ( + Category, + CategoryGroup, + CategoryRule, + CategorySchedule, + Tag, + TagRule, +) from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert @@ -49,9 +56,9 @@ class Manager: self.args["name"], self.args["country"] ) - case Operation.CategoryAdd: + case Operation.CategoryAdd | Operation.TagAdd: with self.db.session() as session: - session.addcategories(params) + session.add(params) case Operation.CategoryUpdate: with self.db.session() as session: @@ -59,34 +66,43 @@ class Manager: case Operation.CategoryRemove: with self.db.session() as session: - session.removecategories(params) + session.remove_by_name(Category, params) case Operation.CategorySchedule: with self.db.session() as session: session.updateschedules(params) - case Operation.RuleAdd: + case Operation.RuleAdd | Operation.TagRuleAdd: with self.db.session() as session: - session.addrules(params) + session.add(params) case Operation.RuleRemove: assert all(isinstance(param, int) for param in params) with self.db.session() as session: - session.removerules(params) + session.remove_by_id(CategoryRule, params) - case Operation.RuleModify: + case Operation.TagRemove: + with self.db.session() as session: + session.remove_by_name(Tag, params) + + case Operation.TagRuleRemove: + assert all(isinstance(param, int) for param in params) + with self.db.session() as session: + session.remove_by_id(TagRule, params) + + case Operation.RuleModify | Operation.TagRuleModify: assert all(isinstance(param, dict) for param in params) with self.db.session() as session: session.updaterules(params) case Operation.GroupAdd: with self.db.session() as session: - session.addgroups(CategoryGroup(params)) + session.add(CategoryGroup(params)) case Operation.GroupRemove: assert all(isinstance(param, CategoryGroup) for param in params) with self.db.session() as session: - session.removegroups(params) + session.remove_by_name(CategoryGroup, params) # def init(self): # client = DatabaseClient(self.__db) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 175fa67..68803e3 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -10,6 +10,8 @@ from pfbudget.db.model import ( CategoryGroup, CategoryRule, CategorySchedule, + Tag, + TagRule, Transaction, ) @@ -81,16 +83,16 @@ class DbClient: def commit(self): self.__session.commit() - def add(self, transactions: list[Transaction]): - self.__session.add_all(transactions) + def add( + self, + rows: list[ + Category | CategoryGroup | CategoryRule | Tag | TagRule | Transaction + ], + ): + self.__session.add_all(rows) - def addcategories(self, category: list[Category]): - self.__session.add_all(category) - - def removecategories(self, categories: list[Category]): - stmt = delete(Category).where( - Category.name.in_([cat.name for cat in categories]) - ) + def remove_by_name(self, type: Category | Tag | Transaction, rows: list): + stmt = delete(type).where(type.name.in_([row.name for row in rows])) self.__session.execute(stmt) def updategroup(self, categories: list[Category], group: CategoryGroup): @@ -113,25 +115,13 @@ class DbClient: ) self.__session.execute(stmt) - def addrules(self, rules: list[CategoryRule]): - self.__session.add_all(rules) - - def removerules(self, ids: list[int]): - stmt = delete(CategoryRule).where(CategoryRule.id.in_(ids)) + def remove_by_id(self, type: CategoryRule | TagRule, ids: list[int]): + stmt = delete(type).where(type.id.in_(ids)) self.__session.execute(stmt) def updaterules(self, rules: list[dict]): self.__session.execute(update(CategoryRule), rules) - def addgroups(self, groups: list[CategoryGroup]): - self.__session.add_all(groups) - - def removegroups(self, groups: list[CategoryGroup]): - stmt = delete(CategoryGroup).where( - CategoryGroup.name.in_([grp.name for grp in groups]) - ) - self.__session.execute(stmt) - def uncategorized(self) -> list[Transaction]: stmt = select(Transaction).where(~Transaction.category.has()) return self.__session.scalars(stmt).all() diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index b506783..0c31380 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -68,14 +68,16 @@ class Bank(Base): bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] -idpk = Annotated[int, mapped_column(BigInteger, primary_key=True)] +idpk = Annotated[ + int, mapped_column(BigInteger, primary_key=True, autoincrement=True, init=False) +] money = Annotated[Decimal, mapped_column(Numeric(16, 2))] class Transaction(Base): __tablename__ = "originals" - id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) + id: Mapped[idpk] date: Mapped[dt.date] description: Mapped[Optional[str]] bank: Mapped[bankfk] @@ -83,7 +85,7 @@ class Transaction(Base): category: Mapped[Optional[TransactionCategory]] = relationship() note: Mapped[Optional[Note]] = relationship(back_populates="original", default=None) - tags: Mapped[Optional[set[Tag]]] = relationship( + tags: Mapped[Optional[set[TransactionTag]]] = relationship( back_populates="original", cascade="all, delete-orphan", passive_deletes=True, @@ -166,10 +168,20 @@ class Nordigen(Base): class Tag(Base): + __tablename__ = "tags_available" + + name: Mapped[str] = mapped_column(primary_key=True) + + rules: Mapped[Optional[set[TagRule]]] = relationship( + cascade="all, delete-orphan", passive_deletes=True, default_factory=set + ) + + +class TransactionTag(Base): __tablename__ = "tags" id: Mapped[idfk] = mapped_column(primary_key=True) - tag: Mapped[str] = mapped_column(primary_key=True) + tag: Mapped[str] = mapped_column(ForeignKey(Tag.name), primary_key=True) original: Mapped[Transaction] = relationship(back_populates="tags") @@ -177,7 +189,7 @@ class Tag(Base): class CategoryRule(Base): __tablename__ = "categories_rules" - id: Mapped[idpk] = mapped_column(autoincrement=True, init=False) + id: Mapped[idpk] name: Mapped[catfk] date: Mapped[Optional[dt.date]] description: Mapped[Optional[str]] @@ -239,3 +251,16 @@ class CategorySchedule(Base): def __repr__(self) -> str: return f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier}, amount={self.amount})" + + +class TagRule(Base): + __tablename__ = "tag_rules" + + id: Mapped[idpk] + tag: Mapped[str] = mapped_column(ForeignKey(Tag.name, ondelete="CASCADE")) + date: Mapped[Optional[dt.date]] + description: Mapped[Optional[str]] + regex: Mapped[Optional[str]] + bank: Mapped[Optional[str]] + min: Mapped[Optional[money]] + max: Mapped[Optional[money]] From e27f2f08cf55d72d8953e9816256c16f501c7d6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 19 Dec 2022 21:59:41 +0000 Subject: [PATCH 27/66] Adds rule based tagging of transactions Tags will work as additional categories to filter/organize by. It makes sense they can also be rule based. Since rules are common to both categories and tags, reorganize the classes in the model. It doesn't affect the DB. --- pfbudget/cli/runnable.py | 1 - pfbudget/core/categorizer.py | 81 +++++++++++++++++++++++---------- pfbudget/core/manager.py | 10 ++--- pfbudget/db/client.py | 4 ++ pfbudget/db/model.py | 86 ++++++++++++++++++++++-------------- 5 files changed, 117 insertions(+), 65 deletions(-) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index cad96cc..18334d4 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -383,7 +383,6 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP def tags(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): - commands = parser.add_subparsers(required=True) add = commands.add_parser("add", parents=[universal]) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 4348986..fb7405a 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -2,12 +2,13 @@ from pfbudget.db.model import ( Category, CategorySelector, Selector, + Tag, Transaction, TransactionCategory, + TransactionTag, ) from datetime import timedelta -import re class Categorizer: @@ -16,7 +17,12 @@ class Categorizer: def __init__(self): self.options["null_days"] = 4 - def categorize(self, transactions: list[Transaction], categories: list[Category]): + def categorize( + self, + transactions: list[Transaction], + categories: list[Category], + tags: list[Tag], + ): """Overarching categorization tool Receives a list of transactions (by ref) and updates their category @@ -26,7 +32,8 @@ class Categorizer: """ self._nullify(transactions) - self._rules(transactions, categories) + self._rule_based_categories(transactions, categories) + self._rule_based_tags(transactions, tags) def _nullify(self, transactions: list[Transaction]): count = 0 @@ -58,31 +65,57 @@ class Categorizer: print(f"Nullified {count} transactions") - def _rules(self, transactions: list[Transaction], categories: list[Category]): + def _rule_based_categories( + self, transactions: list[Transaction], categories: list[Category] + ): + d = {} for category in [c for c in categories if c.rules]: for rule in category.rules: - for transaction in [t for t in transactions if not t.category]: - if rule.date: - if rule.date < transaction.date: - continue - if rule.description and transaction.description: - if rule.description not in transaction.description: - continue - if rule.regex and transaction.description: - p = re.compile(rule.regex, re.IGNORECASE) - if not p.search(transaction.description): - continue - if rule.bank: - if rule.bank != transaction.bank: - continue - if rule.min: - if rule.min > transaction.amount: - continue - if rule.max: - if rule.max < transaction.amount: - continue + # for transaction in [t for t in transactions if not t.category]: + for transaction in [ + t + for t in transactions + if not t.category or t.category.name != "null" + ]: + if not rule.matches(transaction): + continue # passed all conditions, assign category transaction.category = TransactionCategory( category.name, CategorySelector(Selector.rules) ) + + if rule in d: + d[rule] += 1 + else: + d[rule] = 1 + + for k, v in d.items(): + print(f"{v}: {k}") + + def _rule_based_tags(self, transactions: list[Transaction], tags: list[Tag]): + d = {} + for tag in [t for t in tags if t.rules]: + for rule in tag.rules: + # for transaction in [t for t in transactions if not t.category]: + for transaction in [ + t + for t in transactions + if tag.name not in [tag.tag for tag in t.tags] + ]: + if not rule.matches(transaction): + continue + + if not transaction.tags: + transaction.tags = {TransactionTag(tag.name)} + else: + transaction.tags.add(TransactionTag(tag.name)) + + if rule in d: + d[rule] += 1 + else: + d[rule] = 1 + + for k, v in d.items(): + print(f"{v}: {k}") + diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index bf6d4bf..d8bc431 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -34,11 +34,13 @@ class Manager: case Operation.Download: # TODO this is a monstrosity, remove when possible download(self, self.args) + case Operation.Categorize: with self.db.session() as session: uncategorized = session.uncategorized() categories = session.categories() - Categorizer().categorize(uncategorized, categories) + tags = session.tags() + Categorizer().categorize(uncategorized, categories, tags) case Operation.Register: # self._db = DbClient(args["database"]) @@ -56,7 +58,7 @@ class Manager: self.args["name"], self.args["country"] ) - case Operation.CategoryAdd | Operation.TagAdd: + case Operation.CategoryAdd | Operation.RuleAdd | Operation.TagAdd | Operation.TagRuleAdd: with self.db.session() as session: session.add(params) @@ -72,10 +74,6 @@ class Manager: with self.db.session() as session: session.updateschedules(params) - case Operation.RuleAdd | Operation.TagRuleAdd: - with self.db.session() as session: - session.add(params) - case Operation.RuleRemove: assert all(isinstance(param, int) for param in params) with self.db.session() as session: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 68803e3..c29cf56 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -130,5 +130,9 @@ class DbClient: stmt = select(Category) return self.__session.scalars(stmt).all() + def tags(self) -> list[Tag]: + stmt = select(Tag) + return self.__session.scalars(stmt).all() + def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 0c31380..7ef354d 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -21,6 +21,7 @@ from decimal import Decimal from typing import Annotated, Optional import datetime as dt import enum +import re class Base(MappedAsDataclass, DeclarativeBase): @@ -68,29 +69,23 @@ class Bank(Base): bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] -idpk = Annotated[ - int, mapped_column(BigInteger, primary_key=True, autoincrement=True, init=False) -] +idpk = Annotated[int, mapped_column(BigInteger, primary_key=True, autoincrement=True)] money = Annotated[Decimal, mapped_column(Numeric(16, 2))] class Transaction(Base): __tablename__ = "originals" - id: Mapped[idpk] + id: Mapped[idpk] = mapped_column(init=False) date: Mapped[dt.date] description: Mapped[Optional[str]] bank: Mapped[bankfk] amount: Mapped[money] category: Mapped[Optional[TransactionCategory]] = relationship() - note: Mapped[Optional[Note]] = relationship(back_populates="original", default=None) - tags: Mapped[Optional[set[TransactionTag]]] = relationship( - back_populates="original", - cascade="all, delete-orphan", - passive_deletes=True, - default=None, ) + note: Mapped[Optional[Note]] = relationship(back_populates="original") + tags: Mapped[Optional[set[TransactionTag]]] = relationship() def __repr__(self) -> str: return f"Transaction(date={self.date}, description={self.description}, bank={self.bank}, amount={self.amount}, category={self.category})" @@ -136,9 +131,9 @@ class TransactionCategory(Base): __tablename__ = "categorized" id: Mapped[idfk] = mapped_column(primary_key=True, init=False) - name: Mapped[str] = mapped_column(ForeignKey(Category.name)) + name: Mapped[catfk] - selector: Mapped[CategorySelector] = relationship() + selector: Mapped[CategorySelector] = relationship(cascade="all, delete-orphan") def __repr__(self) -> str: return f"Category({self.name})" @@ -147,7 +142,7 @@ class TransactionCategory(Base): class Note(Base): __tablename__ = "notes" - id: Mapped[idfk] = mapped_column(primary_key=True) + id: Mapped[idfk] = mapped_column(primary_key=True, init=False) note: Mapped[str] original: Mapped[Transaction] = relationship(back_populates="note") @@ -180,24 +175,9 @@ class Tag(Base): class TransactionTag(Base): __tablename__ = "tags" - id: Mapped[idfk] = mapped_column(primary_key=True) + id: Mapped[idfk] = mapped_column(primary_key=True, init=False) tag: Mapped[str] = mapped_column(ForeignKey(Tag.name), primary_key=True) - original: Mapped[Transaction] = relationship(back_populates="tags") - - -class CategoryRule(Base): - __tablename__ = "categories_rules" - - id: Mapped[idpk] - name: Mapped[catfk] - date: Mapped[Optional[dt.date]] - description: Mapped[Optional[str]] - regex: Mapped[Optional[str]] - bank: Mapped[Optional[str]] - min: Mapped[Optional[money]] - max: Mapped[Optional[money]] - def __hash__(self): return hash(self.id) @@ -253,14 +233,52 @@ class CategorySchedule(Base): return f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier}, amount={self.amount})" -class TagRule(Base): - __tablename__ = "tag_rules" - - id: Mapped[idpk] - tag: Mapped[str] = mapped_column(ForeignKey(Tag.name, ondelete="CASCADE")) +class Rule: date: Mapped[Optional[dt.date]] description: Mapped[Optional[str]] regex: Mapped[Optional[str]] bank: Mapped[Optional[str]] min: Mapped[Optional[money]] max: Mapped[Optional[money]] + + def matches(self, transaction: Transaction) -> bool: + if ( + (self.date and self.date < transaction.date) + or ( + self.description + and transaction.description + and self.description not in transaction.description + ) + or ( + self.regex + and transaction.description + and not re.compile(self.regex, re.IGNORECASE).search( + transaction.description + ) + ) + or (self.bank and self.bank != transaction.bank) + or (self.min and self.min > transaction.amount) + or (self.max and self.max < transaction.amount) + ): + return False + return True + + +class CategoryRule(Base, Rule): + __tablename__ = "categories_rules" + + id: Mapped[idpk] = mapped_column(init=False) + name: Mapped[catfk] + + def __hash__(self): + return hash(self.id) + + +class TagRule(Base, Rule): + __tablename__ = "tag_rules" + + id: Mapped[idpk] = mapped_column(init=False) + tag: Mapped[str] = mapped_column(ForeignKey(Tag.name, ondelete="CASCADE")) + + def __hash__(self): + return hash(self.id) From e57859f601e55d0589b3f0bf30214dfda04a1a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 19 Dec 2022 22:02:11 +0000 Subject: [PATCH 28/66] Updates verbosity options Now it's used by adding -v to the command called. No verbosity=quiet. --- pfbudget/__main__.py | 9 ++++++--- pfbudget/cli/runnable.py | 7 +------ pfbudget/core/manager.py | 9 ++++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index e5c0303..4f3f2f9 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -6,10 +6,13 @@ if __name__ == "__main__": args = vars(argparser.parse_args()) assert "op" in args, "No pfbudget.Operation selected" - op: pfbudget.Operation = args["op"] + op: pfbudget.Operation = args.pop("op") assert "database" in args, "No database selected" - db = args["database"] + db = args.pop("database") + + assert "verbose" in args, "No verbose level specified" + verbosity = args.pop("verbose") params = None match (op): @@ -146,4 +149,4 @@ if __name__ == "__main__": assert "group" in args, "argparser ill defined" params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] - pfbudget.Manager(db, args).action(op, params) + pfbudget.Manager(db, verbosity, args).action(op, params) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 18334d4..8d62abc 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -40,12 +40,7 @@ def argparser() -> argparse.ArgumentParser: help="select current database", default=DEFAULT_DB, ) - universal.add_argument( - "-q", "--quiet", action="store_true", help="reduces the amount of verbose" - ) - universal.add_argument( - "-v", "--verbose", action="store_true", help="increases the amount of verbose" - ) + universal.add_argument("-v", "--verbose", action="count", default=0) period = argparse.ArgumentParser(add_help=False).add_mutually_exclusive_group() period.add_argument( diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index d8bc431..f143951 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -18,13 +18,16 @@ from pfbudget.cli.runnable import download, parse class Manager: - def __init__(self, db: str, args: dict): + def __init__(self, db: str, verbosity: int = 0, args: dict = {}): self._args = args - print(args) self._db = db + self._verbosity = verbosity def action(self, op: Operation, params: list): + if self._verbosity > 0: + print(f"op={op}, params={params}") + match (op): case Operation.Init: pass @@ -141,7 +144,7 @@ class Manager: @property def db(self) -> DbClient: - return DbClient(self._db, self.args["verbose"]) + return DbClient(self._db, self._verbosity > 0) @db.setter def db(self, url: str): From f20cf685adf32812a9bdcc9a68b1b7374ae77d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 19 Dec 2022 22:09:34 +0000 Subject: [PATCH 29/66] Update Foreign Key delete cascade for categories --- .../versions/a910e1b2214d_rule_inheritance.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 alembic/versions/a910e1b2214d_rule_inheritance.py diff --git a/alembic/versions/a910e1b2214d_rule_inheritance.py b/alembic/versions/a910e1b2214d_rule_inheritance.py new file mode 100644 index 0000000..fb2bf59 --- /dev/null +++ b/alembic/versions/a910e1b2214d_rule_inheritance.py @@ -0,0 +1,56 @@ +"""Rule inheritance + +Revision ID: a910e1b2214d +Revises: 9028b0f3b985 +Create Date: 2022-12-19 20:48:04.682812+00:00 + +""" +from alembic import op + + +# revision identifiers, used by Alembic. +revision = "a910e1b2214d" +down_revision = "9028b0f3b985" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + "fk_categorized_name_categories_available", + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_categorized_name_categories_available"), + "categorized", + "categories_available", + ["name"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_categorized_name_categories_available"), + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_categorized_name_categories_available", + "categorized", + "categories_available", + ["name"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + # ### end Alembic commands ### From 7fe5b6bd32d0ffdba3934540e7f60c07bc2e7833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 19 Dec 2022 22:48:49 +0000 Subject: [PATCH 30/66] Adds links between transactions Sometimes transctions are directly related to one another w/o being of an equal value, e.g. someone pays for the meal w/ the CC, and everyone pays him/her. Clear leftover __repr__ methods in the model classes, the dataclass decorator will create those automatically. --- alembic/versions/8cc9870b0d74_links.py | 46 ++++++++++++++++++++++++++ pfbudget/__main__.py | 6 ++++ pfbudget/cli/runnable.py | 19 +++++++++-- pfbudget/common/types.py | 2 ++ pfbudget/core/manager.py | 10 ++++++ pfbudget/db/client.py | 13 ++++---- pfbudget/db/model.py | 18 ++++------ 7 files changed, 93 insertions(+), 21 deletions(-) create mode 100644 alembic/versions/8cc9870b0d74_links.py diff --git a/alembic/versions/8cc9870b0d74_links.py b/alembic/versions/8cc9870b0d74_links.py new file mode 100644 index 0000000..7043f81 --- /dev/null +++ b/alembic/versions/8cc9870b0d74_links.py @@ -0,0 +1,46 @@ +"""Links + +Revision ID: 8cc9870b0d74 +Revises: a910e1b2214d +Create Date: 2022-12-19 22:10:25.136479+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "8cc9870b0d74" +down_revision = "a910e1b2214d" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "links", + sa.Column("original", sa.BigInteger(), nullable=False), + sa.Column("link", sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint( + ["link"], + ["transactions.originals.id"], + name=op.f("fk_links_link_originals"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["original"], + ["transactions.originals.id"], + name=op.f("fk_links_original_originals"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("original", "link", name=op.f("pk_links")), + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("links", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 4f3f2f9..c62c96f 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -149,4 +149,10 @@ if __name__ == "__main__": assert "group" in args, "argparser ill defined" params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] + case pfbudget.Operation.Forge | pfbudget.Operation.Dismantle: + assert args.keys() >= {"original", "links"}, "argparser ill defined" + params = [ + pfbudget.types.Link(args["original"][0], link) for link in args["links"] + ] + pfbudget.Manager(db, verbosity, args).action(op, params) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 8d62abc..aa780f2 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -5,9 +5,7 @@ import decimal import re from pfbudget.common.types import Operation -from pfbudget.core.categories import categorize_data from pfbudget.db.model import Period -from pfbudget.input.json import JsonParser from pfbudget.input.nordigen import NordigenInput from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph @@ -253,6 +251,9 @@ def argparser() -> argparse.ArgumentParser: # Tag tags(subparsers.add_parser("tag", parents=[universal]), universal) + # Link + link(subparsers.add_parser("link")) + return parser @@ -420,6 +421,20 @@ def rules(parser: argparse.ArgumentParser): parser.add_argument("--max", nargs=1, type=decimal.Decimal) +def link(parser: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + forge = commands.add_parser("forge") + forge.set_defaults(op=Operation.Forge) + forge.add_argument("original", nargs=1, type=int) + forge.add_argument("links", nargs="+", type=int) + + dismantle = commands.add_parser("dismantle") + dismantle.set_defaults(op=Operation.Dismantle) + dismantle.add_argument("original", nargs=1, type=int) + dismantle.add_argument("links", nargs="+", type=int) + + def run(): args = vars(argparser().parse_args()) assert "op" in args, "No operation selected" diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 8a6c0e4..0213c03 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -27,6 +27,8 @@ class Operation(Enum): TagRuleAdd = auto() TagRuleRemove = auto() TagRuleModify = auto() + Forge = auto() + Dismantle = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index f143951..b2c516e 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -105,6 +105,16 @@ class Manager: with self.db.session() as session: session.remove_by_name(CategoryGroup, params) + case Operation.Forge: + with self.db.session() as session: + session.add(params) + + case Operation.Dismantle: + with self.db.session() as session: + original = params[0].original + links = [link.link for link in params] + session.remove_links(original, links) + # def init(self): # client = DatabaseClient(self.__db) # client.init() diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index c29cf56..5af6181 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,4 +1,3 @@ -from copy import deepcopy from dataclasses import asdict from sqlalchemy import create_engine, delete, select, update from sqlalchemy.dialects.postgresql import insert @@ -10,6 +9,7 @@ from pfbudget.db.model import ( CategoryGroup, CategoryRule, CategorySchedule, + Link, Tag, TagRule, Transaction, @@ -83,12 +83,7 @@ class DbClient: def commit(self): self.__session.commit() - def add( - self, - rows: list[ - Category | CategoryGroup | CategoryRule | Tag | TagRule | Transaction - ], - ): + def add(self, rows: list): self.__session.add_all(rows) def remove_by_name(self, type: Category | Tag | Transaction, rows: list): @@ -122,6 +117,10 @@ class DbClient: def updaterules(self, rules: list[dict]): self.__session.execute(update(CategoryRule), rules) + def remove_links(self, original, links: list): + stmt = delete(Link).where(Link.original == original, Link.link.in_(link for link in links)) + self.__session.execute(stmt) + def uncategorized(self) -> list[Transaction]: stmt = select(Transaction).where(~Transaction.category.has()) return self.__session.scalars(stmt).all() diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 7ef354d..5211cf1 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -63,9 +63,6 @@ class Bank(Base): back_populates="bank", lazy="joined" ) - def __repr__(self) -> str: - return f"Bank(name={self.name}, BIC={self.BIC}, type={self.type}, nordigen={self.nordigen})" - bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] @@ -83,13 +80,9 @@ class Transaction(Base): amount: Mapped[money] category: Mapped[Optional[TransactionCategory]] = relationship() - ) note: Mapped[Optional[Note]] = relationship(back_populates="original") tags: Mapped[Optional[set[TransactionTag]]] = relationship() - def __repr__(self) -> str: - return f"Transaction(date={self.date}, description={self.description}, bank={self.bank}, amount={self.amount}, category={self.category})" - idfk = Annotated[ int, mapped_column(BigInteger, ForeignKey(Transaction.id, ondelete="CASCADE")) @@ -158,9 +151,6 @@ class Nordigen(Base): bank: Mapped[Bank] = relationship(back_populates="nordigen") - def __repr__(self) -> str: - return f"(bank_id={self.bank_id}, requisition_id={self.requisition_id}, invert={self.invert})" - class Tag(Base): __tablename__ = "tags_available" @@ -229,8 +219,12 @@ class CategorySchedule(Base): category: Mapped[Category] = relationship(back_populates="schedule") - def __repr__(self) -> str: - return f"{self.name} schedule=Schedule(period={self.period}, multiplier={self.period_multiplier}, amount={self.amount})" + +class Link(Base): + __tablename__ = "links" + + original: Mapped[idfk] = mapped_column(primary_key=True) + link: Mapped[idfk] = mapped_column(primary_key=True) class Rule: From 071711dbdbfd443dff46c88019244659fa474c83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 30 Dec 2022 16:09:40 +0000 Subject: [PATCH 31/66] Fix CategoryRule and TagRule initializations and re-adds the remove option to the modify category rule cli. --- pfbudget/__main__.py | 4 ++-- pfbudget/cli/runnable.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index c62c96f..baa1049 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -57,13 +57,13 @@ if __name__ == "__main__": params = [ pfbudget.types.CategoryRule( - cat, args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, args["bank"][0] if args["bank"] else None, args["min"][0] if args["min"] else None, args["max"][0] if args["max"] else None, + cat, ) for cat in args["category"] ] @@ -109,13 +109,13 @@ if __name__ == "__main__": params = [ pfbudget.types.TagRule( - tag, args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, args["bank"][0] if args["bank"] else None, args["min"][0] if args["min"] else None, args["max"][0] if args["max"] else None, + tag, ) for tag in args["tag"] ] diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index aa780f2..bdb1cfd 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -376,6 +376,7 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP modify.add_argument("id", nargs="+", type=int) modify.add_argument("--category", nargs=1, type=str) rules(modify) + modify.add_argument("--remove", nargs="*", default=[], type=str) def tags(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): From 55a5b09c45c70eb9e681afd50e2f0d82115134c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 1 Jan 2023 18:40:19 +0000 Subject: [PATCH 32/66] Fix download, bank, token and renew->eua ops - Update the download, token and eua cli with the new operations - Change the bank and nordigen cli to be more in line with the other add/modify/remove operations. Also update manager logic. - Fix some model.py leftovers - Add __lt__ to Transaction to enable sorting - Remove universal from child argparsers --- pfbudget/__init__.py | 1 + pfbudget/__main__.py | 92 ++++++++++++++++++ pfbudget/cli/runnable.py | 185 ++++++++++++++++++------------------- pfbudget/common/types.py | 10 +- pfbudget/core/manager.py | 53 +++++++---- pfbudget/db/client.py | 9 +- pfbudget/db/model.py | 17 ++-- pfbudget/input/input.py | 16 +--- pfbudget/input/nordigen.py | 66 ++++++++----- 9 files changed, 283 insertions(+), 166 deletions(-) diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index abbb667..75c5718 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -7,5 +7,6 @@ from pfbudget.core.categories import categorize_data from pfbudget.core.manager import Manager from pfbudget.cli.runnable import argparser from pfbudget.input.parsers import parse_data +from pfbudget.utils.utils import parse_args_period import pfbudget.db.model as types diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index baa1049..4cb673a 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -16,6 +16,98 @@ if __name__ == "__main__": params = None match (op): + case pfbudget.Operation.RequisitionId: + assert args.keys() >= {"name", "country"}, "argparser ill defined" + params = [args["name"][0], args["country"][0]] + + case pfbudget.Operation.Download: + assert args.keys() >= { + "id", + "name", + "all", + "interval", + "start", + "end", + "year", + }, "argparser ill defined" + start, end = pfbudget.parse_args_period(args) + params = [start, end] + + case pfbudget.Operation.BankAdd: + assert args.keys() >= { + "bank", + "bic", + "type", + }, "argparser ill defined" + + params = [ + pfbudget.types.Bank( + args["bank"][0], + args["bic"][0], + args["type"][0], + ) + ] + + case pfbudget.Operation.BankMod: + assert args.keys() >= { + "bank", + "bic", + "type", + "remove", + }, "argparser ill defined" + + nargs_1 = ["bic", "type"] + + param = {"name": args["bank"][0]} + param |= {k: v[0] for k, v in args.items() if k in nargs_1 and args[k]} + param |= {k: None for k in args["remove"] if k in nargs_1} + + params = [param] + + case pfbudget.Operation.BankDel: + assert len(args["bank"]) > 0, "argparser ill defined" + params = args["bank"] + + case pfbudget.Operation.NordigenAdd: + assert args.keys() >= { + "bank", + "bank_id", + "requisition_id", + "invert", + }, "argparser ill defined" + + params = [ + pfbudget.types.Nordigen( + args["bank"][0], + args["bank_id"][0] if args["bank_id"] else None, + args["requisition_id"][0] if args["requisition_id"] else None, + args["invert"] if args["invert"] else None, + ) + ] + + case pfbudget.Operation.NordigenMod: + assert args.keys() >= { + "bank", + "bank_id", + "requisition_id", + "invert", + "remove", + }, "argparser ill defined" + + nargs_1 = ["bank_id", "requisition_id"] + nargs_0 = ["invert"] + + param = {"name": args["bank"][0]} + param |= {k: v[0] for k, v in args.items() if k in nargs_1 and args[k]} + param |= {k: v for k, v in args.items() if k in nargs_0} + param |= {k: None for k in args["remove"] if k in nargs_1} + + params = [param] + + case pfbudget.Operation.NordigenDel: + assert len(args["bank"]) > 0, "argparser ill defined" + params = args["bank"] + case pfbudget.Operation.CategoryAdd: assert args.keys() >= {"category", "group"}, "argparser ill defined" params = [ diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index bdb1cfd..5f797a8 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -5,7 +5,7 @@ import decimal import re from pfbudget.common.types import Operation -from pfbudget.db.model import Period +from pfbudget.db.model import AccountType, Period from pfbudget.input.nordigen import NordigenInput from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph @@ -150,71 +150,24 @@ def argparser() -> argparse.ArgumentParser: ) p_report.set_defaults(func=report) - """ - Register bank - """ - p_register = subparsers.add_parser( - "register", - description="Register a bank", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_register.add_argument("bank", type=str, nargs=1, help="bank option help") - p_register.add_argument( - "--requisition", type=str, nargs=1, help="requisition option help" - ) - p_register.add_argument("--invert", action="store_true") - p_register.set_defaults(command=Operation.Register) + # Banks + bank(subparsers.add_parser("bank")) - """ - Unregister bank - """ - p_register = subparsers.add_parser( - "unregister", - description="Unregister a bank", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_register.add_argument("bank", type=str, nargs=1, help="bank option help") - p_register.set_defaults(command=Operation.Unregister) + # Nordigen access token + subparsers.add_parser("token").set_defaults(op=Operation.Token) - """ - Nordigen API - """ - p_nordigen_access = subparsers.add_parser( - "token", - description="Get new access token", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_nordigen_access.set_defaults(command=Operation.Token) + # Nordigen requisition id + requisition = subparsers.add_parser("eua") + requisition.set_defaults(op=Operation.RequisitionId) + requisition.add_argument("name", nargs=1, type=str) + requisition.add_argument("country", nargs=1, type=str) - """ - (Re)new bank requisition ID - """ - p_nordigen_access = subparsers.add_parser( - "renew", - description="(Re)new the Bank requisition ID", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_nordigen_access.add_argument("name", nargs=1, type=str) - p_nordigen_access.add_argument("country", nargs=1, type=str) - p_nordigen_access.set_defaults(command=Operation.Renew) - - """ - Downloading through Nordigen API - """ - p_nordigen_download = subparsers.add_parser( - "download", - description="Downloads transactions using Nordigen API", - parents=[universal, period], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_nordigen_download.add_argument("--id", nargs="+", type=str) - p_nordigen_download.add_argument("--name", nargs="+", type=str) - p_nordigen_download.add_argument("--all", action="store_true") - p_nordigen_download.set_defaults(command=Operation.Download) + # Download through the Nordigen API + download = subparsers.add_parser("download", parents=[period]) + download.set_defaults(op=Operation.Download) + download.add_argument("--id", nargs="+", type=str) + download.add_argument("--name", nargs="+", type=str) + download.add_argument("--all", action="store_true") # """ # List available banks on Nordigen API @@ -245,11 +198,10 @@ def argparser() -> argparse.ArgumentParser: # ) # Categories - category_parser = subparsers.add_parser("category", parents=[universal]) - category(category_parser, universal) + category(subparsers.add_parser("category")) # Tag - tags(subparsers.add_parser("tag", parents=[universal]), universal) + tags(subparsers.add_parser("tag")) # Link link(subparsers.add_parser("link")) @@ -312,66 +264,107 @@ def report(args): # input.list(vars(args)["country"][0]) -def download(manager, args: dict): - start, end = pfbudget.utils.parse_args_period(args) - manager.parser(NordigenInput(manager, args, start, end)) - - -def category(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): +def bank(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) - add = commands.add_parser("add", parents=[universal]) + add = commands.add_parser("add") + add.set_defaults(op=Operation.BankAdd) + add.add_argument("bank", nargs=1, type=str) + add.add_argument("bic", nargs=1, type=str) + add.add_argument("type", nargs=1, type=str, choices=[e.name for e in AccountType]) + + rem = commands.add_parser("del") + rem.set_defaults(op=Operation.BankDel) + rem.add_argument("bank", nargs="+", type=str) + + mod = commands.add_parser("mod") + mod.set_defaults(op=Operation.BankMod) + mod.add_argument("bank", nargs=1, type=str) + mod.add_argument("--bic", nargs=1, type=str) + mod.add_argument("--type", nargs=1, type=str, choices=[e.name for e in AccountType]) + mod.add_argument("--remove", nargs="*", default=[], type=str) + + nordigen(commands.add_parser("nordigen")) + + +def nordigen(parser: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add") + add.set_defaults(op=Operation.NordigenAdd) + add.add_argument("bank", nargs=1, type=str) + add.add_argument("--bank_id", nargs=1, type=str) + add.add_argument("--requisition_id", nargs=1, type=str) + add.add_argument("--invert", action="store_true") + + rem = commands.add_parser("del") + rem.set_defaults(op=Operation.NordigenDel) + rem.add_argument("bank", nargs="+", type=str) + + mod = commands.add_parser("mod") + mod.set_defaults(op=Operation.NordigenMod) + mod.add_argument("bank", nargs=1, type=str) + mod.add_argument("--bank_id", nargs=1, type=str) + mod.add_argument("--requisition_id", nargs=1, type=str) + mod.add_argument("--invert", action="store_true") + mod.add_argument("--remove", nargs="*", default=[], type=str) + + +def category(parser: argparse.ArgumentParser): + commands = parser.add_subparsers(required=True) + + add = commands.add_parser("add") add.set_defaults(op=Operation.CategoryAdd) add.add_argument("category", nargs="+", type=str) add.add_argument("--group", nargs="?", type=str) - remove = commands.add_parser("remove", parents=[universal]) + remove = commands.add_parser("remove") remove.set_defaults(op=Operation.CategoryRemove) remove.add_argument("category", nargs="+", type=str) - update = commands.add_parser("update", parents=[universal]) + update = commands.add_parser("update") update.set_defaults(op=Operation.CategoryUpdate) update.add_argument("category", nargs="+", type=str) update.add_argument("--group", nargs="?", type=str) - schedule = commands.add_parser("schedule", parents=[universal]) + schedule = commands.add_parser("schedule") schedule.set_defaults(op=Operation.CategorySchedule) schedule.add_argument("category", nargs="+", type=str) schedule.add_argument("period", nargs=1, choices=[e.value for e in Period]) schedule.add_argument("--frequency", nargs=1, default=[1], type=int) - rule = commands.add_parser("rule", parents=[universal]) - category_rule(rule, universal) + rule = commands.add_parser("rule") + category_rule(rule) - group = commands.add_parser("group", parents=[universal]) - category_group(group, universal) + group = commands.add_parser("group") + category_group(group) -def category_group(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): +def category_group(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) - add = commands.add_parser("add", parents=[universal]) + add = commands.add_parser("add") add.set_defaults(op=Operation.GroupAdd) add.add_argument("group", nargs="+", type=str) - remove = commands.add_parser("remove", parents=[universal]) + remove = commands.add_parser("remove") remove.set_defaults(op=Operation.GroupRemove) remove.add_argument("group", nargs="+", type=str) -def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): +def category_rule(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) - add = commands.add_parser("add", parents=[universal]) + add = commands.add_parser("add") add.set_defaults(op=Operation.RuleAdd) add.add_argument("category", nargs="+", type=str) rules(add) - remove = commands.add_parser("remove", parents=[universal]) + remove = commands.add_parser("remove") remove.set_defaults(op=Operation.RuleRemove) remove.add_argument("id", nargs="+", type=int) - modify = commands.add_parser("modify", parents=[universal]) + modify = commands.add_parser("modify") modify.set_defaults(op=Operation.RuleModify) modify.add_argument("id", nargs="+", type=int) modify.add_argument("--category", nargs=1, type=str) @@ -379,34 +372,34 @@ def category_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentP modify.add_argument("--remove", nargs="*", default=[], type=str) -def tags(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): +def tags(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) - add = commands.add_parser("add", parents=[universal]) + add = commands.add_parser("add") add.set_defaults(op=Operation.TagAdd) add.add_argument("tag", nargs="+", type=str) - remove = commands.add_parser("remove", parents=[universal]) + remove = commands.add_parser("remove") remove.set_defaults(op=Operation.TagRemove) remove.add_argument("tag", nargs="+", type=str) - rule = commands.add_parser("rule", parents=[universal]) - tag_rule(rule, universal) + rule = commands.add_parser("rule") + tag_rule(rule) -def tag_rule(parser: argparse.ArgumentParser, universal: argparse.ArgumentParser): +def tag_rule(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) - add = commands.add_parser("add", parents=[universal]) + add = commands.add_parser("add") add.set_defaults(op=Operation.TagRuleAdd) add.add_argument("tag", nargs="+", type=str) rules(add) - remove = commands.add_parser("remove", parents=[universal]) + remove = commands.add_parser("remove") remove.set_defaults(op=Operation.TagRuleRemove) remove.add_argument("id", nargs="+", type=int) - modify = commands.add_parser("modify", parents=[universal]) + modify = commands.add_parser("modify") modify.set_defaults(op=Operation.TagRuleModify) modify.add_argument("id", nargs="+", type=int) modify.add_argument("--tag", nargs=1, type=str) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 0213c03..91f130a 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -9,10 +9,8 @@ class Operation(Enum): Parse = auto() Download = auto() Categorize = auto() - Register = auto() - Unregister = auto() Token = auto() - Renew = auto() + RequisitionId = auto() CategoryAdd = auto() CategoryUpdate = auto() CategoryRemove = auto() @@ -29,6 +27,12 @@ class Operation(Enum): TagRuleModify = auto() Forge = auto() Dismantle = auto() + BankAdd = auto() + BankMod = auto() + BankDel = auto() + NordigenAdd = auto() + NordigenMod = auto() + NordigenDel = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index b2c516e..48aa835 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,12 +1,15 @@ from pfbudget.input.input import Input -from pfbudget.input.nordigen import NordigenClient +from pfbudget.input.nordigen import NordigenInput from pfbudget.input.parsers import parse_data from pfbudget.db.client import DbClient from pfbudget.db.model import ( + Bank, Category, CategoryGroup, CategoryRule, CategorySchedule, + Nordigen, + Rule, Tag, TagRule, ) @@ -14,7 +17,7 @@ from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert -from pfbudget.cli.runnable import download, parse +from pfbudget.cli.runnable import parse class Manager: @@ -34,9 +37,15 @@ class Manager: case Operation.Parse: # TODO this is a monstrosity, remove when possible parse(self, self.args) + case Operation.Download: - # TODO this is a monstrosity, remove when possible - download(self, self.args) + client = NordigenInput() + client.banks = self.get_banks() + client.start = params[0] + client.end = params[1] + transactions = client.parse() + with self.db.session() as session: + session.add(transactions) case Operation.Categorize: with self.db.session() as session: @@ -45,23 +54,29 @@ class Manager: tags = session.tags() Categorizer().categorize(uncategorized, categories, tags) - case Operation.Register: - # self._db = DbClient(args["database"]) - # self.register(args) - pass - case Operation.Unregister: - # self._db = DbClient(args["database"]) - # self.unregister(args) - pass + case Operation.BankMod: + with self.db.session() as session: + session.update(Bank, params) + + case Operation.NordigenMod: + with self.db.session() as session: + session.update(Nordigen, params) + + case Operation.BankDel: + with self.db.session() as session: + session.remove_by_name(Bank, params) + + case Operation.NordigenDel: + with self.db.session() as session: + session.remove_by_name(Nordigen, params) + case Operation.Token: - NordigenClient(self).token() + NordigenInput().token() - case Operation.Renew: - NordigenClient(self).requisition( - self.args["name"], self.args["country"] - ) + case Operation.RequisitionId: + NordigenInput().requisition(params[0], params[1]) - case Operation.CategoryAdd | Operation.RuleAdd | Operation.TagAdd | Operation.TagRuleAdd: + case Operation.BankAdd | Operation.CategoryAdd | Operation.NordigenAdd | Operation.RuleAdd | Operation.TagAdd | Operation.TagRuleAdd: with self.db.session() as session: session.add(params) @@ -94,7 +109,7 @@ class Manager: case Operation.RuleModify | Operation.TagRuleModify: assert all(isinstance(param, dict) for param in params) with self.db.session() as session: - session.updaterules(params) + session.update(Rule, params) case Operation.GroupAdd: with self.db.session() as session: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 5af6181..41a2411 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -114,11 +114,14 @@ class DbClient: stmt = delete(type).where(type.id.in_(ids)) self.__session.execute(stmt) - def updaterules(self, rules: list[dict]): - self.__session.execute(update(CategoryRule), rules) + def update(self, type, values: list[dict]): + print(type, values) + self.__session.execute(update(type), values) def remove_links(self, original, links: list): - stmt = delete(Link).where(Link.original == original, Link.link.in_(link for link in links)) + stmt = delete(Link).where( + Link.original == original, Link.link.in_(link for link in links) + ) self.__session.execute(stmt) def uncategorized(self) -> list[Transaction]: diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 5211cf1..b961d6d 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -59,9 +59,7 @@ class Bank(Base): BIC: Mapped[str] = mapped_column(String(8), primary_key=True) type: Mapped[accounttype] = mapped_column(primary_key=True) - nordigen: Mapped[Optional[Nordigen]] = relationship( - back_populates="bank", lazy="joined" - ) + nordigen: Mapped[Optional[Nordigen]] = relationship(lazy="joined") bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] @@ -79,9 +77,12 @@ class Transaction(Base): bank: Mapped[bankfk] amount: Mapped[money] - category: Mapped[Optional[TransactionCategory]] = relationship() - note: Mapped[Optional[Note]] = relationship(back_populates="original") - tags: Mapped[Optional[set[TransactionTag]]] = relationship() + category: Mapped[Optional[TransactionCategory]] = relationship(init=False) + note: Mapped[Optional[Note]] = relationship(init=False) + tags: Mapped[Optional[set[TransactionTag]]] = relationship(init=False) + + def __lt__(self, other): + return self.date < other.date idfk = Annotated[ @@ -138,8 +139,6 @@ class Note(Base): id: Mapped[idfk] = mapped_column(primary_key=True, init=False) note: Mapped[str] - original: Mapped[Transaction] = relationship(back_populates="note") - class Nordigen(Base): __tablename__ = "nordigen" @@ -149,8 +148,6 @@ class Nordigen(Base): requisition_id: Mapped[Optional[str]] invert: Mapped[Optional[bool]] - bank: Mapped[Bank] = relationship(back_populates="nordigen") - class Tag(Base): __tablename__ = "tags_available" diff --git a/pfbudget/input/input.py b/pfbudget/input/input.py index 5dc5a46..2793fb2 100644 --- a/pfbudget/input/input.py +++ b/pfbudget/input/input.py @@ -1,21 +1,9 @@ -from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING -from pfbudget.common.types import Transactions - -if TYPE_CHECKING: - from pfbudget.core.manager import Manager +from pfbudget.db.model import Transaction class Input(ABC): - def __init__(self, manager: Manager): - self._manager = manager - @abstractmethod - def parse(self) -> Transactions: + def parse(self) -> list[Transaction]: return NotImplemented - - @property - def manager(self): - return self._manager diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index e43f878..4f2482d 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -17,8 +17,8 @@ load_dotenv() class NordigenInput(Input): - def __init__(self, manager, options: dict = {}, start=date.min, end=date.max): - super().__init__(manager) + def __init__(self): + super().__init__() self._client = NordigenClient( secret_key=os.environ.get("SECRET_KEY"), secret_id=os.environ.get("SECRET_ID"), @@ -28,33 +28,33 @@ class NordigenInput(Input): # print(options) - if "all" in options and options["all"]: - self.__banks = self.manager.get_banks() - elif "id" in options and options["id"]: - self.__banks = [ - self.manager.get_bank_by("nordigen_id", b) for b in options["id"] - ] - elif "name" in options and options["name"]: - self.__banks = [ - self.manager.get_bank_by("name", b) for b in options["name"] - ] - else: - self.__banks = None + # if "all" in options and options["all"]: + # self.__banks = self.manager.get_banks() + # elif "id" in options and options["id"]: + # self.__banks = [ + # self.manager.get_bank_by("nordigen_id", b) for b in options["id"] + # ] + # elif "name" in options and options["name"]: + # self.__banks = [ + # self.manager.get_bank_by("name", b) for b in options["name"] + # ] + # else: + # self.__banks = None - self.__from = start - self.__to = end + self._start = date.min + self._end = date.max def parse(self) -> list[Transaction]: transactions = [] - if not self.__banks: - raise NoBankSelected + assert len(self._banks) > 0 - for bank in self.__banks: + for bank in self._banks: print(f"Downloading from {bank}...") requisition = self.client.requisition.get_requisition_by_id( bank.nordigen.requisition_id ) + print(requisition) for acc in requisition["accounts"]: account = self._client.account_api(acc) @@ -85,10 +85,10 @@ class NordigenInput(Input): ] transactions.extend( - [t for t in converted if self.__from <= t.date <= self.__to] + [t for t in converted if self._start <= t.date <= self._end] ) - return transactions + return sorted(transactions) def token(self): token = self._client.generate_token() @@ -106,6 +106,30 @@ class NordigenInput(Input): def client(self): return self._client + @property + def banks(self): + return self._banks + + @banks.setter + def banks(self, value): + self._banks = value + + @property + def start(self): + return self._start + + @start.setter + def start(self, value): + self._start = value + + @property + def end(self): + return self._end + + @end.setter + def end(self, value): + self._end = value + def __token(self): if token := os.environ.get("TOKEN"): return token From 9500e808de000f354b9a3a98d6a44be357dacf73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 6 Jan 2023 22:05:01 +0000 Subject: [PATCH 33/66] Update the parse operation to coordinate with the manager --- pfbudget/__main__.py | 4 ++++ pfbudget/cli/runnable.py | 36 ++++++------------------------------ pfbudget/core/manager.py | 35 +++++++++++++++++++++-------------- pfbudget/input/parsers.py | 7 ++++--- 4 files changed, 35 insertions(+), 47 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 4cb673a..9e1f45d 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -16,6 +16,10 @@ if __name__ == "__main__": params = None match (op): + case pfbudget.Operation.Parse: + assert args.keys() >= {"path", "bank", "creditcard"} + params = [args["path"], args["bank"], args["creditcard"]] + case pfbudget.Operation.RequisitionId: assert args.keys() >= {"name", "country"}, "argparser ill defined" params = [args["name"][0], args["country"][0]] diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 5f797a8..230e611 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -86,20 +86,12 @@ def argparser() -> argparse.ArgumentParser: ) p_export.set_defaults(func=lambda args: DatabaseClient(args.database).export()) - """ - Parsing - """ - p_parse = subparsers.add_parser( - "parse", - description="Parses and adds the requested transactions into the selected database", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_parse.add_argument("path", nargs="+", type=str) - p_parse.add_argument("--bank", nargs=1, type=str) - p_parse.add_argument("--creditcard", nargs=1, type=str) - p_parse.add_argument("--category", nargs=1, type=int) - p_parse.set_defaults(command=Operation.Parse) + # Parse from .csv + parse = subparsers.add_parser("parse") + parse.set_defaults(op=Operation.Parse) + parse.add_argument("path", nargs="+", type=str) + parse.add_argument("--bank", nargs=1, type=str) + parse.add_argument("--creditcard", nargs=1, type=str) """ Categorizing @@ -209,22 +201,6 @@ def argparser() -> argparse.ArgumentParser: return parser -def parse(manager, args): - """Parses the contents of the path in args to the selected database. - - Args: - args (dict): argparse variables - """ - for path in args.path: - if (dir := Path(path)).is_dir(): - for file in dir.iterdir(): - manager.parse(file, vars(args)) - elif Path(path).is_file(): - manager.parse(path, vars(args)) - else: - raise FileNotFoundError - - def graph(args): """Plots the transactions over a period of time. diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 48aa835..ff2af96 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,3 +1,5 @@ +from pathlib import Path + from pfbudget.input.input import Input from pfbudget.input.nordigen import NordigenInput from pfbudget.input.parsers import parse_data @@ -17,8 +19,6 @@ from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer from pfbudget.utils import convert -from pfbudget.cli.runnable import parse - class Manager: def __init__(self, db: str, verbosity: int = 0, args: dict = {}): @@ -34,9 +34,23 @@ class Manager: match (op): case Operation.Init: pass + case Operation.Parse: - # TODO this is a monstrosity, remove when possible - parse(self, self.args) + # Adapter for the parse_data method. Can be refactored. + args = {"bank": params[1], "creditcard": params[2], "category": None} + transactions = [] + for path in params[0]: + if (dir := Path(path)).is_dir(): + for file in dir.iterdir(): + transactions.extend(self.parse(file, args)) + elif Path(path).is_file(): + transactions.extend(self.parse(path, args)) + else: + raise FileNotFoundError(path) + + print(transactions) + if len(transactions) > 0 and input("Commit? (y/n)") == "y": + self.add_transactions(sorted(transactions)) case Operation.Download: client = NordigenInput() @@ -44,8 +58,7 @@ class Manager: client.start = params[0] client.end = params[1] transactions = client.parse() - with self.db.session() as session: - session.add(transactions) + self.add_transactions(transactions) case Operation.Categorize: with self.db.session() as session: @@ -143,14 +156,8 @@ class Manager: # client = DatabaseClient(self.__db) # client.unregister_bank(self.args["bank"][0]) - def parser(self, parser: Input): - transactions = parser.parse() - print(transactions) - # self.add_transactions(transactions) - - # def parse(self, filename: str): - # transactions = parse_data(filename, self.args) - # self.add_transactions(transactions) + def parse(self, filename: str, args: dict): + return parse_data(filename, args) # def transactions() -> list[Transaction]: # pass diff --git a/pfbudget/input/parsers.py b/pfbudget/input/parsers.py index 22bd888..facacf8 100644 --- a/pfbudget/input/parsers.py +++ b/pfbudget/input/parsers.py @@ -4,7 +4,8 @@ from importlib import import_module import datetime as dt import yaml -from pfbudget.common.types import NoBankSelected, Transaction, Transactions +from pfbudget.common.types import NoBankSelected +from pfbudget.db.model import Transaction from pfbudget.utils import utils Index = namedtuple( @@ -43,7 +44,7 @@ Options = namedtuple( ) -def parse_data(filename: str, args: dict) -> Transactions: +def parse_data(filename: str, args: dict) -> list[Transaction]: cfg: dict = yaml.safe_load(open("parsers.yaml")) assert ( "Banks" in cfg @@ -157,7 +158,7 @@ class Parser: category = line[options.category] transaction = Transaction(date, text, bank, value, category) else: - transaction = Transaction(date, text, bank, value, options.category) + transaction = Transaction(date, text, bank, value) if options.additional_parser: func(transaction) From 9b45ee48175f83ca379c484515a4226981756b15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 8 Jan 2023 17:35:48 +0000 Subject: [PATCH 34/66] Update the export operation to work with the Manager. Also removes the run method from the runnable.py, since everything is done in the __main__.py file of the pfbudget module. --- pfbudget/__main__.py | 10 +++++++++- pfbudget/cli/runnable.py | 26 +++++++------------------- pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 35 +++++++++++++++++++++-------------- pfbudget/db/client.py | 13 +++++++++++++ pfbudget/input/input.py | 2 +- pfbudget/output/__init__.py | 1 + pfbudget/output/csv.py | 17 +++++++++++++++++ pfbudget/output/output.py | 9 +++++++++ pfbudget/utils/utils.py | 1 - 10 files changed, 79 insertions(+), 36 deletions(-) create mode 100644 pfbudget/output/__init__.py create mode 100644 pfbudget/output/csv.py create mode 100644 pfbudget/output/output.py diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 9e1f45d..971b4b0 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -251,4 +251,12 @@ if __name__ == "__main__": pfbudget.types.Link(args["original"][0], link) for link in args["links"] ] - pfbudget.Manager(db, verbosity, args).action(op, params) + case pfbudget.Operation.Export: + assert args.keys() >= {"interval", "start", "end", "year", "all", "banks", "file"} + start, end = pfbudget.parse_args_period(args) + params = [start, end] + if not args["all"]: + params.append(args["banks"]) + params.append(args["file"][0]) + + pfbudget.Manager(db, verbosity).action(op, params) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 230e611..3e07e92 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -1,4 +1,3 @@ -from pathlib import Path import argparse import datetime as dt import decimal @@ -6,7 +5,6 @@ import re from pfbudget.common.types import Operation from pfbudget.db.model import AccountType, Period -from pfbudget.input.nordigen import NordigenInput from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph import pfbudget.reporting.report @@ -29,7 +27,6 @@ class DataFileMissing(Exception): def argparser() -> argparse.ArgumentParser: - universal = argparse.ArgumentParser(add_help=False) universal.add_argument( "-db", @@ -75,16 +72,13 @@ def argparser() -> argparse.ArgumentParser: ) p_init.set_defaults(command=Operation.Init) - """ - Exporting - """ - p_export = subparsers.add_parser( - "export", - description="Exports the selected database to a .csv file", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_export.set_defaults(func=lambda args: DatabaseClient(args.database).export()) + # Exports transactions to .csv file + export = subparsers.add_parser("export", parents=[period]) + export.set_defaults(op=Operation.Export) + export.add_argument("file", nargs=1, type=str) + export_banks = export.add_mutually_exclusive_group() + export_banks.add_argument("--all", action="store_true") + export_banks.add_argument("--banks", nargs="+", type=str) # Parse from .csv parse = subparsers.add_parser("parse") @@ -403,9 +397,3 @@ def link(parser: argparse.ArgumentParser): dismantle.set_defaults(op=Operation.Dismantle) dismantle.add_argument("original", nargs=1, type=int) dismantle.add_argument("links", nargs="+", type=int) - - -def run(): - args = vars(argparser().parse_args()) - assert "op" in args, "No operation selected" - return args["op"], args diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 91f130a..da37557 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -33,6 +33,7 @@ class Operation(Enum): NordigenAdd = auto() NordigenMod = auto() NordigenDel = auto() + Export = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index ff2af96..b2a1f25 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,8 +1,7 @@ from pathlib import Path -from pfbudget.input.input import Input -from pfbudget.input.nordigen import NordigenInput -from pfbudget.input.parsers import parse_data +from pfbudget.common.types import Operation +from pfbudget.core.categorizer import Categorizer from pfbudget.db.client import DbClient from pfbudget.db.model import ( Bank, @@ -15,15 +14,14 @@ from pfbudget.db.model import ( Tag, TagRule, ) -from pfbudget.common.types import Operation -from pfbudget.core.categorizer import Categorizer -from pfbudget.utils import convert +from pfbudget.input.nordigen import NordigenInput +from pfbudget.input.parsers import parse_data +from pfbudget.output.csv import CSV +from pfbudget.output.output import Output class Manager: - def __init__(self, db: str, verbosity: int = 0, args: dict = {}): - self._args = args - + def __init__(self, db: str, verbosity: int = 0): self._db = db self._verbosity = verbosity @@ -143,6 +141,19 @@ class Manager: links = [link.link for link in params] session.remove_links(original, links) + case Operation.Export: + with self.db.session() as session: + if len(params) < 4: + banks = [bank.name for bank in session.banks()] + transactions = session.transactions(params[0], params[1], banks) + else: + transactions = session.transactions( + params[0], params[1], params[2] + ) + + csvwriter: Output = CSV(params[-1]) + csvwriter.report(transactions) + # def init(self): # client = DatabaseClient(self.__db) # client.init() @@ -176,12 +187,8 @@ class Manager: @property def db(self) -> DbClient: - return DbClient(self._db, self._verbosity > 0) + return DbClient(self._db, self._verbosity > 2) @db.setter def db(self, url: str): self._db = url - - @property - def args(self) -> dict: - return self._args diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 41a2411..ea63461 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,4 +1,5 @@ from dataclasses import asdict +from datetime import date from sqlalchemy import create_engine, delete, select, update from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session, joinedload, selectinload @@ -128,6 +129,14 @@ class DbClient: stmt = select(Transaction).where(~Transaction.category.has()) return self.__session.scalars(stmt).all() + def transactions(self, min: date, max: date, banks: list[str]): + stmt = select(Transaction).where( + Transaction.date >= min, + Transaction.date <= max, + Transaction.bank.in_(banks), + ) + return self.__session.scalars(stmt).all() + def categories(self) -> list[Category]: stmt = select(Category) return self.__session.scalars(stmt).all() @@ -136,5 +145,9 @@ class DbClient: stmt = select(Tag) return self.__session.scalars(stmt).all() + def banks(self) -> list[Bank]: + stmt = select(Bank) + return self.__session.scalars(stmt).all() + def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/input/input.py b/pfbudget/input/input.py index 2793fb2..3e58851 100644 --- a/pfbudget/input/input.py +++ b/pfbudget/input/input.py @@ -6,4 +6,4 @@ from pfbudget.db.model import Transaction class Input(ABC): @abstractmethod def parse(self) -> list[Transaction]: - return NotImplemented + return NotImplementedError diff --git a/pfbudget/output/__init__.py b/pfbudget/output/__init__.py new file mode 100644 index 0000000..376b17c --- /dev/null +++ b/pfbudget/output/__init__.py @@ -0,0 +1 @@ +__all__ = ["csv", "output"] diff --git a/pfbudget/output/csv.py b/pfbudget/output/csv.py new file mode 100644 index 0000000..55135e8 --- /dev/null +++ b/pfbudget/output/csv.py @@ -0,0 +1,17 @@ +from csv import writer + +from pfbudget.db.model import Transaction + +from .output import Output + + +class CSV(Output): + def __init__(self, filename: str): + self.fn = filename + + def report(self, transactions: list[Transaction]): + with open(self.fn, "w", newline="") as f: + w = writer(f, delimiter="\t") + w.writerows( + [(t.date, t.description, t.amount, t.bank) for t in transactions] + ) diff --git a/pfbudget/output/output.py b/pfbudget/output/output.py new file mode 100644 index 0000000..bf7e918 --- /dev/null +++ b/pfbudget/output/output.py @@ -0,0 +1,9 @@ +from abc import ABC, abstractmethod + +from pfbudget.db.model import Transaction + + +class Output(ABC): + @abstractmethod + def report(self, transactions: list[Transaction]): + raise NotImplementedError diff --git a/pfbudget/utils/utils.py b/pfbudget/utils/utils.py index 6cacab6..eb0420a 100644 --- a/pfbudget/utils/utils.py +++ b/pfbudget/utils/utils.py @@ -61,7 +61,6 @@ def find_credit_institution(fn, banks, creditcards): def parse_args_period(args: dict): start, end = date.min, date.max - print(args) if args["start"]: start = datetime.strptime(args["start"][0], "%Y/%m/%d").date() From 86afa992176ff3f32262e430cfd8fda542c2ffc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 8 Jan 2023 19:35:19 +0000 Subject: [PATCH 35/66] Finish the remaining Nordigen operations from the Manager POV and the update on the argparses. Also clears unnecessary methods from the DB client interface. Better assert information on the __main__.py --- pfbudget/__main__.py | 133 +++++++++++++++-------------------- pfbudget/cli/runnable.py | 38 +++------- pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 36 +++++++--- pfbudget/db/client.py | 66 ++++------------- pfbudget/input/json.py | 30 -------- pfbudget/input/nordigen.py | 50 ++++--------- pfbudget/utils/converters.py | 44 ------------ 8 files changed, 120 insertions(+), 278 deletions(-) delete mode 100644 pfbudget/input/json.py diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 971b4b0..fa5eb31 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -17,32 +17,32 @@ if __name__ == "__main__": params = None match (op): case pfbudget.Operation.Parse: - assert args.keys() >= {"path", "bank", "creditcard"} + keys = {"path", "bank", "creditcard"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [args["path"], args["bank"], args["creditcard"]] case pfbudget.Operation.RequisitionId: - assert args.keys() >= {"name", "country"}, "argparser ill defined" + keys = {"name", "country"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [args["name"][0], args["country"][0]] case pfbudget.Operation.Download: - assert args.keys() >= { - "id", - "name", - "all", - "interval", - "start", - "end", - "year", - }, "argparser ill defined" + keys = {"all", "banks", "interval", "start", "end", "year", "dry_run"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + start, end = pfbudget.parse_args_period(args) - params = [start, end] + params = [start, end, args["dry_run"]] + + if not args["all"]: + params.append(args["banks"]) + else: + params.append([]) case pfbudget.Operation.BankAdd: - assert args.keys() >= { - "bank", - "bic", - "type", - }, "argparser ill defined" + keys = {"bank", "bic", "type"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ pfbudget.types.Bank( @@ -53,12 +53,8 @@ if __name__ == "__main__": ] case pfbudget.Operation.BankMod: - assert args.keys() >= { - "bank", - "bic", - "type", - "remove", - }, "argparser ill defined" + keys = {"bank", "bic", "type", "remove"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" nargs_1 = ["bic", "type"] @@ -73,12 +69,8 @@ if __name__ == "__main__": params = args["bank"] case pfbudget.Operation.NordigenAdd: - assert args.keys() >= { - "bank", - "bank_id", - "requisition_id", - "invert", - }, "argparser ill defined" + keys = {"bank", "bank_id", "requisition_id", "invert"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ pfbudget.types.Nordigen( @@ -90,13 +82,8 @@ if __name__ == "__main__": ] case pfbudget.Operation.NordigenMod: - assert args.keys() >= { - "bank", - "bank_id", - "requisition_id", - "invert", - "remove", - }, "argparser ill defined" + keys = {"bank", "bank_id", "requisition_id", "invert", "remove"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" nargs_1 = ["bank_id", "requisition_id"] nargs_0 = ["invert"] @@ -112,14 +99,24 @@ if __name__ == "__main__": assert len(args["bank"]) > 0, "argparser ill defined" params = args["bank"] + case pfbudget.Operation.NordigenCountryBanks: + keys = {"country"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + + params = [args["country"][0]] + case pfbudget.Operation.CategoryAdd: - assert args.keys() >= {"category", "group"}, "argparser ill defined" + keys = {"category", "group"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [ pfbudget.types.Category(cat, args["group"]) for cat in args["category"] ] case pfbudget.Operation.CategoryUpdate: - assert args.keys() >= {"category", "group"}, "argparser ill defined" + keys = {"category", "group"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [pfbudget.types.Category(cat) for cat in args["category"]] params.append(args["group"]) @@ -128,11 +125,8 @@ if __name__ == "__main__": params = [pfbudget.types.Category(cat) for cat in args["category"]] case pfbudget.Operation.CategorySchedule: - assert args.keys() >= { - "category", - "period", - "frequency", - }, "argparser ill defined" + keys = {"category", "period", "frequency"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ pfbudget.types.CategorySchedule( @@ -142,14 +136,8 @@ if __name__ == "__main__": ] case pfbudget.Operation.RuleAdd: - assert args.keys() >= { - "category", - "date", - "description", - "bank", - "min", - "max", - }, "argparser ill defined" + keys = {"category", "date", "description", "bank", "min", "max"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ pfbudget.types.CategoryRule( @@ -165,11 +153,13 @@ if __name__ == "__main__": ] case pfbudget.Operation.RuleRemove | pfbudget.Operation.TagRuleRemove: - assert args.keys() >= {"id"}, "argparser ill defined" + keys = {"id"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = args["id"] case pfbudget.Operation.RuleModify: - assert args.keys() >= { + keys = { "id", "category", "date", @@ -178,7 +168,8 @@ if __name__ == "__main__": "min", "max", "remove", - }, "argparser ill defined" + } + assert args.keys() >= keys, f"missing {args.keys() - keys}" nargs_1 = ["category", "date", "description", "regex", "bank", "min", "max"] params = [] @@ -190,18 +181,14 @@ if __name__ == "__main__": params.append(param) case pfbudget.Operation.TagAdd: - assert args.keys() >= {"tag"}, "argparser ill defined" + keys = {"tag"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [pfbudget.types.Tag(tag) for tag in args["tag"]] case pfbudget.Operation.TagRuleAdd: - assert args.keys() >= { - "tag", - "date", - "description", - "bank", - "min", - "max", - }, "argparser ill defined" + keys = {"tag", "date", "description", "bank", "min", "max"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ pfbudget.types.TagRule( @@ -217,16 +204,8 @@ if __name__ == "__main__": ] case pfbudget.Operation.TagRuleModify: - assert args.keys() >= { - "id", - "tag", - "date", - "description", - "bank", - "min", - "max", - "remove", - }, "argparser ill defined" + keys = {"id", "tag", "date", "description", "bank", "min", "max", "remove"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" nargs_1 = ["tag", "date", "description", "regex", "bank", "min", "max"] params = [] @@ -246,13 +225,17 @@ if __name__ == "__main__": params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] case pfbudget.Operation.Forge | pfbudget.Operation.Dismantle: - assert args.keys() >= {"original", "links"}, "argparser ill defined" + keys = {"original", "links"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + params = [ pfbudget.types.Link(args["original"][0], link) for link in args["links"] ] case pfbudget.Operation.Export: - assert args.keys() >= {"interval", "start", "end", "year", "all", "banks", "file"} + keys = {"interval", "start", "end", "year", "all", "banks", "file"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + start, end = pfbudget.parse_args_period(args) params = [start, end] if not args["all"]: diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 3e07e92..6b5fb42 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -151,37 +151,15 @@ def argparser() -> argparse.ArgumentParser: # Download through the Nordigen API download = subparsers.add_parser("download", parents=[period]) download.set_defaults(op=Operation.Download) - download.add_argument("--id", nargs="+", type=str) - download.add_argument("--name", nargs="+", type=str) - download.add_argument("--all", action="store_true") + download_banks = download.add_mutually_exclusive_group() + download_banks.add_argument("--all", action="store_true") + download_banks.add_argument("--banks", nargs="+", type=str) + download.add_argument("--dry-run", action="store_true") - # """ - # List available banks on Nordigen API - # """ - # p_nordigen_list = subparsers.add_parser( - # "list", - # description="Lists banks in {country}", - # parents=[help], - # formatter_class=argparse.ArgumentDefaultsHelpFormatter, - # ) - # p_nordigen_list.add_argument("country", nargs=1, type=str) - # p_nordigen_list.set_defaults(func=lambda args: nordigen_banks(manager, args)) - - # """ - # Nordigen JSONs - # """ - # p_nordigen_json = subparsers.add_parser( - # "json", - # description="", - # parents=[help], - # formatter_class=argparse.ArgumentDefaultsHelpFormatter, - # ) - # p_nordigen_json.add_argument("json", nargs=1, type=str) - # p_nordigen_json.add_argument("bank", nargs=1, type=str) - # p_nordigen_json.add_argument("--invert", action=argparse.BooleanOptionalAction) - # p_nordigen_json.set_defaults( - # func=lambda args: manager.parser(JsonParser(vars(args))) - # ) + # List available banks in country C + banks = subparsers.add_parser("banks") + banks.set_defaults(op=Operation.NordigenCountryBanks) + banks.add_argument("country", nargs=1, type=str) # Categories category(subparsers.add_parser("category")) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index da37557..71a763f 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -33,6 +33,7 @@ class Operation(Enum): NordigenAdd = auto() NordigenMod = auto() NordigenDel = auto() + NordigenCountryBanks = auto() Export = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index b2a1f25..60a5259 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,4 +1,5 @@ from pathlib import Path +import webbrowser from pfbudget.common.types import Operation from pfbudget.core.categorizer import Categorizer @@ -8,11 +9,11 @@ from pfbudget.db.model import ( Category, CategoryGroup, CategoryRule, - CategorySchedule, Nordigen, Rule, Tag, TagRule, + Transaction, ) from pfbudget.input.nordigen import NordigenInput from pfbudget.input.parsers import parse_data @@ -52,17 +53,27 @@ class Manager: case Operation.Download: client = NordigenInput() - client.banks = self.get_banks() + with self.db.session() as session: + if len(params[3]) == 0: + client.banks = session.get(Bank, Bank.nordigen) + else: + client.banks = session.get(Bank, Bank.name, params[3]) + session.expunge_all() client.start = params[0] client.end = params[1] transactions = client.parse() - self.add_transactions(transactions) + + # dry-run + if not params[2]: + self.add_transactions(transactions) + else: + print(transactions) case Operation.Categorize: with self.db.session() as session: - uncategorized = session.uncategorized() - categories = session.categories() - tags = session.tags() + uncategorized = session.get(Transaction, ~Transaction.category) + categories = session.get(Category) + tags = session.get(Tag) Categorizer().categorize(uncategorized, categories, tags) case Operation.BankMod: @@ -85,7 +96,13 @@ class Manager: NordigenInput().token() case Operation.RequisitionId: - NordigenInput().requisition(params[0], params[1]) + link, _ = NordigenInput().requisition(params[0], params[1]) + print(f"Opening {link} to request access to {params[0]}") + webbrowser.open(link) + + case Operation.NordigenCountryBanks: + banks = NordigenInput().country_banks(params[0]) + print(banks) case Operation.BankAdd | Operation.CategoryAdd | Operation.NordigenAdd | Operation.RuleAdd | Operation.TagAdd | Operation.TagRuleAdd: with self.db.session() as session: @@ -144,7 +161,7 @@ class Manager: case Operation.Export: with self.db.session() as session: if len(params) < 4: - banks = [bank.name for bank in session.banks()] + banks = [bank.name for bank in session.get(Bank)] transactions = session.transactions(params[0], params[1], banks) else: transactions = session.transactions( @@ -182,9 +199,6 @@ class Manager: # bank = client.get_bank(key, value) # return convert(bank) - def get_banks(self): - return self.db.get_nordigen_banks() - @property def db(self) -> DbClient: return DbClient(self._db, self._verbosity > 2) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index ea63461..538e82a 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -2,10 +2,9 @@ from dataclasses import asdict from datetime import date from sqlalchemy import create_engine, delete, select, update from sqlalchemy.dialects.postgresql import insert -from sqlalchemy.orm import Session, joinedload, selectinload +from sqlalchemy.orm import Session from pfbudget.db.model import ( - Bank, Category, CategoryGroup, CategoryRule, @@ -32,39 +31,6 @@ class DbClient: def __init__(self, url: str, echo=False) -> None: self._engine = create_engine(url, echo=echo) - def get_transactions(self): - """¿Non-optimized? get_transactions, will load the entire Transaction""" - with Session(self.engine) as session: - stmt = select(Transaction).options( - joinedload("*"), selectinload(Transaction.tags) - ) - return session.scalars(stmt).all() - - def get_uncategorized(self): - with Session(self.engine) as session: - stmt = select(Transaction).where(~Transaction.category.has()) - return session.scalars(stmt).all() - - def get_categorized(self): - with Session(self.engine) as session: - stmt = select(Transaction).where(Transaction.category.has()) - return session.scalars(stmt).all() - - def insert_transactions(self, input: list[Transaction]): - with Session(self.engine) as session: - session.add_all(input) - session.commit() - - def get_banks(self): - with Session(self.engine) as session: - stmt = select(Bank) - return session.scalars(stmt).all() - - def get_nordigen_banks(self): - with Session(self.engine) as session: - stmt = select(Bank).where(Bank.nordigen.has()) - return session.scalars(stmt).all() - @property def engine(self): return self._engine @@ -84,6 +50,20 @@ class DbClient: def commit(self): self.__session.commit() + def expunge_all(self): + self.__session.expunge_all() + + def get(self, type, column=None, values=None): + if column: + if values: + stmt = select(type).where(column.in_(values)) + else: + stmt = select(type).where(column.has()) + else: + stmt = select(type) + + return self.__session.scalars(stmt).all() + def add(self, rows: list): self.__session.add_all(rows) @@ -125,10 +105,6 @@ class DbClient: ) self.__session.execute(stmt) - def uncategorized(self) -> list[Transaction]: - stmt = select(Transaction).where(~Transaction.category.has()) - return self.__session.scalars(stmt).all() - def transactions(self, min: date, max: date, banks: list[str]): stmt = select(Transaction).where( Transaction.date >= min, @@ -137,17 +113,5 @@ class DbClient: ) return self.__session.scalars(stmt).all() - def categories(self) -> list[Category]: - stmt = select(Category) - return self.__session.scalars(stmt).all() - - def tags(self) -> list[Tag]: - stmt = select(Tag) - return self.__session.scalars(stmt).all() - - def banks(self) -> list[Bank]: - stmt = select(Bank) - return self.__session.scalars(stmt).all() - def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/input/json.py b/pfbudget/input/json.py deleted file mode 100644 index e4f81b2..0000000 --- a/pfbudget/input/json.py +++ /dev/null @@ -1,30 +0,0 @@ -import json - -from .input import Input -from pfbudget.common.types import Transactions -from pfbudget.utils import convert, parse_decimal - - -class JsonParser(Input): - def __init__(self, manager, options): - super().__init__(manager) - self.options = options - - def parse(self) -> Transactions: - try: - with open(self.options["json"][0], "r") as f: - return [ - convert( - [ - t["bookingDate"], - t["remittanceInformationUnstructured"], - self.options["bank"][0], - parse_decimal(t["transactionAmount"]["amount"]) - if not self.options["invert"] - else -parse_decimal(t["transactionAmount"]["amount"]), - ], - ) - for t in json.load(f)["transactions"]["booked"] - ] - except KeyError: - print("No json file defined") diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index 4f2482d..d9cef9a 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -6,17 +6,18 @@ from nordigen import NordigenClient from uuid import uuid4 import json import os -import webbrowser -from .input import Input -from pfbudget.common.types import NoBankSelected from pfbudget.db.model import Transaction from pfbudget.utils import convert +from .input import Input + load_dotenv() class NordigenInput(Input): + redirect_url = "https://murta.dev" + def __init__(self): super().__init__() self._client = NordigenClient( @@ -24,23 +25,7 @@ class NordigenInput(Input): secret_id=os.environ.get("SECRET_ID"), ) - self.client.token = self.__token() - - # print(options) - - # if "all" in options and options["all"]: - # self.__banks = self.manager.get_banks() - # elif "id" in options and options["id"]: - # self.__banks = [ - # self.manager.get_bank_by("nordigen_id", b) for b in options["id"] - # ] - # elif "name" in options and options["name"]: - # self.__banks = [ - # self.manager.get_bank_by("name", b) for b in options["name"] - # ] - # else: - # self.__banks = None - + self._client.token = self.__token() self._start = date.min self._end = date.max @@ -96,11 +81,15 @@ class NordigenInput(Input): return token def requisition(self, institution: str, country: str = "PT"): - link, _ = self.__requisition_id(institution, country) - webbrowser.open(link) + id = self._client.institution.get_institution_id_by_name(country, institution) + return self._client.initialize_session( + redirect_uri=self.redirect_url, + institution_id=id, + reference_id=str(uuid4()), + ) - def list(self, country: str): - print(self._client.institution.get_institutions(country)) + def country_banks(self, country: str): + return self._client.institution.get_institutions(country) @property def client(self): @@ -137,16 +126,3 @@ class NordigenInput(Input): token = self._client.generate_token() print(f"New access token: {token}") return token - - def __requisition_id(self, i: str, c: str): - id = self._client.institution.get_institution_id_by_name( - country=c, institution=i - ) - init = self._client.initialize_session( - redirect_uri="https://murta.dev", - institution_id=id, - reference_id=str(uuid4()), - ) - - print(f"{i}({c}) link: {init.link} and requisition ID: {init.requisition_id}") - return (init.link, init.requisition_id) diff --git a/pfbudget/utils/converters.py b/pfbudget/utils/converters.py index 287ea1b..d1d6630 100644 --- a/pfbudget/utils/converters.py +++ b/pfbudget/utils/converters.py @@ -3,7 +3,6 @@ from functools import singledispatch from pfbudget.common.types import TransactionError from pfbudget.db.model import Bank, Transaction -from pfbudget.db.schema import DbBank, DbTransaction from .utils import parse_decimal @@ -13,49 +12,6 @@ def convert(t): pass -# @convert.register -# def _(t: Transaction) -> DbTransaction: -# return DbTransaction( -# t.date, -# t.description, -# t.bank, -# t.value, -# t.category, -# t.original, -# t.additional_comment, -# ) - - -# @convert.register -# def _(db: DbTransaction) -> Transaction: -# try: -# return Transaction(db) -# except TransactionError: -# print(f"{db} is in the wrong format") - - -# @convert.register -# def _(db: DbBank, key: str = "") -> Bank: -# bank = Bank(db.name, db.bic, db.requisition_id, db.invert, db.offset, key=key) -# if not bank.invert: -# bank.invert = False -# if not bank.offset: -# bank.offset = 0 -# return bank - - -# @convert.register -# def _(bank: Bank) -> DbBank: -# bank = DbBank( -# bank.name, bank.bic, "", "", bank.requisition_id, bank.invert, bank.offset -# ) -# if not bank.invert: -# bank.invert = False -# if not bank.offset: -# bank.offset = 0 -# return bank - - @convert.register def _(json: dict, bank: Bank) -> Transaction: i = -1 if bank.nordigen.invert else 1 From c37e7eb37cb7fbd8edd69c3a01ecfd6444da995f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Tue, 10 Jan 2023 21:32:08 +0000 Subject: [PATCH 36/66] Readds manual categorization Also fixes a categorization bug in the Manager, in the DB client method. --- pfbudget/cli/runnable.py | 13 +++----- pfbudget/common/types.py | 1 + pfbudget/core/categorizer.py | 64 +++++++++++++++++++++++++++++++----- pfbudget/core/manager.py | 11 +++++-- pfbudget/db/client.py | 4 +-- 5 files changed, 71 insertions(+), 22 deletions(-) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 6b5fb42..f41187d 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -87,15 +87,10 @@ def argparser() -> argparse.ArgumentParser: parse.add_argument("--bank", nargs=1, type=str) parse.add_argument("--creditcard", nargs=1, type=str) - """ - Categorizing - """ - categorize = subparsers.add_parser( - "categorize", - description="Categorizes the transactions in the selected database", - parents=[universal], - ) - categorize.set_defaults(op=Operation.Categorize) + # Automatic/manual categorization + categorize = subparsers.add_parser("categorize").add_subparsers(required=True) + categorize.add_parser("auto").set_defaults(op=Operation.Categorize) + categorize.add_parser("manual").set_defaults(op=Operation.ManualCategorization) """ Graph diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 71a763f..2f98d41 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -9,6 +9,7 @@ class Operation(Enum): Parse = auto() Download = auto() Categorize = auto() + ManualCategorization = auto() Token = auto() RequisitionId = auto() CategoryAdd = auto() diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index fb7405a..d2918e3 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -10,6 +10,8 @@ from pfbudget.db.model import ( from datetime import timedelta +Transactions = list[Transaction] + class Categorizer: options = {} @@ -17,25 +19,44 @@ class Categorizer: def __init__(self): self.options["null_days"] = 4 - def categorize( + def rules( self, - transactions: list[Transaction], + transactions: Transactions, categories: list[Category], tags: list[Tag], ): """Overarching categorization tool - Receives a list of transactions (by ref) and updates their category + Receives a list of transactions (by ref) and updates their category according + to the rules defined for each category Args: transactions (list[Transaction]): uncategorized transactions + categories (list[Category]): available categories + tags (list[Tag]): currently available tags """ self._nullify(transactions) + self._rule_based_categories(transactions, categories) self._rule_based_tags(transactions, tags) - def _nullify(self, transactions: list[Transaction]): + def manual( + self, + transactions: Transactions, + categories: list[Category], + tags: list[Tag], + ): + """Manual categorization input + + Args: + transactions (list[Transaction]): uncategorized transactions + categories (list[Category]): available categories + tags (list[Tag]): currently available tags + """ + self._manual(transactions) + + def _nullify(self, transactions: Transactions): count = 0 matching = [] for transaction in transactions: @@ -66,7 +87,7 @@ class Categorizer: print(f"Nullified {count} transactions") def _rule_based_categories( - self, transactions: list[Transaction], categories: list[Category] + self, transactions: Transactions, categories: list[Category] ): d = {} for category in [c for c in categories if c.rules]: @@ -81,9 +102,17 @@ class Categorizer: continue # passed all conditions, assign category - transaction.category = TransactionCategory( - category.name, CategorySelector(Selector.rules) - ) + if transaction.category: + if ( + input(f"Overwrite {transaction} with {category}? (y/n)") + == "y" + ): + transaction.category.name = category.name + transaction.category.selector.selector = Selector.rules + else: + transaction.category = TransactionCategory( + category.name, CategorySelector(Selector.rules) + ) if rule in d: d[rule] += 1 @@ -93,7 +122,7 @@ class Categorizer: for k, v in d.items(): print(f"{v}: {k}") - def _rule_based_tags(self, transactions: list[Transaction], tags: list[Tag]): + def _rule_based_tags(self, transactions: Transactions, tags: list[Tag]): d = {} for tag in [t for t in tags if t.rules]: for rule in tag.rules: @@ -119,3 +148,20 @@ class Categorizer: for k, v in d.items(): print(f"{v}: {k}") + def _manual(self, transactions: Transactions): + uncategorized = [t for t in transactions if not t.category] + print(f"{len(uncategorized)} transactions left to categorize") + + for transaction in uncategorized: + while True: + category = input(f"{transaction} category: ") + if category == "quit": + return + if not category: + print("{category} doesn't exist") + continue + transaction.category = TransactionCategory( + category, CategorySelector(Selector.manual) + ) + + break diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 60a5259..9cdc044 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -71,10 +71,17 @@ class Manager: case Operation.Categorize: with self.db.session() as session: - uncategorized = session.get(Transaction, ~Transaction.category) + uncategorized = session.get(Transaction, ~Transaction.category.has()) categories = session.get(Category) tags = session.get(Tag) - Categorizer().categorize(uncategorized, categories, tags) + Categorizer().rules(uncategorized, categories, tags) + + case Operation.ManualCategorization: + with self.db.session() as session: + uncategorized = session.get(Transaction, ~Transaction.category.has()) + categories = session.get(Category) + tags = session.get(Tag) + Categorizer().manual(uncategorized, categories, tags) case Operation.BankMod: with self.db.session() as session: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 538e82a..29940d8 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -54,11 +54,11 @@ class DbClient: self.__session.expunge_all() def get(self, type, column=None, values=None): - if column: + if column is not None: if values: stmt = select(type).where(column.in_(values)) else: - stmt = select(type).where(column.has()) + stmt = select(type).where(column) else: stmt = select(type) From 0d287624c4669c0045a1909af7f1f50a5c7ae0e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Tue, 10 Jan 2023 21:35:43 +0000 Subject: [PATCH 37/66] Load the default DB from the .env file --- pfbudget/cli/runnable.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index f41187d..b367ad2 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -1,6 +1,8 @@ +from dotenv import load_dotenv import argparse import datetime as dt import decimal +import os import re from pfbudget.common.types import Operation @@ -10,8 +12,9 @@ import pfbudget.reporting.graph import pfbudget.reporting.report import pfbudget.utils +load_dotenv() -DEFAULT_DB = "data.db" +DEFAULT_DB = os.environ.get("DEFAULT_DB") class PfBudgetInitialized(Exception): From 478bd251904dcc35f4d32c01ec8c9adaf2d8226c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Tue, 10 Jan 2023 23:42:37 +0000 Subject: [PATCH 38/66] Subclass the Transaction with multiple children Each children is essentually a type of transaction. We currently have: - bank transactions - money transactions - split transactions The table inheritance is implemented as a single table, with a polymorphic type and Null columns. Adds a IsSplit interface, which will later be used for the category views, so as to not repeat transactions. --- alembic/versions/37d80de801a7_inheritance.py | 74 ++++++++++++++++++++ pfbudget/db/model.py | 27 ++++++- pfbudget/input/nordigen.py | 4 +- pfbudget/utils/converters.py | 8 +-- 4 files changed, 105 insertions(+), 8 deletions(-) create mode 100644 alembic/versions/37d80de801a7_inheritance.py diff --git a/alembic/versions/37d80de801a7_inheritance.py b/alembic/versions/37d80de801a7_inheritance.py new file mode 100644 index 0000000..f9d7ee4 --- /dev/null +++ b/alembic/versions/37d80de801a7_inheritance.py @@ -0,0 +1,74 @@ +"""Inheritance + +Revision ID: 37d80de801a7 +Revises: 8cc9870b0d74 +Create Date: 2023-01-10 22:41:03.540108+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "37d80de801a7" +down_revision = "8cc9870b0d74" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "originals", + sa.Column("type", sa.String(), nullable=False), + schema="transactions", + ) + op.add_column( + "originals", + sa.Column("split", sa.Boolean(), nullable=True), + schema="transactions", + ) + op.add_column( + "originals", + sa.Column("original", sa.BigInteger(), nullable=True), + schema="transactions", + ) + op.alter_column( + "originals", + "bank", + existing_type=sa.TEXT(), + nullable=True, + schema="transactions", + ) + op.create_foreign_key( + op.f("fk_originals_original_originals"), + "originals", + "originals", + ["original"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_originals_original_originals"), + "originals", + schema="transactions", + type_="foreignkey", + ) + op.alter_column( + "originals", + "bank", + existing_type=sa.TEXT(), + nullable=False, + schema="transactions", + ) + op.drop_column("originals", "original", schema="transactions") + op.drop_column("originals", "split", schema="transactions") + op.drop_column("originals", "type", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index b961d6d..adfdf2b 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -74,14 +74,17 @@ class Transaction(Base): id: Mapped[idpk] = mapped_column(init=False) date: Mapped[dt.date] description: Mapped[Optional[str]] - bank: Mapped[bankfk] amount: Mapped[money] + type: Mapped[str] = mapped_column(init=False) + category: Mapped[Optional[TransactionCategory]] = relationship(init=False) note: Mapped[Optional[Note]] = relationship(init=False) tags: Mapped[Optional[set[TransactionTag]]] = relationship(init=False) - def __lt__(self, other): + __mapper_args__ = {"polymorphic_on": "type", "polymorphic_identity": "transaction"} + + def __lt__(self, other: Transaction): return self.date < other.date @@ -90,6 +93,26 @@ idfk = Annotated[ ] +class IsSplit: + split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) + + +class BankTransaction(IsSplit, Transaction): + bank: Mapped[bankfk] = mapped_column(nullable=True) + + __mapper_args__ = {"polymorphic_identity": "bank", "polymorphic_load": "inline"} + + +class MoneyTransaction(IsSplit, Transaction): + __mapper_args__ = {"polymorphic_identity": "money"} + + +class SplitTransaction(Transaction): + original: Mapped[idfk] = mapped_column(nullable=True) + + __mapper_args__ = {"polymorphic_identity": "split", "polymorphic_load": "inline"} + + class CategoryGroup(Base): __tablename__ = "categories_groups" diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index d9cef9a..14e166e 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -7,7 +7,7 @@ from uuid import uuid4 import json import os -from pfbudget.db.model import Transaction +from pfbudget.db.model import BankTransaction from pfbudget.utils import convert from .input import Input @@ -29,7 +29,7 @@ class NordigenInput(Input): self._start = date.min self._end = date.max - def parse(self) -> list[Transaction]: + def parse(self) -> list[BankTransaction]: transactions = [] assert len(self._banks) > 0 diff --git a/pfbudget/utils/converters.py b/pfbudget/utils/converters.py index d1d6630..62461c1 100644 --- a/pfbudget/utils/converters.py +++ b/pfbudget/utils/converters.py @@ -1,8 +1,8 @@ -from datetime import date, timedelta +from datetime import date from functools import singledispatch from pfbudget.common.types import TransactionError -from pfbudget.db.model import Bank, Transaction +from pfbudget.db.model import Bank, BankTransaction from .utils import parse_decimal @@ -13,10 +13,10 @@ def convert(t): @convert.register -def _(json: dict, bank: Bank) -> Transaction: +def _(json: dict, bank: Bank) -> BankTransaction: i = -1 if bank.nordigen.invert else 1 try: - transaction = Transaction( + transaction = BankTransaction( date=date.fromisoformat(json["bookingDate"]), description=json["remittanceInformationUnstructured"], bank=bank.name, From c42a399d3d08e49620234aaf7025909568615ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Tue, 10 Jan 2023 23:45:09 +0000 Subject: [PATCH 39/66] Adds the import operation and a timer to the categorization. We can now import transactions from a csv file, and later automatically categorize them all. --- pfbudget/__main__.py | 6 ++++++ pfbudget/cli/runnable.py | 4 ++++ pfbudget/common/types.py | 1 + pfbudget/core/categorizer.py | 9 ++++++++- pfbudget/core/manager.py | 22 ++++++++++++++++++++-- pfbudget/output/csv.py | 22 ++++++++++++++++++++-- 6 files changed, 59 insertions(+), 5 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index fa5eb31..c732656 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -242,4 +242,10 @@ if __name__ == "__main__": params.append(args["banks"]) params.append(args["file"][0]) + case pfbudget.Operation.Import: + keys = {"file"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + + params = args["file"] + pfbudget.Manager(db, verbosity).action(op, params) diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index b367ad2..5524c3f 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -83,6 +83,10 @@ def argparser() -> argparse.ArgumentParser: export_banks.add_argument("--all", action="store_true") export_banks.add_argument("--banks", nargs="+", type=str) + pimport = subparsers.add_parser("import") + pimport.set_defaults(op=Operation.Import) + pimport.add_argument("file", nargs=1, type=str) + # Parse from .csv parse = subparsers.add_parser("parse") parse.set_defaults(op=Operation.Parse) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 2f98d41..d5eba7d 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -36,6 +36,7 @@ class Operation(Enum): NordigenDel = auto() NordigenCountryBanks = auto() Export = auto() + Import = auto() class TransactionError(Exception): diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index d2918e3..0cfc5af 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -8,6 +8,7 @@ from pfbudget.db.model import ( TransactionTag, ) +from codetiming import Timer from datetime import timedelta Transactions = list[Transaction] @@ -56,6 +57,7 @@ class Categorizer: """ self._manual(transactions) + @Timer(name="nullify") def _nullify(self, transactions: Transactions): count = 0 matching = [] @@ -86,6 +88,7 @@ class Categorizer: print(f"Nullified {count} transactions") + @Timer(name="categoryrules") def _rule_based_categories( self, transactions: Transactions, categories: list[Category] ): @@ -102,7 +105,10 @@ class Categorizer: continue # passed all conditions, assign category - if transaction.category: + if ( + transaction.category + and transaction.category.name == category.name + ): if ( input(f"Overwrite {transaction} with {category}? (y/n)") == "y" @@ -122,6 +128,7 @@ class Categorizer: for k, v in d.items(): print(f"{v}: {k}") + @Timer(name="tagrules") def _rule_based_tags(self, transactions: Transactions, tags: list[Tag]): d = {} for tag in [t for t in tags if t.rules]: diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 9cdc044..0801e5e 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -71,14 +71,18 @@ class Manager: case Operation.Categorize: with self.db.session() as session: - uncategorized = session.get(Transaction, ~Transaction.category.has()) + uncategorized = session.get( + Transaction, ~Transaction.category.has() + ) categories = session.get(Category) tags = session.get(Tag) Categorizer().rules(uncategorized, categories, tags) case Operation.ManualCategorization: with self.db.session() as session: - uncategorized = session.get(Transaction, ~Transaction.category.has()) + uncategorized = session.get( + Transaction, ~Transaction.category.has() + ) categories = session.get(Category) tags = session.get(Tag) Categorizer().manual(uncategorized, categories, tags) @@ -178,6 +182,20 @@ class Manager: csvwriter: Output = CSV(params[-1]) csvwriter.report(transactions) + case Operation.Import: + csvwriter: Output = CSV(params[0]) # Output is strange here + transactions = csvwriter.load() + + if ( + len(transactions) > 0 + and input( + f"{transactions[:5]}\nDoes the import seem correct? (y/n)" + ) + == "y" + ): + with self.db.session() as session: + session.add(transactions) + # def init(self): # client = DatabaseClient(self.__db) # client.init() diff --git a/pfbudget/output/csv.py b/pfbudget/output/csv.py index 55135e8..77b1191 100644 --- a/pfbudget/output/csv.py +++ b/pfbudget/output/csv.py @@ -1,6 +1,10 @@ -from csv import writer +from csv import DictReader, writer -from pfbudget.db.model import Transaction +from pfbudget.db.model import ( + BankTransaction, + MoneyTransaction, + Transaction, +) from .output import Output @@ -9,6 +13,20 @@ class CSV(Output): def __init__(self, filename: str): self.fn = filename + def load(self) -> list[Transaction]: + with open(self.fn, "r", newline="") as f: + r = DictReader(f) + return [ + BankTransaction( + row["date"], row["description"], row["amount"], False, row["bank"] + ) + if row["bank"] + else MoneyTransaction( + row["date"], row["description"], False, row["amount"] + ) + for row in r + ] + def report(self, transactions: list[Transaction]): with open(self.fn, "w", newline="") as f: w = writer(f, delimiter="\t") From 1cce7d421e8a18f2614b06e7347df94275e0639f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Tue, 10 Jan 2023 23:49:06 +0000 Subject: [PATCH 40/66] Update requirements.txt with the SQL ORM --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 848a17f..5be029b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ +codetiming==1.4.0 matplotlib==3.6.1 nordigen==1.3.0 python-dateutil==2.8.2 python-dotenv==0.21.0 PyYAML==6.0 +SQLAlchemy==2.0.0rc2 From 6110858d4818b1a0873922aaaeb5509076334bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 15 Jan 2023 23:06:20 +0000 Subject: [PATCH 41/66] Extend export/import to rules Removes additional bank/all options from the transactions export command line. Deletes the brief lived CSV class. This patch start using pickle for simple export/import, other options can be added later. An issue with the .csv is the lack of a Null field. Moves logic to Manager, it is simple enough. --- pfbudget/__main__.py | 12 +----- pfbudget/cli/runnable.py | 27 +++++++++--- pfbudget/common/types.py | 4 ++ pfbudget/core/manager.py | 76 +++++++++++++++++++++++++++------ pfbudget/db/model.py | 85 ++++++++++++++++++++++++++++++------- pfbudget/output/__init__.py | 1 - pfbudget/output/csv.py | 35 --------------- pfbudget/output/output.py | 9 ---- 8 files changed, 159 insertions(+), 90 deletions(-) delete mode 100644 pfbudget/output/__init__.py delete mode 100644 pfbudget/output/csv.py delete mode 100644 pfbudget/output/output.py diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index c732656..a9c0b2b 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -232,17 +232,7 @@ if __name__ == "__main__": pfbudget.types.Link(args["original"][0], link) for link in args["links"] ] - case pfbudget.Operation.Export: - keys = {"interval", "start", "end", "year", "all", "banks", "file"} - assert args.keys() >= keys, f"missing {args.keys() - keys}" - - start, end = pfbudget.parse_args_period(args) - params = [start, end] - if not args["all"]: - params.append(args["banks"]) - params.append(args["file"][0]) - - case pfbudget.Operation.Import: + case pfbudget.Operation.Export | pfbudget.Operation.Import | pfbudget.Operation.ExportCategoryRules | pfbudget.Operation.ImportCategoryRules | pfbudget.Operation.ExportTagRules | pfbudget.Operation.ImportTagRules: keys = {"file"} assert args.keys() >= keys, f"missing {args.keys() - keys}" diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 5524c3f..e8bbc4b 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -76,12 +76,9 @@ def argparser() -> argparse.ArgumentParser: p_init.set_defaults(command=Operation.Init) # Exports transactions to .csv file - export = subparsers.add_parser("export", parents=[period]) + export = subparsers.add_parser("export") export.set_defaults(op=Operation.Export) - export.add_argument("file", nargs=1, type=str) - export_banks = export.add_mutually_exclusive_group() - export_banks.add_argument("--all", action="store_true") - export_banks.add_argument("--banks", nargs="+", type=str) + export_args(export) pimport = subparsers.add_parser("import") pimport.set_defaults(op=Operation.Import) @@ -321,6 +318,14 @@ def category_rule(parser: argparse.ArgumentParser): rules(modify) modify.add_argument("--remove", nargs="*", default=[], type=str) + export = commands.add_parser("export") + export.set_defaults(op=Operation.ExportCategoryRules) + export_args(export) + + pimport = commands.add_parser("import") + pimport.set_defaults(op=Operation.ImportCategoryRules) + export_args(pimport) + def tags(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) @@ -355,6 +360,14 @@ def tag_rule(parser: argparse.ArgumentParser): modify.add_argument("--tag", nargs=1, type=str) rules(modify) + export = commands.add_parser("export") + export.set_defaults(op=Operation.ExportTagRules) + export_args(export) + + pimport = commands.add_parser("import") + pimport.set_defaults(op=Operation.ImportTagRules) + export_args(pimport) + def rules(parser: argparse.ArgumentParser): parser.add_argument("--date", nargs=1, type=dt.date.fromisoformat) @@ -377,3 +390,7 @@ def link(parser: argparse.ArgumentParser): dismantle.set_defaults(op=Operation.Dismantle) dismantle.add_argument("original", nargs=1, type=int) dismantle.add_argument("links", nargs="+", type=int) + + +def export_args(parser: argparse.ArgumentParser): + parser.add_argument("file", nargs=1, type=str) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index d5eba7d..a77cc52 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -37,6 +37,10 @@ class Operation(Enum): NordigenCountryBanks = auto() Export = auto() Import = auto() + ExportCategoryRules = auto() + ImportCategoryRules = auto() + ExportTagRules = auto() + ImportTagRules = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 0801e5e..0c515b5 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,4 +1,5 @@ from pathlib import Path +import pickle import webbrowser from pfbudget.common.types import Operation @@ -6,19 +7,21 @@ from pfbudget.core.categorizer import Categorizer from pfbudget.db.client import DbClient from pfbudget.db.model import ( Bank, + BankTransaction, Category, CategoryGroup, CategoryRule, + CategorySelector, + MoneyTransaction, Nordigen, Rule, Tag, TagRule, Transaction, + TransactionCategory, ) from pfbudget.input.nordigen import NordigenInput from pfbudget.input.parsers import parse_data -from pfbudget.output.csv import CSV -from pfbudget.output.output import Output class Manager: @@ -171,20 +174,37 @@ class Manager: case Operation.Export: with self.db.session() as session: - if len(params) < 4: - banks = [bank.name for bank in session.get(Bank)] - transactions = session.transactions(params[0], params[1], banks) - else: - transactions = session.transactions( - params[0], params[1], params[2] - ) - - csvwriter: Output = CSV(params[-1]) - csvwriter.report(transactions) + self.dump(params[0], session.get(Transaction)) case Operation.Import: - csvwriter: Output = CSV(params[0]) # Output is strange here - transactions = csvwriter.load() + transactions = [] + for row in self.load(params[0]): + match row["type"]: + case "bank": + transaction = BankTransaction( + row["date"], + row["description"], + row["amount"], + row["bank"], + False, + ) + + case "money": + transaction = MoneyTransaction( + row["date"], row["description"], row["amount"], False + ) + + # TODO case "split" how to match to original transaction?? also save ids? + case _: + continue + + if category := row.pop("category", None): + transaction.category = TransactionCategory( + category["name"], + CategorySelector(category["selector"]["selector"]), + ) + + transactions.append(transaction) if ( len(transactions) > 0 @@ -196,6 +216,26 @@ class Manager: with self.db.session() as session: session.add(transactions) + case Operation.ExportCategoryRules: + with self.db.session() as session: + self.dump(params[0], session.get(CategoryRule)) + + case Operation.ImportCategoryRules: + rules = [CategoryRule(**row) for row in self.load(params[0])] + + with self.db.session() as session: + session.add(rules) + + case Operation.ExportTagRules: + with self.db.session() as session: + self.dump(params[0], session.get(TagRule)) + + case Operation.ImportTagRules: + rules = [TagRule(**row) for row in self.load(params[0])] + + with self.db.session() as session: + session.add(rules) + # def init(self): # client = DatabaseClient(self.__db) # client.init() @@ -224,6 +264,14 @@ class Manager: # bank = client.get_bank(key, value) # return convert(bank) + def dump(self, fn, sequence): + with open(fn, "wb") as f: + pickle.dump([e.format for e in sequence], f) + + def load(self, fn): + with open(fn, "rb") as f: + return pickle.load(f) + @property def db(self) -> DbClient: return DbClient(self._db, self._verbosity > 2) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index adfdf2b..7e129b5 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -52,6 +52,12 @@ accounttype = Annotated[ ] +class Export: + @property + def format(self) -> dict[str, Any]: + raise NotImplementedError + + class Bank(Base): __tablename__ = "banks" @@ -68,7 +74,7 @@ idpk = Annotated[int, mapped_column(BigInteger, primary_key=True, autoincrement= money = Annotated[Decimal, mapped_column(Numeric(16, 2))] -class Transaction(Base): +class Transaction(Base, Export): __tablename__ = "originals" id: Mapped[idpk] = mapped_column(init=False) @@ -84,6 +90,18 @@ class Transaction(Base): __mapper_args__ = {"polymorphic_on": "type", "polymorphic_identity": "transaction"} + @property + def format(self) -> dict[str, Any]: + return dict( + date=self.date, + description=self.description, + amount=self.amount, + type=self.type, + category=self.category.format if self.category else None, + # TODO note + tags=[tag.format for tag in self.tags] if self.tags else None, + ) + def __lt__(self, other: Transaction): return self.date < other.date @@ -93,17 +111,20 @@ idfk = Annotated[ ] -class IsSplit: - split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) - - -class BankTransaction(IsSplit, Transaction): +class BankTransaction(Transaction): bank: Mapped[bankfk] = mapped_column(nullable=True) + split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) __mapper_args__ = {"polymorphic_identity": "bank", "polymorphic_load": "inline"} + @property + def format(self) -> dict[str, Any]: + return super().format | dict(bank=self.bank) + + +class MoneyTransaction(Transaction): + split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) -class MoneyTransaction(IsSplit, Transaction): __mapper_args__ = {"polymorphic_identity": "money"} @@ -112,6 +133,10 @@ class SplitTransaction(Transaction): __mapper_args__ = {"polymorphic_identity": "split", "polymorphic_load": "inline"} + @property + def format(self) -> dict[str, Any]: + return super().format | dict(original=self.original) + class CategoryGroup(Base): __tablename__ = "categories_groups" @@ -144,16 +169,19 @@ catfk = Annotated[ ] -class TransactionCategory(Base): +class TransactionCategory(Base, Export): __tablename__ = "categorized" id: Mapped[idfk] = mapped_column(primary_key=True, init=False) name: Mapped[catfk] - selector: Mapped[CategorySelector] = relationship(cascade="all, delete-orphan") + selector: Mapped[CategorySelector] = relationship( + cascade="all, delete-orphan", lazy="joined" + ) - def __repr__(self) -> str: - return f"Category({self.name})" + @property + def format(self): + return dict(name=self.name, selector=self.selector.format) class Note(Base): @@ -182,12 +210,16 @@ class Tag(Base): ) -class TransactionTag(Base): +class TransactionTag(Base, Export): __tablename__ = "tags" id: Mapped[idfk] = mapped_column(primary_key=True, init=False) tag: Mapped[str] = mapped_column(ForeignKey(Tag.name), primary_key=True) + @property + def format(self): + return dict(tag=self.tag) + def __hash__(self): return hash(self.id) @@ -207,7 +239,7 @@ categoryselector = Annotated[ ] -class CategorySelector(Base): +class CategorySelector(Base, Export): __tablename__ = "categories_selector" id: Mapped[int] = mapped_column( @@ -218,6 +250,10 @@ class CategorySelector(Base): ) selector: Mapped[categoryselector] + @property + def format(self): + return dict(selector=self.selector) + class Period(enum.Enum): daily = "daily" @@ -247,7 +283,7 @@ class Link(Base): link: Mapped[idfk] = mapped_column(primary_key=True) -class Rule: +class Rule(Export): date: Mapped[Optional[dt.date]] description: Mapped[Optional[str]] regex: Mapped[Optional[str]] @@ -255,7 +291,7 @@ class Rule: min: Mapped[Optional[money]] max: Mapped[Optional[money]] - def matches(self, transaction: Transaction) -> bool: + def matches(self, transaction: BankTransaction) -> bool: if ( (self.date and self.date < transaction.date) or ( @@ -277,6 +313,17 @@ class Rule: return False return True + @property + def format(self) -> dict[str, Any]: + return dict( + date=self.date, + description=self.description, + regex=self.regex, + bank=self.bank, + min=self.min, + max=self.max, + ) + class CategoryRule(Base, Rule): __tablename__ = "categories_rules" @@ -284,6 +331,10 @@ class CategoryRule(Base, Rule): id: Mapped[idpk] = mapped_column(init=False) name: Mapped[catfk] + @property + def format(self) -> dict[str, Any]: + return super().format | dict(name=self.name) + def __hash__(self): return hash(self.id) @@ -294,5 +345,9 @@ class TagRule(Base, Rule): id: Mapped[idpk] = mapped_column(init=False) tag: Mapped[str] = mapped_column(ForeignKey(Tag.name, ondelete="CASCADE")) + @property + def format(self) -> dict[str, Any]: + return super().format | dict(tag=self.tag) + def __hash__(self): return hash(self.id) diff --git a/pfbudget/output/__init__.py b/pfbudget/output/__init__.py deleted file mode 100644 index 376b17c..0000000 --- a/pfbudget/output/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__all__ = ["csv", "output"] diff --git a/pfbudget/output/csv.py b/pfbudget/output/csv.py deleted file mode 100644 index 77b1191..0000000 --- a/pfbudget/output/csv.py +++ /dev/null @@ -1,35 +0,0 @@ -from csv import DictReader, writer - -from pfbudget.db.model import ( - BankTransaction, - MoneyTransaction, - Transaction, -) - -from .output import Output - - -class CSV(Output): - def __init__(self, filename: str): - self.fn = filename - - def load(self) -> list[Transaction]: - with open(self.fn, "r", newline="") as f: - r = DictReader(f) - return [ - BankTransaction( - row["date"], row["description"], row["amount"], False, row["bank"] - ) - if row["bank"] - else MoneyTransaction( - row["date"], row["description"], False, row["amount"] - ) - for row in r - ] - - def report(self, transactions: list[Transaction]): - with open(self.fn, "w", newline="") as f: - w = writer(f, delimiter="\t") - w.writerows( - [(t.date, t.description, t.amount, t.bank) for t in transactions] - ) diff --git a/pfbudget/output/output.py b/pfbudget/output/output.py deleted file mode 100644 index bf7e918..0000000 --- a/pfbudget/output/output.py +++ /dev/null @@ -1,9 +0,0 @@ -from abc import ABC, abstractmethod - -from pfbudget.db.model import Transaction - - -class Output(ABC): - @abstractmethod - def report(self, transactions: list[Transaction]): - raise NotImplementedError From d4b5f1f11a355572c9d56b104fb8327cbff464b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 22 Jan 2023 20:22:46 +0000 Subject: [PATCH 42/66] Rule inheritance Both rules, categorries and tags, now derive from the rule base type. This clears up some type definitions. --- .../versions/6b293f78cc97_rule_inheritance.py | 152 ++++++++++++++++++ pfbudget/core/manager.py | 18 ++- pfbudget/db/model.py | 39 ++++- 3 files changed, 200 insertions(+), 9 deletions(-) create mode 100644 alembic/versions/6b293f78cc97_rule_inheritance.py diff --git a/alembic/versions/6b293f78cc97_rule_inheritance.py b/alembic/versions/6b293f78cc97_rule_inheritance.py new file mode 100644 index 0000000..e1e259d --- /dev/null +++ b/alembic/versions/6b293f78cc97_rule_inheritance.py @@ -0,0 +1,152 @@ +"""Rule inheritance + +Revision ID: 6b293f78cc97 +Revises: 37d80de801a7 +Create Date: 2023-01-22 20:05:32.887092+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "6b293f78cc97" +down_revision = "37d80de801a7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "rules", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("date", sa.Date(), nullable=True), + sa.Column("description", sa.String(), nullable=True), + sa.Column("regex", sa.String(), nullable=True), + sa.Column("bank", sa.String(), nullable=True), + sa.Column("min", sa.Numeric(precision=16, scale=2), nullable=True), + sa.Column("max", sa.Numeric(precision=16, scale=2), nullable=True), + sa.Column("type", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_rules")), + schema="transactions", + ) + op.create_foreign_key( + op.f("fk_categories_rules_id_rules"), + "categories_rules", + "rules", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + op.drop_column("categories_rules", "bank", schema="transactions") + op.drop_column("categories_rules", "min", schema="transactions") + op.drop_column("categories_rules", "date", schema="transactions") + op.drop_column("categories_rules", "regex", schema="transactions") + op.drop_column("categories_rules", "description", schema="transactions") + op.drop_column("categories_rules", "max", schema="transactions") + op.create_foreign_key( + op.f("fk_tag_rules_id_rules"), + "tag_rules", + "rules", + ["id"], + ["id"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + op.drop_column("tag_rules", "bank", schema="transactions") + op.drop_column("tag_rules", "min", schema="transactions") + op.drop_column("tag_rules", "date", schema="transactions") + op.drop_column("tag_rules", "regex", schema="transactions") + op.drop_column("tag_rules", "description", schema="transactions") + op.drop_column("tag_rules", "max", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "tag_rules", + sa.Column( + "max", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + schema="transactions", + ) + op.add_column( + "tag_rules", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "tag_rules", + sa.Column("regex", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "tag_rules", + sa.Column("date", sa.DATE(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "tag_rules", + sa.Column( + "min", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + schema="transactions", + ) + op.add_column( + "tag_rules", + sa.Column("bank", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.drop_constraint( + op.f("fk_tag_rules_id_rules"), + "tag_rules", + schema="transactions", + type_="foreignkey", + ) + op.add_column( + "categories_rules", + sa.Column( + "max", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("regex", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("date", sa.DATE(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column( + "min", sa.NUMERIC(precision=16, scale=2), autoincrement=False, nullable=True + ), + schema="transactions", + ) + op.add_column( + "categories_rules", + sa.Column("bank", sa.VARCHAR(), autoincrement=False, nullable=True), + schema="transactions", + ) + op.drop_constraint( + op.f("fk_categories_rules_id_rules"), + "categories_rules", + schema="transactions", + type_="foreignkey", + ) + op.drop_table("rules", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 0c515b5..20e9c68 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -223,8 +223,13 @@ class Manager: case Operation.ImportCategoryRules: rules = [CategoryRule(**row) for row in self.load(params[0])] - with self.db.session() as session: - session.add(rules) + if ( + len(rules) > 0 + and input(f"{rules[:5]}\nDoes the import seem correct? (y/n)") + == "y" + ): + with self.db.session() as session: + session.add(rules) case Operation.ExportTagRules: with self.db.session() as session: @@ -233,8 +238,13 @@ class Manager: case Operation.ImportTagRules: rules = [TagRule(**row) for row in self.load(params[0])] - with self.db.session() as session: - session.add(rules) + if ( + len(rules) > 0 + and input(f"{rules[:5]}\nDoes the import seem correct? (y/n)") + == "y" + ): + with self.db.session() as session: + session.add(rules) # def init(self): # client = DatabaseClient(self.__db) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 7e129b5..7ed3dec 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -283,7 +283,10 @@ class Link(Base): link: Mapped[idfk] = mapped_column(primary_key=True) -class Rule(Export): +class Rule(Base, Export): + __tablename__ = "rules" + + id: Mapped[idpk] = mapped_column(init=False) date: Mapped[Optional[dt.date]] description: Mapped[Optional[str]] regex: Mapped[Optional[str]] @@ -291,6 +294,13 @@ class Rule(Export): min: Mapped[Optional[money]] max: Mapped[Optional[money]] + type: Mapped[str] = mapped_column(init=False) + + __mapper_args__ = { + "polymorphic_identity": "rule", + "polymorphic_on": "type", + } + def matches(self, transaction: BankTransaction) -> bool: if ( (self.date and self.date < transaction.date) @@ -322,15 +332,25 @@ class Rule(Export): bank=self.bank, min=self.min, max=self.max, + type=self.type, ) -class CategoryRule(Base, Rule): +class CategoryRule(Rule): __tablename__ = "categories_rules" - id: Mapped[idpk] = mapped_column(init=False) + id: Mapped[int] = mapped_column( + BigInteger, + ForeignKey(Rule.id, ondelete="CASCADE"), + primary_key=True, + init=False, + ) name: Mapped[catfk] + __mapper_args__ = { + "polymorphic_identity": "category_rule", + } + @property def format(self) -> dict[str, Any]: return super().format | dict(name=self.name) @@ -339,12 +359,21 @@ class CategoryRule(Base, Rule): return hash(self.id) -class TagRule(Base, Rule): +class TagRule(Rule): __tablename__ = "tag_rules" - id: Mapped[idpk] = mapped_column(init=False) + id: Mapped[int] = mapped_column( + BigInteger, + ForeignKey(Rule.id, ondelete="CASCADE"), + primary_key=True, + init=False, + ) tag: Mapped[str] = mapped_column(ForeignKey(Tag.name, ondelete="CASCADE")) + __mapper_args__ = { + "polymorphic_identity": "tag_rule", + } + @property def format(self) -> dict[str, Any]: return super().format | dict(tag=self.tag) From fd6793b4f426a52b40fde385243a5ddfe6878ad8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 22 Jan 2023 20:42:14 +0000 Subject: [PATCH 43/66] Turned on type checking and as a result, had to fix a LOT of minor potential future issue. It also reorders and clears unused imports. When exporting transactions, it will sort by date. --- pfbudget/__init__.py | 2 +- pfbudget/__main__.py | 28 +++++++------- pfbudget/cli/runnable.py | 50 ++++++++++-------------- pfbudget/core/categorizer.py | 75 +++++++++++++++++------------------- pfbudget/core/manager.py | 56 +++++++++------------------ pfbudget/db/client.py | 29 ++++---------- pfbudget/db/model.py | 27 ++++++------- pfbudget/input/nordigen.py | 53 ++++++++++++++----------- pfbudget/input/parsers.py | 5 ++- pfbudget/utils/converters.py | 22 ++++++----- 10 files changed, 155 insertions(+), 192 deletions(-) diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index 75c5718..ed82db6 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -9,4 +9,4 @@ from pfbudget.cli.runnable import argparser from pfbudget.input.parsers import parse_data from pfbudget.utils.utils import parse_args_period -import pfbudget.db.model as types +import pfbudget.db.model as t diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index a9c0b2b..89c2780 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -14,7 +14,7 @@ if __name__ == "__main__": assert "verbose" in args, "No verbose level specified" verbosity = args.pop("verbose") - params = None + params = [] match (op): case pfbudget.Operation.Parse: keys = {"path", "bank", "creditcard"} @@ -45,7 +45,7 @@ if __name__ == "__main__": assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.Bank( + pfbudget.t.Bank( args["bank"][0], args["bic"][0], args["type"][0], @@ -73,7 +73,7 @@ if __name__ == "__main__": assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.Nordigen( + pfbudget.t.Nordigen( args["bank"][0], args["bank_id"][0] if args["bank_id"] else None, args["requisition_id"][0] if args["requisition_id"] else None, @@ -110,27 +110,27 @@ if __name__ == "__main__": assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.Category(cat, args["group"]) for cat in args["category"] + pfbudget.t.Category(cat, args["group"]) for cat in args["category"] ] case pfbudget.Operation.CategoryUpdate: keys = {"category", "group"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [pfbudget.types.Category(cat) for cat in args["category"]] + params = [pfbudget.t.Category(cat) for cat in args["category"]] params.append(args["group"]) case pfbudget.Operation.CategoryRemove: assert "category" in args, "argparser ill defined" - params = [pfbudget.types.Category(cat) for cat in args["category"]] + params = [pfbudget.t.Category(cat) for cat in args["category"]] case pfbudget.Operation.CategorySchedule: keys = {"category", "period", "frequency"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.CategorySchedule( - cat, True, args["period"][0], args["frequency"][0] + pfbudget.t.CategorySchedule( + cat, args["period"][0], args["frequency"][0], None ) for cat in args["category"] ] @@ -140,7 +140,7 @@ if __name__ == "__main__": assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.CategoryRule( + pfbudget.t.CategoryRule( args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, @@ -184,14 +184,14 @@ if __name__ == "__main__": keys = {"tag"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [pfbudget.types.Tag(tag) for tag in args["tag"]] + params = [pfbudget.t.Tag(tag) for tag in args["tag"]] case pfbudget.Operation.TagRuleAdd: keys = {"tag", "date", "description", "bank", "min", "max"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.TagRule( + pfbudget.t.TagRule( args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, @@ -218,18 +218,18 @@ if __name__ == "__main__": case pfbudget.Operation.GroupAdd: assert "group" in args, "argparser ill defined" - params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] + params = [pfbudget.t.CategoryGroup(group) for group in args["group"]] case pfbudget.Operation.GroupRemove: assert "group" in args, "argparser ill defined" - params = [pfbudget.types.CategoryGroup(group) for group in args["group"]] + params = [pfbudget.t.CategoryGroup(group) for group in args["group"]] case pfbudget.Operation.Forge | pfbudget.Operation.Dismantle: keys = {"original", "links"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.types.Link(args["original"][0], link) for link in args["links"] + pfbudget.t.Link(args["original"][0], link) for link in args["links"] ] case pfbudget.Operation.Export | pfbudget.Operation.Import | pfbudget.Operation.ExportCategoryRules | pfbudget.Operation.ImportCategoryRules | pfbudget.Operation.ExportTagRules | pfbudget.Operation.ImportTagRules: diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index e8bbc4b..d368f7e 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -1,12 +1,13 @@ -from dotenv import load_dotenv import argparse import datetime as dt import decimal +from dotenv import load_dotenv import os import re from pfbudget.common.types import Operation from pfbudget.db.model import AccountType, Period + from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph import pfbudget.reporting.report @@ -38,42 +39,38 @@ def argparser() -> argparse.ArgumentParser: help="select current database", default=DEFAULT_DB, ) + universal.add_argument("-v", "--verbose", action="count", default=0) - period = argparse.ArgumentParser(add_help=False).add_mutually_exclusive_group() - period.add_argument( + period = argparse.ArgumentParser(add_help=False) + period_group = period.add_mutually_exclusive_group() + period_group.add_argument( "--interval", type=str, nargs=2, help="graph interval", metavar=("START", "END") ) - period.add_argument("--start", type=str, nargs=1, help="graph start date") - period.add_argument("--end", type=str, nargs=1, help="graph end date") - period.add_argument("--year", type=str, nargs=1, help="graph year") + period_group.add_argument("--start", type=str, nargs=1, help="graph start date") + period_group.add_argument("--end", type=str, nargs=1, help="graph end date") + period_group.add_argument("--year", type=str, nargs=1, help="graph year") parser = argparse.ArgumentParser( description="does cool finance stuff", parents=[universal], formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) - parser.add_argument( - "--version", - action="version", - version=re.search( - r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', - open("pfbudget/__init__.py").read(), - ).group(1), - ) + + if version := re.search( + r'__version__\s*=\s*[\'"]([^\'"]*)[\'"]', open("pfbudget/__init__.py").read() + ): + parser.add_argument( + "--version", + action="version", + version=version.group(1), + ) subparsers = parser.add_subparsers(required=True) - """ - Init - """ - p_init = subparsers.add_parser( - "init", - description="Initializes the SQLite3 database", - parents=[universal], - formatter_class=argparse.ArgumentDefaultsHelpFormatter, - ) - p_init.set_defaults(command=Operation.Init) + # TODO Init + # init = subparsers.add_parser("init") + # init.set_defaults(op=Operation.Init) # Exports transactions to .csv file export = subparsers.add_parser("export") @@ -206,11 +203,6 @@ def report(args): pfbudget.reporting.report.detailed(DatabaseClient(args.database), start, end) -# def nordigen_banks(manager: Manager, args): -# input = NordigenInput(manager) -# input.list(vars(args)["country"][0]) - - def bank(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 0cfc5af..ca83aa6 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -1,17 +1,8 @@ -from pfbudget.db.model import ( - Category, - CategorySelector, - Selector, - Tag, - Transaction, - TransactionCategory, - TransactionTag, -) - from codetiming import Timer from datetime import timedelta +from typing import Sequence -Transactions = list[Transaction] +import pfbudget.db.model as t class Categorizer: @@ -22,9 +13,9 @@ class Categorizer: def rules( self, - transactions: Transactions, - categories: list[Category], - tags: list[Tag], + transactions: Sequence[t.BankTransaction], + categories: Sequence[t.Category], + tags: Sequence[t.Tag], ): """Overarching categorization tool @@ -32,9 +23,9 @@ class Categorizer: to the rules defined for each category Args: - transactions (list[Transaction]): uncategorized transactions - categories (list[Category]): available categories - tags (list[Tag]): currently available tags + transactions (Sequence[BankTransaction]): uncategorized transactions + categories (Sequence[Category]): available categories + tags (Sequence[Tag]): currently available tags """ self._nullify(transactions) @@ -44,21 +35,21 @@ class Categorizer: def manual( self, - transactions: Transactions, - categories: list[Category], - tags: list[Tag], + transactions: Sequence[t.Transaction], + categories: Sequence[t.Category], + tags: Sequence[t.Tag], ): """Manual categorization input Args: - transactions (list[Transaction]): uncategorized transactions - categories (list[Category]): available categories - tags (list[Tag]): currently available tags + transactions (Sequence[Transaction]): uncategorized transactions + categories (Sequence[Category]): available categories + tags (Sequence[Tag]): currently available tags """ self._manual(transactions) @Timer(name="nullify") - def _nullify(self, transactions: Transactions): + def _nullify(self, transactions: Sequence[t.BankTransaction]): count = 0 matching = [] for transaction in transactions: @@ -76,11 +67,13 @@ class Categorizer: and cancel.amount == -transaction.amount ) ): - transaction.category = TransactionCategory( - name="null", selector=CategorySelector(Selector.nullifier) + transaction.category = t.TransactionCategory( + name="null", + selector=t.CategorySelector(t.Selector.nullifier), ) - cancel.category = TransactionCategory( - name="null", selector=CategorySelector(Selector.nullifier) + cancel.category = t.TransactionCategory( + name="null", + selector=t.CategorySelector(t.Selector.nullifier), ) matching.extend([transaction, cancel]) count += 2 @@ -90,7 +83,9 @@ class Categorizer: @Timer(name="categoryrules") def _rule_based_categories( - self, transactions: Transactions, categories: list[Category] + self, + transactions: Sequence[t.BankTransaction], + categories: Sequence[t.Category], ): d = {} for category in [c for c in categories if c.rules]: @@ -114,10 +109,10 @@ class Categorizer: == "y" ): transaction.category.name = category.name - transaction.category.selector.selector = Selector.rules + transaction.category.selector.selector = t.Selector.rules else: - transaction.category = TransactionCategory( - category.name, CategorySelector(Selector.rules) + transaction.category = t.TransactionCategory( + category.name, t.CategorySelector(t.Selector.rules) ) if rule in d: @@ -129,9 +124,11 @@ class Categorizer: print(f"{v}: {k}") @Timer(name="tagrules") - def _rule_based_tags(self, transactions: Transactions, tags: list[Tag]): + def _rule_based_tags( + self, transactions: Sequence[t.BankTransaction], tags: Sequence[t.Tag] + ): d = {} - for tag in [t for t in tags if t.rules]: + for tag in [t for t in tags if len(t.rules) > 0]: for rule in tag.rules: # for transaction in [t for t in transactions if not t.category]: for transaction in [ @@ -143,9 +140,9 @@ class Categorizer: continue if not transaction.tags: - transaction.tags = {TransactionTag(tag.name)} + transaction.tags = {t.TransactionTag(tag.name)} else: - transaction.tags.add(TransactionTag(tag.name)) + transaction.tags.add(t.TransactionTag(tag.name)) if rule in d: d[rule] += 1 @@ -155,7 +152,7 @@ class Categorizer: for k, v in d.items(): print(f"{v}: {k}") - def _manual(self, transactions: Transactions): + def _manual(self, transactions: Sequence[t.Transaction]): uncategorized = [t for t in transactions if not t.category] print(f"{len(uncategorized)} transactions left to categorize") @@ -167,8 +164,8 @@ class Categorizer: if not category: print("{category} doesn't exist") continue - transaction.category = TransactionCategory( - category, CategorySelector(Selector.manual) + transaction.category = t.TransactionCategory( + category, t.CategorySelector(t.Selector.manual) ) break diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 20e9c68..130e43e 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -12,6 +12,7 @@ from pfbudget.db.model import ( CategoryGroup, CategoryRule, CategorySelector, + Link, MoneyTransaction, Nordigen, Rule, @@ -41,18 +42,21 @@ class Manager: # Adapter for the parse_data method. Can be refactored. args = {"bank": params[1], "creditcard": params[2], "category": None} transactions = [] - for path in params[0]: - if (dir := Path(path)).is_dir(): - for file in dir.iterdir(): + for path in [Path(p) for p in params[0]]: + if path.is_dir(): + for file in path.iterdir(): transactions.extend(self.parse(file, args)) - elif Path(path).is_file(): + elif path.is_file(): transactions.extend(self.parse(path, args)) else: raise FileNotFoundError(path) - print(transactions) - if len(transactions) > 0 and input("Commit? (y/n)") == "y": - self.add_transactions(sorted(transactions)) + if ( + len(transactions) > 0 + and input(f"{transactions[:5]}\nCommit? (y/n)") == "y" + ): + with self.db.session() as session: + session.add(sorted(transactions)) case Operation.Download: client = NordigenInput() @@ -68,14 +72,15 @@ class Manager: # dry-run if not params[2]: - self.add_transactions(transactions) + with self.db.session() as session: + session.add(sorted(transactions)) else: print(transactions) case Operation.Categorize: with self.db.session() as session: uncategorized = session.get( - Transaction, ~Transaction.category.has() + BankTransaction, ~BankTransaction.category.has() ) categories = session.get(Category) tags = session.get(Tag) @@ -155,7 +160,7 @@ class Manager: case Operation.GroupAdd: with self.db.session() as session: - session.add(CategoryGroup(params)) + session.add(params) case Operation.GroupRemove: assert all(isinstance(param, CategoryGroup) for param in params) @@ -167,6 +172,8 @@ class Manager: session.add(params) case Operation.Dismantle: + assert all(isinstance(param, Link) for param in params) + with self.db.session() as session: original = params[0].original links = [link.link for link in params] @@ -174,7 +181,7 @@ class Manager: case Operation.Export: with self.db.session() as session: - self.dump(params[0], session.get(Transaction)) + self.dump(params[0], sorted(session.get(Transaction))) case Operation.Import: transactions = [] @@ -246,34 +253,9 @@ class Manager: with self.db.session() as session: session.add(rules) - # def init(self): - # client = DatabaseClient(self.__db) - # client.init() - - # def register(self): - # bank = Bank(self.args["bank"][0], "", self.args["requisition"][0], self.args["invert"]) - # client = DatabaseClient(self.__db) - # client.register_bank(convert(bank)) - - # def unregister(self): - # client = DatabaseClient(self.__db) - # client.unregister_bank(self.args["bank"][0]) - - def parse(self, filename: str, args: dict): + def parse(self, filename: Path, args: dict): return parse_data(filename, args) - # def transactions() -> list[Transaction]: - # pass - - def add_transactions(self, transactions): - with self.db.session() as session: - session.add(transactions) - - # def get_bank_by(self, key: str, value: str) -> Bank: - # client = DatabaseClient(self.__db) - # bank = client.get_bank(key, value) - # return convert(bank) - def dump(self, fn, sequence): with open(fn, "wb") as f: pickle.dump([e.format for e in sequence], f) diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 29940d8..cf42b0f 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -1,25 +1,16 @@ from dataclasses import asdict -from datetime import date from sqlalchemy import create_engine, delete, select, update from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from typing import Sequence, Type, TypeVar from pfbudget.db.model import ( Category, CategoryGroup, - CategoryRule, CategorySchedule, Link, - Tag, - TagRule, - Transaction, ) -# import logging - -# logging.basicConfig() -# logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) - class DbClient: """ @@ -53,7 +44,9 @@ class DbClient: def expunge_all(self): self.__session.expunge_all() - def get(self, type, column=None, values=None): + T = TypeVar("T") + + def get(self, type: Type[T], column=None, values=None) -> Sequence[T]: if column is not None: if values: stmt = select(type).where(column.in_(values)) @@ -67,7 +60,7 @@ class DbClient: def add(self, rows: list): self.__session.add_all(rows) - def remove_by_name(self, type: Category | Tag | Transaction, rows: list): + def remove_by_name(self, type, rows: list): stmt = delete(type).where(type.name.in_([row.name for row in rows])) self.__session.execute(stmt) @@ -91,7 +84,7 @@ class DbClient: ) self.__session.execute(stmt) - def remove_by_id(self, type: CategoryRule | TagRule, ids: list[int]): + def remove_by_id(self, type, ids: list[int]): stmt = delete(type).where(type.id.in_(ids)) self.__session.execute(stmt) @@ -99,19 +92,11 @@ class DbClient: print(type, values) self.__session.execute(update(type), values) - def remove_links(self, original, links: list): + def remove_links(self, original: int, links: list[int]): stmt = delete(Link).where( Link.original == original, Link.link.in_(link for link in links) ) self.__session.execute(stmt) - def transactions(self, min: date, max: date, banks: list[str]): - stmt = select(Transaction).where( - Transaction.date >= min, - Transaction.date <= max, - Transaction.bank.in_(banks), - ) - return self.__session.scalars(stmt).all() - def session(self) -> ClientSession: return self.ClientSession(self.engine) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 7ed3dec..bd44706 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -1,4 +1,9 @@ from __future__ import annotations +import datetime as dt +import decimal +import enum +import re +from typing import Annotated, Any, Optional from sqlalchemy import ( BigInteger, @@ -17,12 +22,6 @@ from sqlalchemy.orm import ( relationship, ) -from decimal import Decimal -from typing import Annotated, Optional -import datetime as dt -import enum -import re - class Base(MappedAsDataclass, DeclarativeBase): __table_args__ = {"schema": "transactions"} @@ -65,13 +64,13 @@ class Bank(Base): BIC: Mapped[str] = mapped_column(String(8), primary_key=True) type: Mapped[accounttype] = mapped_column(primary_key=True) - nordigen: Mapped[Optional[Nordigen]] = relationship(lazy="joined") + nordigen: Mapped[Optional[Nordigen]] = relationship(lazy="joined", init=False) bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] idpk = Annotated[int, mapped_column(BigInteger, primary_key=True, autoincrement=True)] -money = Annotated[Decimal, mapped_column(Numeric(16, 2))] +money = Annotated[decimal.Decimal, mapped_column(Numeric(16, 2))] class Transaction(Base, Export): @@ -86,7 +85,7 @@ class Transaction(Base, Export): category: Mapped[Optional[TransactionCategory]] = relationship(init=False) note: Mapped[Optional[Note]] = relationship(init=False) - tags: Mapped[Optional[set[TransactionTag]]] = relationship(init=False) + tags: Mapped[set[TransactionTag]] = relationship(init=False) __mapper_args__ = {"polymorphic_on": "type", "polymorphic_identity": "transaction"} @@ -152,11 +151,11 @@ class Category(Base): ForeignKey(CategoryGroup.name), default=None ) - rules: Mapped[Optional[set[CategoryRule]]] = relationship( + rules: Mapped[set[CategoryRule]] = relationship( cascade="all, delete-orphan", passive_deletes=True, default_factory=set ) - schedule: Mapped[CategorySchedule] = relationship( - back_populates="category", default=None + schedule: Mapped[Optional[CategorySchedule]] = relationship( + cascade="all, delete-orphan", passive_deletes=True, default=None ) def __repr__(self) -> str: @@ -205,7 +204,7 @@ class Tag(Base): name: Mapped[str] = mapped_column(primary_key=True) - rules: Mapped[Optional[set[TagRule]]] = relationship( + rules: Mapped[set[TagRule]] = relationship( cascade="all, delete-orphan", passive_deletes=True, default_factory=set ) @@ -273,8 +272,6 @@ class CategorySchedule(Base): period_multiplier: Mapped[Optional[int]] amount: Mapped[Optional[int]] - category: Mapped[Category] = relationship(back_populates="schedule") - class Link(Base): __tablename__ = "links" diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index 14e166e..5fa9453 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -1,18 +1,18 @@ -from datetime import date -from time import sleep -from requests import HTTPError, ReadTimeout -from dotenv import load_dotenv -from nordigen import NordigenClient -from uuid import uuid4 +import datetime as dt +import dotenv import json +import nordigen import os +import requests +import time +import uuid -from pfbudget.db.model import BankTransaction -from pfbudget.utils import convert +import pfbudget.db.model as t +import pfbudget.utils as utils from .input import Input -load_dotenv() +dotenv.load_dotenv() class NordigenInput(Input): @@ -20,16 +20,22 @@ class NordigenInput(Input): def __init__(self): super().__init__() - self._client = NordigenClient( - secret_key=os.environ.get("SECRET_KEY"), - secret_id=os.environ.get("SECRET_ID"), + + if not (key := os.environ.get("SECRET_KEY")) or not ( + id := os.environ.get("SECRET_ID") + ): + raise + + self._client = nordigen.NordigenClient( + secret_key=key, + secret_id=id, ) self._client.token = self.__token() - self._start = date.min - self._end = date.max + self._start = dt.date.min + self._end = dt.date.max - def parse(self) -> list[BankTransaction]: + def parse(self) -> list[t.BankTransaction]: transactions = [] assert len(self._banks) > 0 @@ -49,14 +55,14 @@ class NordigenInput(Input): try: downloaded = account.get_transactions() break - except ReadTimeout: + except requests.ReadTimeout: retries += 1 print(f"Request #{retries} timed-out, retrying in 1s") - sleep(1) - except HTTPError as e: + time.sleep(1) + except requests.HTTPError as e: retries += 1 print(f"Request #{retries} failed with {e}, retrying in 1s") - sleep(1) + time.sleep(1) if not downloaded: print(f"Couldn't download transactions for {account}") @@ -66,7 +72,7 @@ class NordigenInput(Input): json.dump(downloaded, f) converted = [ - convert(t, bank) for t in downloaded["transactions"]["booked"] + utils.convert(t, bank) for t in downloaded["transactions"]["booked"] ] transactions.extend( @@ -82,11 +88,12 @@ class NordigenInput(Input): def requisition(self, institution: str, country: str = "PT"): id = self._client.institution.get_institution_id_by_name(country, institution) - return self._client.initialize_session( + requisition = self._client.initialize_session( redirect_uri=self.redirect_url, institution_id=id, - reference_id=str(uuid4()), + reference_id=str(uuid.uuid4()), ) + return requisition.link, requisition.requisition_id def country_banks(self, country: str): return self._client.institution.get_institutions(country) @@ -125,4 +132,4 @@ class NordigenInput(Input): else: token = self._client.generate_token() print(f"New access token: {token}") - return token + return token["access"] diff --git a/pfbudget/input/parsers.py b/pfbudget/input/parsers.py index facacf8..89460e3 100644 --- a/pfbudget/input/parsers.py +++ b/pfbudget/input/parsers.py @@ -1,6 +1,7 @@ from collections import namedtuple from decimal import Decimal from importlib import import_module +from pathlib import Path import datetime as dt import yaml @@ -44,7 +45,7 @@ Options = namedtuple( ) -def parse_data(filename: str, args: dict) -> list[Transaction]: +def parse_data(filename: Path, args: dict) -> list[Transaction]: cfg: dict = yaml.safe_load(open("parsers.yaml")) assert ( "Banks" in cfg @@ -85,7 +86,7 @@ def parse_data(filename: str, args: dict) -> list[Transaction]: class Parser: - def __init__(self, filename: str, bank: str, options: dict): + def __init__(self, filename: Path, bank: str, options: dict): self.filename = filename self.bank = bank diff --git a/pfbudget/utils/converters.py b/pfbudget/utils/converters.py index 62461c1..c4f72b6 100644 --- a/pfbudget/utils/converters.py +++ b/pfbudget/utils/converters.py @@ -1,23 +1,25 @@ -from datetime import date -from functools import singledispatch +import datetime as dt +import functools +from typing import Any from pfbudget.common.types import TransactionError -from pfbudget.db.model import Bank, BankTransaction +import pfbudget.db.model as t + from .utils import parse_decimal -@singledispatch -def convert(t): - print("No converter as been found") +@functools.singledispatch +def convert(t) -> Any: + print("No converter has been found") pass @convert.register -def _(json: dict, bank: Bank) -> BankTransaction: - i = -1 if bank.nordigen.invert else 1 +def _(json: dict, bank: t.Bank) -> t.BankTransaction | None: + i = -1 if bank.nordigen and bank.nordigen.invert else 1 try: - transaction = BankTransaction( - date=date.fromisoformat(json["bookingDate"]), + transaction = t.BankTransaction( + date=dt.date.fromisoformat(json["bookingDate"]), description=json["remittanceInformationUnstructured"], bank=bank.name, amount=i * parse_decimal(json["transactionAmount"]["amount"]), From dd0aaa01b8d17110b7807be453e524b57204c4b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 22 Jan 2023 23:35:29 +0000 Subject: [PATCH 44/66] Export/import for banks --- pfbudget/__main__.py | 11 ++++++++- pfbudget/cli/runnable.py | 8 +++++++ pfbudget/common/types.py | 2 ++ pfbudget/core/manager.py | 48 ++++++++++++++++++++++++---------------- pfbudget/db/model.py | 22 ++++++++++++++++-- 5 files changed, 69 insertions(+), 22 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 89c2780..0ceb1cc 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -232,7 +232,16 @@ if __name__ == "__main__": pfbudget.t.Link(args["original"][0], link) for link in args["links"] ] - case pfbudget.Operation.Export | pfbudget.Operation.Import | pfbudget.Operation.ExportCategoryRules | pfbudget.Operation.ImportCategoryRules | pfbudget.Operation.ExportTagRules | pfbudget.Operation.ImportTagRules: + case ( + pfbudget.Operation.Export + | pfbudget.Operation.Import + | pfbudget.Operation.ExportBanks + | pfbudget.Operation.ImportBanks + | pfbudget.Operation.ExportCategoryRules + | pfbudget.Operation.ImportCategoryRules + | pfbudget.Operation.ExportTagRules + | pfbudget.Operation.ImportTagRules + ): keys = {"file"} assert args.keys() >= keys, f"missing {args.keys() - keys}" diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index d368f7e..207848f 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -225,6 +225,14 @@ def bank(parser: argparse.ArgumentParser): nordigen(commands.add_parser("nordigen")) + export = commands.add_parser("export") + export.set_defaults(op=Operation.ExportBanks) + export_args(export) + + pimport = commands.add_parser("import") + pimport.set_defaults(op=Operation.ImportBanks) + export_args(pimport) + def nordigen(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index a77cc52..ae533c2 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -37,6 +37,8 @@ class Operation(Enum): NordigenCountryBanks = auto() Export = auto() Import = auto() + ExportBanks = auto() + ImportBanks = auto() ExportCategoryRules = auto() ImportCategoryRules = auto() ExportTagRules = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 130e43e..f50eec9 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -213,16 +213,26 @@ class Manager: transactions.append(transaction) - if ( - len(transactions) > 0 - and input( - f"{transactions[:5]}\nDoes the import seem correct? (y/n)" - ) - == "y" - ): + if self.certify(transactions): with self.db.session() as session: session.add(transactions) + case Operation.ExportBanks: + with self.db.session() as session: + self.dump(params[0], session.get(Bank)) + + case Operation.ImportBanks: + banks = [] + for row in self.load(params[0]): + bank = Bank(row["name"], row["BIC"], row["type"]) + if row["nordigen"]: + bank.nordigen = Nordigen(**row["nordigen"]) + banks.append(bank) + + if self.certify(banks): + with self.db.session() as session: + session.add(banks) + case Operation.ExportCategoryRules: with self.db.session() as session: self.dump(params[0], session.get(CategoryRule)) @@ -230,11 +240,7 @@ class Manager: case Operation.ImportCategoryRules: rules = [CategoryRule(**row) for row in self.load(params[0])] - if ( - len(rules) > 0 - and input(f"{rules[:5]}\nDoes the import seem correct? (y/n)") - == "y" - ): + if self.certify(rules): with self.db.session() as session: session.add(rules) @@ -245,25 +251,29 @@ class Manager: case Operation.ImportTagRules: rules = [TagRule(**row) for row in self.load(params[0])] - if ( - len(rules) > 0 - and input(f"{rules[:5]}\nDoes the import seem correct? (y/n)") - == "y" - ): + if self.certify(rules): with self.db.session() as session: session.add(rules) def parse(self, filename: Path, args: dict): return parse_data(filename, args) - def dump(self, fn, sequence): + @staticmethod + def dump(fn, sequence): with open(fn, "wb") as f: pickle.dump([e.format for e in sequence], f) - def load(self, fn): + @staticmethod + def load(fn): with open(fn, "rb") as f: return pickle.load(f) + @staticmethod + def certify(imports: list) -> bool: + if input(f"{imports[:10]}\nDoes the import seem correct? (y/n)") == "y": + return True + return False + @property def db(self) -> DbClient: return DbClient(self._db, self._verbosity > 2) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index bd44706..a1e888c 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -57,7 +57,7 @@ class Export: raise NotImplementedError -class Bank(Base): +class Bank(Base, Export): __tablename__ = "banks" name: Mapped[str] = mapped_column(unique=True) @@ -66,6 +66,15 @@ class Bank(Base): nordigen: Mapped[Optional[Nordigen]] = relationship(lazy="joined", init=False) + @property + def format(self) -> dict[str, Any]: + return dict( + name=self.name, + BIC=self.BIC, + type=self.type, + nordigen=self.nordigen.format if self.nordigen else None, + ) + bankfk = Annotated[str, mapped_column(Text, ForeignKey(Bank.name))] @@ -190,7 +199,7 @@ class Note(Base): note: Mapped[str] -class Nordigen(Base): +class Nordigen(Base, Export): __tablename__ = "nordigen" name: Mapped[bankfk] = mapped_column(primary_key=True) @@ -198,6 +207,15 @@ class Nordigen(Base): requisition_id: Mapped[Optional[str]] invert: Mapped[Optional[bool]] + @property + def format(self) -> dict[str, Any]: + return dict( + name=self.name, + bank_id=self.bank_id, + requisition_id=self.requisition_id, + invert=self.invert, + ) + class Tag(Base): __tablename__ = "tags_available" From 8760f5a0a4853f59c093ded6d21850a82c62c264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 00:06:36 +0000 Subject: [PATCH 45/66] Export/Import categories and groups --- pfbudget/__main__.py | 4 ++++ pfbudget/cli/runnable.py | 16 ++++++++++++++++ pfbudget/common/types.py | 4 ++++ pfbudget/core/manager.py | 36 ++++++++++++++++++++++++++++++++++++ pfbudget/db/model.py | 28 +++++++++++++++++++++++++--- 5 files changed, 85 insertions(+), 3 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 0ceb1cc..06259ac 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -241,6 +241,10 @@ if __name__ == "__main__": | pfbudget.Operation.ImportCategoryRules | pfbudget.Operation.ExportTagRules | pfbudget.Operation.ImportTagRules + | pfbudget.Operation.ExportCategories + | pfbudget.Operation.ImportCategories + | pfbudget.Operation.ExportCategoryGroups + | pfbudget.Operation.ImportCategoryGroups ): keys = {"file"} assert args.keys() >= keys, f"missing {args.keys() - keys}" diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index 207848f..e174224 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -286,6 +286,14 @@ def category(parser: argparse.ArgumentParser): group = commands.add_parser("group") category_group(group) + export = commands.add_parser("export") + export.set_defaults(op=Operation.ExportCategories) + export_args(export) + + pimport = commands.add_parser("import") + pimport.set_defaults(op=Operation.ImportCategories) + export_args(pimport) + def category_group(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) @@ -298,6 +306,14 @@ def category_group(parser: argparse.ArgumentParser): remove.set_defaults(op=Operation.GroupRemove) remove.add_argument("group", nargs="+", type=str) + export = commands.add_parser("export") + export.set_defaults(op=Operation.ExportCategoryGroups) + export_args(export) + + pimport = commands.add_parser("import") + pimport.set_defaults(op=Operation.ImportCategoryGroups) + export_args(pimport) + def category_rule(parser: argparse.ArgumentParser): commands = parser.add_subparsers(required=True) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index ae533c2..0ce7d54 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -43,6 +43,10 @@ class Operation(Enum): ImportCategoryRules = auto() ExportTagRules = auto() ImportTagRules = auto() + ExportCategories = auto() + ImportCategories = auto() + ExportCategoryGroups = auto() + ImportCategoryGroups = auto() class TransactionError(Exception): diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index f50eec9..ccae418 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -11,6 +11,7 @@ from pfbudget.db.model import ( Category, CategoryGroup, CategoryRule, + CategorySchedule, CategorySelector, Link, MoneyTransaction, @@ -255,6 +256,41 @@ class Manager: with self.db.session() as session: session.add(rules) + case Operation.ExportCategories: + with self.db.session() as session: + self.dump(params[0], session.get(Category)) + + case Operation.ImportCategories: + # rules = [Category(**row) for row in self.load(params[0])] + categories = [] + for row in self.load(params[0]): + category = Category(row["name"], row["group"]) + if len(row["rules"]) > 0: + # Only category rules could have been created with a rule + rules = row["rules"] + for rule in rules: + del rule["type"] + + category.rules = set(CategoryRule(**rule) for rule in rules) + if row["schedule"]: + category.schedule = CategorySchedule(**row["schedule"]) + categories.append(category) + + if self.certify(categories): + with self.db.session() as session: + session.add(categories) + + case Operation.ExportCategoryGroups: + with self.db.session() as session: + self.dump(params[0], session.get(CategoryGroup)) + + case Operation.ImportCategoryGroups: + groups = [CategoryGroup(**row) for row in self.load(params[0])] + + if self.certify(groups): + with self.db.session() as session: + session.add(groups) + def parse(self, filename: Path, args: dict): return parse_data(filename, args) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index a1e888c..a00dec4 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -146,13 +146,17 @@ class SplitTransaction(Transaction): return super().format | dict(original=self.original) -class CategoryGroup(Base): +class CategoryGroup(Base, Export): __tablename__ = "categories_groups" name: Mapped[str] = mapped_column(primary_key=True) + @property + def format(self) -> dict[str, Any]: + return dict(name=self.name) -class Category(Base): + +class Category(Base, Export): __tablename__ = "categories_available" name: Mapped[str] = mapped_column(primary_key=True) @@ -170,6 +174,15 @@ class Category(Base): def __repr__(self) -> str: return f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)}, schedule={self.schedule})" + @property + def format(self) -> dict[str, Any]: + return dict( + name=self.name, + group=self.group.format if self.group else None, + rules=[rule.format for rule in self.rules], + schedule=self.schedule.format if self.schedule else None, + ) + catfk = Annotated[ str, @@ -282,7 +295,7 @@ class Period(enum.Enum): scheduleperiod = Annotated[Selector, mapped_column(Enum(Period, inherit_schema=True))] -class CategorySchedule(Base): +class CategorySchedule(Base, Export): __tablename__ = "categories_schedules" name: Mapped[catfk] = mapped_column(primary_key=True) @@ -290,6 +303,15 @@ class CategorySchedule(Base): period_multiplier: Mapped[Optional[int]] amount: Mapped[Optional[int]] + @property + def format(self) -> dict[str, Any]: + return dict( + name=self.name, + period=self.period, + period_multiplier=self.period_multiplier, + amount=self.amount, + ) + class Link(Base): __tablename__ = "links" From ae0a1950693e49ff1470c3b719f3863ffd6cc657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 00:31:31 +0000 Subject: [PATCH 46/66] Split tables per different schemas --- alembic/env.py | 2 +- .../e455c78df789_divide_by_schemas.py | 452 ++++++++++++++++++ pfbudget/core/categorizer.py | 10 +- pfbudget/db/model.py | 37 +- 4 files changed, 481 insertions(+), 20 deletions(-) create mode 100644 alembic/versions/e455c78df789_divide_by_schemas.py diff --git a/alembic/env.py b/alembic/env.py index 4b46b18..69ba638 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -27,7 +27,7 @@ target_metadata = Base.metadata def include_name(name, type_, parent_names): if type_ == "schema": - return name in ["transactions"] + return name in ["bank", "category", "tag", "transactions"] else: return True diff --git a/alembic/versions/e455c78df789_divide_by_schemas.py b/alembic/versions/e455c78df789_divide_by_schemas.py new file mode 100644 index 0000000..fb9e26d --- /dev/null +++ b/alembic/versions/e455c78df789_divide_by_schemas.py @@ -0,0 +1,452 @@ +"""Divide by schemas + +Revision ID: e455c78df789 +Revises: 6b293f78cc97 +Create Date: 2023-01-22 23:38:23.266906+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "e455c78df789" +down_revision = "6b293f78cc97" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "banks", + sa.Column("name", sa.String(), nullable=False), + sa.Column("BIC", sa.String(length=8), nullable=False), + sa.Column( + "type", + sa.Enum( + "checking", + "savings", + "investment", + "mealcard", + "VISA", + "MASTERCARD", + name="accounttype", + schema="bank", + inherit_schema=True, + ), + nullable=False, + ), + sa.PrimaryKeyConstraint("BIC", "type", name=op.f("pk_banks")), + sa.UniqueConstraint("name", name=op.f("uq_banks_name")), + schema="bank", + ) + op.create_table( + "groups", + sa.Column("name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_groups")), + schema="category", + ) + op.create_table( + "available", + sa.Column("name", sa.String(), nullable=False), + sa.PrimaryKeyConstraint("name", name=op.f("pk_available")), + schema="tag", + ) + op.create_table( + "nordigen", + sa.Column("name", sa.Text(), nullable=False), + sa.Column("bank_id", sa.String(), nullable=True), + sa.Column("requisition_id", sa.String(), nullable=True), + sa.Column("invert", sa.Boolean(), nullable=True), + sa.ForeignKeyConstraint( + ["name"], ["bank.banks.name"], name=op.f("fk_nordigen_name_banks") + ), + sa.PrimaryKeyConstraint("name", name=op.f("pk_nordigen")), + schema="bank", + ) + op.create_table( + "available", + sa.Column("name", sa.String(), nullable=False), + sa.Column("group", sa.String(), nullable=True), + sa.ForeignKeyConstraint( + ["group"], ["category.groups.name"], name=op.f("fk_available_group_groups") + ), + sa.PrimaryKeyConstraint("name", name=op.f("pk_available")), + schema="category", + ) + op.create_table( + "rules", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("tag", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.rules.id"], + name=op.f("fk_rules_id_rules"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag"], + ["tag.available.name"], + name=op.f("fk_rules_tag_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_rules")), + schema="tag", + ) + op.create_table( + "rules", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.rules.id"], + name=op.f("fk_rules_id_rules"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["name"], + ["category.available.name"], + name=op.f("fk_rules_name_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_rules")), + schema="category", + ) + op.create_table( + "schedules", + sa.Column("name", sa.String(), nullable=False), + sa.Column( + "period", + sa.Enum( + "daily", + "weekly", + "monthly", + "yearly", + name="period", + schema="category", + inherit_schema=True, + ), + nullable=True, + ), + sa.Column("period_multiplier", sa.Integer(), nullable=True), + sa.Column("amount", sa.Integer(), nullable=True), + sa.ForeignKeyConstraint( + ["name"], + ["category.available.name"], + name=op.f("fk_schedules_name_available"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("name", name=op.f("pk_schedules")), + schema="category", + ) + op.create_table( + "selector", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column( + "selector", + sa.Enum( + "unknown", + "nullifier", + "vacations", + "rules", + "algorithm", + "manual", + name="selector_t", + schema="category", + inherit_schema=True, + ), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.categorized.id"], + name=op.f("fk_selector_id_categorized"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_selector")), + schema="category", + ) + op.drop_constraint( + "fk_categorized_name_categories_available", + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_categorized_name_available"), + "categorized", + "available", + ["name"], + ["name"], + source_schema="transactions", + referent_schema="category", + ondelete="CASCADE", + ) + op.drop_constraint( + "fk_originals_bank_banks", + "originals", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + op.f("fk_originals_bank_banks"), + "originals", + "banks", + ["bank"], + ["name"], + source_schema="transactions", + referent_schema="bank", + ) + op.drop_constraint( + "fk_tags_tag_tags_available", "tags", schema="transactions", type_="foreignkey" + ) + op.create_foreign_key( + op.f("fk_tags_tag_available"), + "tags", + "available", + ["tag"], + ["name"], + source_schema="transactions", + referent_schema="tag", + ) + op.drop_table("categories_schedules", schema="transactions") + op.drop_table("categories_rules", schema="transactions") + op.drop_table("categories_available", schema="transactions") + op.drop_table("tag_rules", schema="transactions") + op.drop_table("nordigen", schema="transactions") + op.drop_table("tags_available", schema="transactions") + op.drop_table("banks", schema="transactions") + op.drop_table("categories_selector", schema="transactions") + op.drop_table("categories_groups", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_tags_tag_available"), "tags", schema="transactions", type_="foreignkey" + ) + op.create_foreign_key( + "fk_tags_tag_tags_available", + "tags", + "tags_available", + ["tag"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + op.drop_constraint( + op.f("fk_originals_bank_banks"), + "originals", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_originals_bank_banks", + "originals", + "banks", + ["bank"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ) + op.drop_constraint( + op.f("fk_categorized_name_available"), + "categorized", + schema="transactions", + type_="foreignkey", + ) + op.create_foreign_key( + "fk_categorized_name_categories_available", + "categorized", + "categories_available", + ["name"], + ["name"], + source_schema="transactions", + referent_schema="transactions", + ondelete="CASCADE", + ) + op.create_table( + "categories_groups", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint("name", name="pk_categories_groups"), + schema="transactions", + postgresql_ignore_search_path=False, + ) + op.create_table( + "categories_selector", + sa.Column("id", sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column( + "selector", + postgresql.ENUM( + "unknown", + "nullifier", + "vacations", + "rules", + "algorithm", + "manual", + name="selector", + schema="transactions", + ), + autoincrement=False, + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.categorized.id"], + name="fk_categories_selector_id_categorized", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_categories_selector"), + schema="transactions", + ) + op.create_table( + "banks", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("BIC", sa.VARCHAR(length=8), autoincrement=False, nullable=False), + sa.Column( + "type", + postgresql.ENUM( + "checking", + "savings", + "investment", + "mealcard", + "VISA", + "MASTERCARD", + name="accounttype", + schema="transactions", + ), + autoincrement=False, + nullable=False, + ), + sa.PrimaryKeyConstraint("BIC", "type", name="pk_banks"), + sa.UniqueConstraint("name", name="uq_banks_name"), + schema="transactions", + postgresql_ignore_search_path=False, + ) + op.create_table( + "tags_available", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint("name", name="pk_tags_available"), + schema="transactions", + postgresql_ignore_search_path=False, + ) + op.create_table( + "nordigen", + sa.Column("name", sa.TEXT(), autoincrement=False, nullable=False), + sa.Column("bank_id", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("requisition_id", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("invert", sa.BOOLEAN(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint( + ["name"], ["transactions.banks.name"], name="fk_nordigen_name_banks" + ), + sa.PrimaryKeyConstraint("name", name="pk_nordigen"), + schema="transactions", + ) + op.create_table( + "tag_rules", + sa.Column( + "id", + sa.BIGINT(), + server_default=sa.text( + "nextval('transactions.tag_rules_id_seq'::regclass)" + ), + autoincrement=True, + nullable=False, + ), + sa.Column("tag", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.rules.id"], + name="fk_tag_rules_id_rules", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tag"], + ["transactions.tags_available.name"], + name="fk_tag_rules_tag_tags_available", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_tag_rules"), + schema="transactions", + ) + op.create_table( + "categories_rules", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column( + "id", + sa.BIGINT(), + server_default=sa.text( + "nextval('transactions.categories_rules_id_seq'::regclass)" + ), + autoincrement=True, + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.rules.id"], + name="fk_categories_rules_id_rules", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["name"], + ["transactions.categories_available.name"], + name="fk_categories_rules_name_categories_available", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_categories_rules"), + schema="transactions", + ) + op.create_table( + "categories_available", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("group", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint( + ["group"], + ["transactions.categories_groups.name"], + name="fk_categories_available_group_categories_groups", + ), + sa.PrimaryKeyConstraint("name", name="pk_categories_available"), + schema="transactions", + postgresql_ignore_search_path=False, + ) + op.create_table( + "categories_schedules", + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column( + "period", + postgresql.ENUM( + "daily", + "weekly", + "monthly", + "yearly", + name="period", + schema="transactions", + ), + autoincrement=False, + nullable=True, + ), + sa.Column( + "period_multiplier", sa.INTEGER(), autoincrement=False, nullable=True + ), + sa.Column("amount", sa.INTEGER(), autoincrement=False, nullable=True), + sa.ForeignKeyConstraint( + ["name"], + ["transactions.categories_available.name"], + name="fk_categories_schedules_name_categories_available", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("name", name="pk_categories_schedules"), + schema="transactions", + ) + op.drop_table("selector", schema="category") + op.drop_table("schedules", schema="category") + op.drop_table("rules", schema="category") + op.drop_table("rules", schema="tag") + op.drop_table("available", schema="category") + op.drop_table("nordigen", schema="bank") + op.drop_table("available", schema="tag") + op.drop_table("groups", schema="category") + op.drop_table("banks", schema="bank") + # ### end Alembic commands ### diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index ca83aa6..664f3b8 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -69,11 +69,11 @@ class Categorizer: ): transaction.category = t.TransactionCategory( name="null", - selector=t.CategorySelector(t.Selector.nullifier), + selector=t.CategorySelector(t.Selector_T.nullifier), ) cancel.category = t.TransactionCategory( name="null", - selector=t.CategorySelector(t.Selector.nullifier), + selector=t.CategorySelector(t.Selector_T.nullifier), ) matching.extend([transaction, cancel]) count += 2 @@ -109,10 +109,10 @@ class Categorizer: == "y" ): transaction.category.name = category.name - transaction.category.selector.selector = t.Selector.rules + transaction.category.selector.selector = t.Selector_T.rules else: transaction.category = t.TransactionCategory( - category.name, t.CategorySelector(t.Selector.rules) + category.name, t.CategorySelector(t.Selector_T.rules) ) if rule in d: @@ -165,7 +165,7 @@ class Categorizer: print("{category} doesn't exist") continue transaction.category = t.TransactionCategory( - category, t.CategorySelector(t.Selector.manual) + category, t.CategorySelector(t.Selector_T.manual) ) break diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index a00dec4..8c1d6bc 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -24,15 +24,15 @@ from sqlalchemy.orm import ( class Base(MappedAsDataclass, DeclarativeBase): - __table_args__ = {"schema": "transactions"} metadata = MetaData( + schema="transactions", naming_convention={ "ix": "ix_%(column_0_label)s", "uq": "uq_%(table_name)s_%(column_0_name)s", "ck": "ck_%(table_name)s_`%(constraint_name)s`", "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", "pk": "pk_%(table_name)s", - } + }, ) @@ -58,6 +58,7 @@ class Export: class Bank(Base, Export): + __table_args__ = {"schema": "bank"} __tablename__ = "banks" name: Mapped[str] = mapped_column(unique=True) @@ -147,7 +148,8 @@ class SplitTransaction(Transaction): class CategoryGroup(Base, Export): - __tablename__ = "categories_groups" + __table_args__ = {"schema": "category"} + __tablename__ = "groups" name: Mapped[str] = mapped_column(primary_key=True) @@ -157,7 +159,8 @@ class CategoryGroup(Base, Export): class Category(Base, Export): - __tablename__ = "categories_available" + __table_args__ = {"schema": "category"} + __tablename__ = "available" name: Mapped[str] = mapped_column(primary_key=True) group: Mapped[Optional[str]] = mapped_column( @@ -178,7 +181,7 @@ class Category(Base, Export): def format(self) -> dict[str, Any]: return dict( name=self.name, - group=self.group.format if self.group else None, + group=self.group if self.group else None, rules=[rule.format for rule in self.rules], schedule=self.schedule.format if self.schedule else None, ) @@ -213,6 +216,7 @@ class Note(Base): class Nordigen(Base, Export): + __table_args__ = {"schema": "bank"} __tablename__ = "nordigen" name: Mapped[bankfk] = mapped_column(primary_key=True) @@ -231,7 +235,8 @@ class Nordigen(Base, Export): class Tag(Base): - __tablename__ = "tags_available" + __table_args__ = {"schema": "tag"} + __tablename__ = "available" name: Mapped[str] = mapped_column(primary_key=True) @@ -254,7 +259,7 @@ class TransactionTag(Base, Export): return hash(self.id) -class Selector(enum.Enum): +class Selector_T(enum.Enum): unknown = enum.auto() nullifier = enum.auto() vacations = enum.auto() @@ -264,13 +269,14 @@ class Selector(enum.Enum): categoryselector = Annotated[ - Selector, - mapped_column(Enum(Selector, inherit_schema=True), default=Selector.unknown), + Selector_T, + mapped_column(Enum(Selector_T, inherit_schema=True), default=Selector_T.unknown), ] class CategorySelector(Base, Export): - __tablename__ = "categories_selector" + __table_args__ = {"schema": "category"} + __tablename__ = "selector" id: Mapped[int] = mapped_column( BigInteger, @@ -292,11 +298,12 @@ class Period(enum.Enum): yearly = "yearly" -scheduleperiod = Annotated[Selector, mapped_column(Enum(Period, inherit_schema=True))] +scheduleperiod = Annotated[Selector_T, mapped_column(Enum(Period, inherit_schema=True))] class CategorySchedule(Base, Export): - __tablename__ = "categories_schedules" + __table_args__ = {"schema": "category"} + __tablename__ = "schedules" name: Mapped[catfk] = mapped_column(primary_key=True) period: Mapped[Optional[scheduleperiod]] @@ -374,7 +381,8 @@ class Rule(Base, Export): class CategoryRule(Rule): - __tablename__ = "categories_rules" + __table_args__ = {"schema": "category"} + __tablename__ = "rules" id: Mapped[int] = mapped_column( BigInteger, @@ -397,7 +405,8 @@ class CategoryRule(Rule): class TagRule(Rule): - __tablename__ = "tag_rules" + __table_args__ = {"schema": "tag"} + __tablename__ = "rules" id: Mapped[int] = mapped_column( BigInteger, From ace51951644f6e7802539806ed9088194f34a851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 00:43:05 +0000 Subject: [PATCH 47/66] Move selector table back to transactions schema Change table name originals to transactions and tags to tagged. --- ...556ab17c56_selector_back_to_transaction.py | 88 +++++++++++++++++++ pfbudget/db/model.py | 5 +- 2 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 alembic/versions/28556ab17c56_selector_back_to_transaction.py diff --git a/alembic/versions/28556ab17c56_selector_back_to_transaction.py b/alembic/versions/28556ab17c56_selector_back_to_transaction.py new file mode 100644 index 0000000..33d16a6 --- /dev/null +++ b/alembic/versions/28556ab17c56_selector_back_to_transaction.py @@ -0,0 +1,88 @@ +"""Selector back to transaction + +Revision ID: 28556ab17c56 +Revises: e455c78df789 +Create Date: 2023-01-23 00:34:39.062562+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "28556ab17c56" +down_revision = "e455c78df789" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.rename_table("tags", "tagged", schema="transactions") + op.create_table( + "selector", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column( + "selector", + sa.Enum( + "unknown", + "nullifier", + "vacations", + "rules", + "algorithm", + "manual", + name="selector_t", + schema="transactions", + inherit_schema=True, + ), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.categorized.id"], + name=op.f("fk_selector_id_categorized"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_selector")), + schema="transactions", + ) + op.drop_table("selector", schema="category") + + op.rename_table("originals", "transactions", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.rename_table("transactions", "originals", schema="transactions") + + op.create_table( + "selector", + sa.Column("id", sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column( + "selector", + postgresql.ENUM( + "unknown", + "nullifier", + "vacations", + "rules", + "algorithm", + "manual", + name="selector_t", + schema="category", + ), + autoincrement=False, + nullable=False, + ), + sa.ForeignKeyConstraint( + ["id"], + ["transactions.categorized.id"], + name="fk_selector_id_categorized", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name="pk_selector"), + schema="category", + ) + op.drop_table("selector", schema="transactions") + op.rename_table("tagged", "tags", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 8c1d6bc..4c5ff55 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -84,7 +84,7 @@ money = Annotated[decimal.Decimal, mapped_column(Numeric(16, 2))] class Transaction(Base, Export): - __tablename__ = "originals" + __tablename__ = "transactions" id: Mapped[idpk] = mapped_column(init=False) date: Mapped[dt.date] @@ -246,7 +246,7 @@ class Tag(Base): class TransactionTag(Base, Export): - __tablename__ = "tags" + __tablename__ = "tagged" id: Mapped[idfk] = mapped_column(primary_key=True, init=False) tag: Mapped[str] = mapped_column(ForeignKey(Tag.name), primary_key=True) @@ -275,7 +275,6 @@ categoryselector = Annotated[ class CategorySelector(Base, Export): - __table_args__ = {"schema": "category"} __tablename__ = "selector" id: Mapped[int] = mapped_column( From 5235fcdfc347b96838072f3c8820fdff180c691c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 20:11:58 +0000 Subject: [PATCH 48/66] Adds business logic for split transaction Moves split member to be part of parent class and removes it from the init. --- ...11d9ff_split_member_of_base_transaction.py | 40 +++++++++++++++++++ pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 30 +++++++++++++- pfbudget/db/model.py | 6 +-- 4 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 alembic/versions/18572111d9ff_split_member_of_base_transaction.py diff --git a/alembic/versions/18572111d9ff_split_member_of_base_transaction.py b/alembic/versions/18572111d9ff_split_member_of_base_transaction.py new file mode 100644 index 0000000..a5847b2 --- /dev/null +++ b/alembic/versions/18572111d9ff_split_member_of_base_transaction.py @@ -0,0 +1,40 @@ +"""Split member of base transaction + +Revision ID: 18572111d9ff +Revises: 28556ab17c56 +Create Date: 2023-01-23 20:09:37.892997+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "18572111d9ff" +down_revision = "28556ab17c56" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "transactions", + "split", + existing_type=sa.BOOLEAN(), + nullable=False, + schema="transactions", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "transactions", + "split", + existing_type=sa.BOOLEAN(), + nullable=True, + schema="transactions", + ) + # ### end Alembic commands ### diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 0ce7d54..0fe3efe 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -28,6 +28,7 @@ class Operation(Enum): TagRuleModify = auto() Forge = auto() Dismantle = auto() + Split = auto() BankAdd = auto() BankMod = auto() BankDel = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index ccae418..333b07d 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -17,6 +17,7 @@ from pfbudget.db.model import ( MoneyTransaction, Nordigen, Rule, + SplitTransaction, Tag, TagRule, Transaction, @@ -180,6 +181,32 @@ class Manager: links = [link.link for link in params] session.remove_links(original, links) + case Operation.Split: + if len(params) < 1 and not all( + isinstance(p, Transaction) for p in params + ): + raise TypeError(f"{params} are not transactions") + + # t -> t1, t2, t3; t.value == Σti.value + original: Transaction = params[0] + if not original.amount == sum(t.amount for t in params[1:]): + raise ValueError( + f"{original.amount}€ != {sum(v for v, _ in params[1:])}€" + ) + + with self.db.session() as session: + originals = session.get(Transaction, Transaction.id, [original.id]) + assert len(originals) == 1, ">1 transactions matched {original.id}!" + + originals[0].split = True + transactions = [ + SplitTransaction( + originals[0].date, t.description, t.amount, originals[0].id + ) + for t in params[1:] + ] + session.add(transactions) + case Operation.Export: with self.db.session() as session: self.dump(params[0], sorted(session.get(Transaction))) @@ -194,12 +221,11 @@ class Manager: row["description"], row["amount"], row["bank"], - False, ) case "money": transaction = MoneyTransaction( - row["date"], row["description"], row["amount"], False + row["date"], row["description"], row["amount"] ) # TODO case "split" how to match to original transaction?? also save ids? diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 4c5ff55..2be6a07 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -91,6 +91,8 @@ class Transaction(Base, Export): description: Mapped[Optional[str]] amount: Mapped[money] + split: Mapped[bool] = mapped_column(init=False) + type: Mapped[str] = mapped_column(init=False) category: Mapped[Optional[TransactionCategory]] = relationship(init=False) @@ -105,6 +107,7 @@ class Transaction(Base, Export): date=self.date, description=self.description, amount=self.amount, + split=self.split, type=self.type, category=self.category.format if self.category else None, # TODO note @@ -122,7 +125,6 @@ idfk = Annotated[ class BankTransaction(Transaction): bank: Mapped[bankfk] = mapped_column(nullable=True) - split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) __mapper_args__ = {"polymorphic_identity": "bank", "polymorphic_load": "inline"} @@ -132,8 +134,6 @@ class BankTransaction(Transaction): class MoneyTransaction(Transaction): - split: Mapped[bool] = mapped_column(use_existing_column=True, nullable=True) - __mapper_args__ = {"polymorphic_identity": "money"} From 36e7f84bd9e4f8d972bfd6e7ba17a592b561244f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 23:05:41 +0000 Subject: [PATCH 49/66] Interactive categorization moved to __main__.py It didn't make sense to have it inside the manager, it should only be used to process commands and its paramaters. --- pfbudget/__main__.py | 47 ++++++++++++++++++++++++++++++++++++ pfbudget/core/categorizer.py | 33 ------------------------- pfbudget/core/manager.py | 9 ------- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 06259ac..1032684 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,6 +1,49 @@ import pfbudget +def interactive(manager: pfbudget.Manager): + with manager.db.session() as session: + categories = session.get(pfbudget.t.Category) + print(f"Available categories: {categories}") + print(f"Available tags: {session.get(pfbudget.t.Tag)}") + transactions = session.get( + pfbudget.t.Transaction, ~pfbudget.t.Transaction.category.has() + ) + print(f"{len(transactions)} transactions left to categorize") + + for transaction in sorted(transactions): + print(f"{transaction}") + quit = False + next = True + while next: + match (input("(/split/tag/note/quit): ")): + case "quit" | "exit": + next = False + quit = True + + case "tag": + tag = input("tag: ") + transaction.tags.add(pfbudget.t.TransactionTag(tag)) + + case "note": + note = input("note: ") + transaction.note = pfbudget.t.Note(note) + + case other: + if other not in [c.name for c in categories]: + print(f"{other} is not a valid category") + continue + + transaction.category = pfbudget.t.TransactionCategory( + other, + pfbudget.t.CategorySelector(pfbudget.t.Selector_T.manual), + ) + next = False + + if quit: + break + + if __name__ == "__main__": argparser = pfbudget.argparser() args = vars(argparser.parse_args()) @@ -16,6 +59,10 @@ if __name__ == "__main__": params = [] match (op): + case pfbudget.Operation.ManualCategorization: + interactive(pfbudget.Manager(db, verbosity)) + exit() + case pfbudget.Operation.Parse: keys = {"path", "bank", "creditcard"} assert args.keys() >= keys, f"missing {args.keys() - keys}" diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 664f3b8..1e0e60a 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -33,21 +33,6 @@ class Categorizer: self._rule_based_categories(transactions, categories) self._rule_based_tags(transactions, tags) - def manual( - self, - transactions: Sequence[t.Transaction], - categories: Sequence[t.Category], - tags: Sequence[t.Tag], - ): - """Manual categorization input - - Args: - transactions (Sequence[Transaction]): uncategorized transactions - categories (Sequence[Category]): available categories - tags (Sequence[Tag]): currently available tags - """ - self._manual(transactions) - @Timer(name="nullify") def _nullify(self, transactions: Sequence[t.BankTransaction]): count = 0 @@ -151,21 +136,3 @@ class Categorizer: for k, v in d.items(): print(f"{v}: {k}") - - def _manual(self, transactions: Sequence[t.Transaction]): - uncategorized = [t for t in transactions if not t.category] - print(f"{len(uncategorized)} transactions left to categorize") - - for transaction in uncategorized: - while True: - category = input(f"{transaction} category: ") - if category == "quit": - return - if not category: - print("{category} doesn't exist") - continue - transaction.category = t.TransactionCategory( - category, t.CategorySelector(t.Selector_T.manual) - ) - - break diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 333b07d..e564cc8 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -88,15 +88,6 @@ class Manager: tags = session.get(Tag) Categorizer().rules(uncategorized, categories, tags) - case Operation.ManualCategorization: - with self.db.session() as session: - uncategorized = session.get( - Transaction, ~Transaction.category.has() - ) - categories = session.get(Category) - tags = session.get(Tag) - Categorizer().manual(uncategorized, categories, tags) - case Operation.BankMod: with self.db.session() as session: session.update(Bank, params) From fd24ac3318b0800e2f4fc565070c4dacf17e8782 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 23 Jan 2023 23:49:16 +0000 Subject: [PATCH 50/66] Remove unnecessary __init__.pys and import using submodules in __main__.py --- pfbudget/__init__.py | 10 --- pfbudget/__main__.py | 136 ++++++++++++++++----------------- pfbudget/cli/__init__.py | 0 pfbudget/cli/runnable.py | 18 +---- pfbudget/common/__init__.py | 0 pfbudget/core/__init__.py | 0 pfbudget/input/__init__.py | 0 pfbudget/input/nordigen.py | 4 +- pfbudget/reporting/__init__.py | 0 pfbudget/utils/__init__.py | 2 - 10 files changed, 72 insertions(+), 98 deletions(-) delete mode 100644 pfbudget/cli/__init__.py delete mode 100644 pfbudget/common/__init__.py delete mode 100644 pfbudget/core/__init__.py delete mode 100644 pfbudget/input/__init__.py delete mode 100644 pfbudget/reporting/__init__.py delete mode 100644 pfbudget/utils/__init__.py diff --git a/pfbudget/__init__.py b/pfbudget/__init__.py index ed82db6..d066bc8 100644 --- a/pfbudget/__init__.py +++ b/pfbudget/__init__.py @@ -1,12 +1,2 @@ -__all__ = ["argparser", "Manager", "parse_data", "categorize_data"] __author__ = "Luís Murta" __version__ = "0.1" - -from pfbudget.common.types import Operation -from pfbudget.core.categories import categorize_data -from pfbudget.core.manager import Manager -from pfbudget.cli.runnable import argparser -from pfbudget.input.parsers import parse_data -from pfbudget.utils.utils import parse_args_period - -import pfbudget.db.model as t diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 1032684..56fdccd 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,14 +1,16 @@ -import pfbudget +from pfbudget.cli.runnable import argparser +from pfbudget.common.types import Operation +from pfbudget.core.manager import Manager +import pfbudget.db.model as type +from pfbudget.utils.utils import parse_args_period -def interactive(manager: pfbudget.Manager): +def interactive(manager: Manager): with manager.db.session() as session: - categories = session.get(pfbudget.t.Category) + categories = session.get(type.Category) print(f"Available categories: {categories}") - print(f"Available tags: {session.get(pfbudget.t.Tag)}") - transactions = session.get( - pfbudget.t.Transaction, ~pfbudget.t.Transaction.category.has() - ) + print(f"Available tags: {session.get(type.Tag)}") + transactions = session.get(type.Transaction, ~type.Transaction.category.has()) print(f"{len(transactions)} transactions left to categorize") for transaction in sorted(transactions): @@ -23,20 +25,20 @@ def interactive(manager: pfbudget.Manager): case "tag": tag = input("tag: ") - transaction.tags.add(pfbudget.t.TransactionTag(tag)) + transaction.tags.add(type.TransactionTag(tag)) case "note": note = input("note: ") - transaction.note = pfbudget.t.Note(note) + transaction.note = type.Note(note) case other: if other not in [c.name for c in categories]: print(f"{other} is not a valid category") continue - transaction.category = pfbudget.t.TransactionCategory( + transaction.category = type.TransactionCategory( other, - pfbudget.t.CategorySelector(pfbudget.t.Selector_T.manual), + type.CategorySelector(type.Selector_T.manual), ) next = False @@ -45,11 +47,11 @@ def interactive(manager: pfbudget.Manager): if __name__ == "__main__": - argparser = pfbudget.argparser() + argparser = argparser() args = vars(argparser.parse_args()) - assert "op" in args, "No pfbudget.Operation selected" - op: pfbudget.Operation = args.pop("op") + assert "op" in args, "No Operation selected" + op: Operation = args.pop("op") assert "database" in args, "No database selected" db = args.pop("database") @@ -59,27 +61,27 @@ if __name__ == "__main__": params = [] match (op): - case pfbudget.Operation.ManualCategorization: - interactive(pfbudget.Manager(db, verbosity)) + case Operation.ManualCategorization: + interactive(Manager(db, verbosity)) exit() - case pfbudget.Operation.Parse: + case Operation.Parse: keys = {"path", "bank", "creditcard"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [args["path"], args["bank"], args["creditcard"]] - case pfbudget.Operation.RequisitionId: + case Operation.RequisitionId: keys = {"name", "country"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [args["name"][0], args["country"][0]] - case pfbudget.Operation.Download: + case Operation.Download: keys = {"all", "banks", "interval", "start", "end", "year", "dry_run"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - start, end = pfbudget.parse_args_period(args) + start, end = parse_args_period(args) params = [start, end, args["dry_run"]] if not args["all"]: @@ -87,19 +89,19 @@ if __name__ == "__main__": else: params.append([]) - case pfbudget.Operation.BankAdd: + case Operation.BankAdd: keys = {"bank", "bic", "type"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.t.Bank( + type.Bank( args["bank"][0], args["bic"][0], args["type"][0], ) ] - case pfbudget.Operation.BankMod: + case Operation.BankMod: keys = {"bank", "bic", "type", "remove"} assert args.keys() >= keys, f"missing {args.keys() - keys}" @@ -111,16 +113,16 @@ if __name__ == "__main__": params = [param] - case pfbudget.Operation.BankDel: + case Operation.BankDel: assert len(args["bank"]) > 0, "argparser ill defined" params = args["bank"] - case pfbudget.Operation.NordigenAdd: + case Operation.NordigenAdd: keys = {"bank", "bank_id", "requisition_id", "invert"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.t.Nordigen( + type.Nordigen( args["bank"][0], args["bank_id"][0] if args["bank_id"] else None, args["requisition_id"][0] if args["requisition_id"] else None, @@ -128,7 +130,7 @@ if __name__ == "__main__": ) ] - case pfbudget.Operation.NordigenMod: + case Operation.NordigenMod: keys = {"bank", "bank_id", "requisition_id", "invert", "remove"} assert args.keys() >= keys, f"missing {args.keys() - keys}" @@ -142,52 +144,50 @@ if __name__ == "__main__": params = [param] - case pfbudget.Operation.NordigenDel: + case Operation.NordigenDel: assert len(args["bank"]) > 0, "argparser ill defined" params = args["bank"] - case pfbudget.Operation.NordigenCountryBanks: + case Operation.NordigenCountryBanks: keys = {"country"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [args["country"][0]] - case pfbudget.Operation.CategoryAdd: + case Operation.CategoryAdd: keys = {"category", "group"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [ - pfbudget.t.Category(cat, args["group"]) for cat in args["category"] - ] + params = [type.Category(cat, args["group"]) for cat in args["category"]] - case pfbudget.Operation.CategoryUpdate: + case Operation.CategoryUpdate: keys = {"category", "group"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [pfbudget.t.Category(cat) for cat in args["category"]] + params = [type.Category(cat) for cat in args["category"]] params.append(args["group"]) - case pfbudget.Operation.CategoryRemove: + case Operation.CategoryRemove: assert "category" in args, "argparser ill defined" - params = [pfbudget.t.Category(cat) for cat in args["category"]] + params = [type.Category(cat) for cat in args["category"]] - case pfbudget.Operation.CategorySchedule: + case Operation.CategorySchedule: keys = {"category", "period", "frequency"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.t.CategorySchedule( + type.CategorySchedule( cat, args["period"][0], args["frequency"][0], None ) for cat in args["category"] ] - case pfbudget.Operation.RuleAdd: + case Operation.RuleAdd: keys = {"category", "date", "description", "bank", "min", "max"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.t.CategoryRule( + type.CategoryRule( args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, @@ -199,13 +199,13 @@ if __name__ == "__main__": for cat in args["category"] ] - case pfbudget.Operation.RuleRemove | pfbudget.Operation.TagRuleRemove: + case Operation.RuleRemove | Operation.TagRuleRemove: keys = {"id"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = args["id"] - case pfbudget.Operation.RuleModify: + case Operation.RuleModify: keys = { "id", "category", @@ -227,18 +227,18 @@ if __name__ == "__main__": params.append(param) - case pfbudget.Operation.TagAdd: + case Operation.TagAdd: keys = {"tag"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [pfbudget.t.Tag(tag) for tag in args["tag"]] + params = [type.Tag(tag) for tag in args["tag"]] - case pfbudget.Operation.TagRuleAdd: + case Operation.TagRuleAdd: keys = {"tag", "date", "description", "bank", "min", "max"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ - pfbudget.t.TagRule( + type.TagRule( args["date"][0] if args["date"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, @@ -250,7 +250,7 @@ if __name__ == "__main__": for tag in args["tag"] ] - case pfbudget.Operation.TagRuleModify: + case Operation.TagRuleModify: keys = {"id", "tag", "date", "description", "bank", "min", "max", "remove"} assert args.keys() >= keys, f"missing {args.keys() - keys}" @@ -263,39 +263,37 @@ if __name__ == "__main__": params.append(param) - case pfbudget.Operation.GroupAdd: + case Operation.GroupAdd: assert "group" in args, "argparser ill defined" - params = [pfbudget.t.CategoryGroup(group) for group in args["group"]] + params = [type.CategoryGroup(group) for group in args["group"]] - case pfbudget.Operation.GroupRemove: + case Operation.GroupRemove: assert "group" in args, "argparser ill defined" - params = [pfbudget.t.CategoryGroup(group) for group in args["group"]] + params = [type.CategoryGroup(group) for group in args["group"]] - case pfbudget.Operation.Forge | pfbudget.Operation.Dismantle: + case Operation.Forge | Operation.Dismantle: keys = {"original", "links"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [ - pfbudget.t.Link(args["original"][0], link) for link in args["links"] - ] + params = [type.Link(args["original"][0], link) for link in args["links"]] case ( - pfbudget.Operation.Export - | pfbudget.Operation.Import - | pfbudget.Operation.ExportBanks - | pfbudget.Operation.ImportBanks - | pfbudget.Operation.ExportCategoryRules - | pfbudget.Operation.ImportCategoryRules - | pfbudget.Operation.ExportTagRules - | pfbudget.Operation.ImportTagRules - | pfbudget.Operation.ExportCategories - | pfbudget.Operation.ImportCategories - | pfbudget.Operation.ExportCategoryGroups - | pfbudget.Operation.ImportCategoryGroups + Operation.Export + | Operation.Import + | Operation.ExportBanks + | Operation.ImportBanks + | Operation.ExportCategoryRules + | Operation.ImportCategoryRules + | Operation.ExportTagRules + | Operation.ImportTagRules + | Operation.ExportCategories + | Operation.ImportCategories + | Operation.ExportCategoryGroups + | Operation.ImportCategoryGroups ): keys = {"file"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = args["file"] - pfbudget.Manager(db, verbosity).action(op, params) + Manager(db, verbosity).action(op, params) diff --git a/pfbudget/cli/__init__.py b/pfbudget/cli/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/runnable.py index e174224..9029920 100644 --- a/pfbudget/cli/runnable.py +++ b/pfbudget/cli/runnable.py @@ -11,25 +11,13 @@ from pfbudget.db.model import AccountType, Period from pfbudget.db.sqlite import DatabaseClient import pfbudget.reporting.graph import pfbudget.reporting.report -import pfbudget.utils +import pfbudget.utils.utils load_dotenv() DEFAULT_DB = os.environ.get("DEFAULT_DB") -class PfBudgetInitialized(Exception): - pass - - -class PfBudgetNotInitialized(Exception): - pass - - -class DataFileMissing(Exception): - pass - - def argparser() -> argparse.ArgumentParser: universal = argparse.ArgumentParser(add_help=False) universal.add_argument( @@ -175,7 +163,7 @@ def graph(args): Args: args (dict): argparse variables """ - start, end = pfbudget.utils.parse_args_period(args) + start, end = pfbudget.utils.utils.parse_args_period(args) if args.option == "monthly": pfbudget.reporting.graph.monthly( DatabaseClient(args.database), vars(args), start, end @@ -196,7 +184,7 @@ def report(args): Args: args (dict): argparse variables """ - start, end = pfbudget.utils.parse_args_period(args) + start, end = pfbudget.utils.utils.parse_args_period(args) if args.option == "net": pfbudget.reporting.report.net(DatabaseClient(args.database), start, end) elif args.option == "detailed": diff --git a/pfbudget/common/__init__.py b/pfbudget/common/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pfbudget/core/__init__.py b/pfbudget/core/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pfbudget/input/__init__.py b/pfbudget/input/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index 5fa9453..a28027b 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -8,7 +8,7 @@ import time import uuid import pfbudget.db.model as t -import pfbudget.utils as utils +from pfbudget.utils.converters import convert from .input import Input @@ -72,7 +72,7 @@ class NordigenInput(Input): json.dump(downloaded, f) converted = [ - utils.convert(t, bank) for t in downloaded["transactions"]["booked"] + convert(t, bank) for t in downloaded["transactions"]["booked"] ] transactions.extend( diff --git a/pfbudget/reporting/__init__.py b/pfbudget/reporting/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/pfbudget/utils/__init__.py b/pfbudget/utils/__init__.py deleted file mode 100644 index e01cf89..0000000 --- a/pfbudget/utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .converters import convert -from .utils import * From 13709b7a04ae7f3f62fd930d71b5c92478c1e98c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 21:14:49 +0000 Subject: [PATCH 51/66] [Interactive] Adds new tag --- pfbudget/__main__.py | 10 ++++++++-- pfbudget/core/manager.py | 9 ++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 56fdccd..b8f312e 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -7,9 +7,12 @@ from pfbudget.utils.utils import parse_args_period def interactive(manager: Manager): with manager.db.session() as session: + categories = session.get(type.Category) - print(f"Available categories: {categories}") - print(f"Available tags: {session.get(type.Tag)}") + print(f"Available categories: {[c.name for c in categories]}") + tags = session.get(type.Tag) + print(f"Available tags: {[t.name for t in tags]}") + transactions = session.get(type.Transaction, ~type.Transaction.category.has()) print(f"{len(transactions)} transactions left to categorize") @@ -25,6 +28,9 @@ def interactive(manager: Manager): case "tag": tag = input("tag: ") + if tag not in [t.name for t in tags]: + session.add([type.Tag(tag)]) + transaction.tags.add(type.TransactionTag(tag)) case "note": diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index e564cc8..ba7c6f2 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -116,7 +116,14 @@ class Manager: banks = NordigenInput().country_banks(params[0]) print(banks) - case Operation.BankAdd | Operation.CategoryAdd | Operation.NordigenAdd | Operation.RuleAdd | Operation.TagAdd | Operation.TagRuleAdd: + case ( + Operation.BankAdd + | Operation.CategoryAdd + | Operation.NordigenAdd + | Operation.RuleAdd + | Operation.TagAdd + | Operation.TagRuleAdd + ): with self.db.session() as session: session.add(params) From e0fc310ef6a5233641f28e736dc95cd5c3f24096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 21:16:05 +0000 Subject: [PATCH 52/66] [Fix] Add split default --- pfbudget/db/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index 2be6a07..fa2a33e 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -91,7 +91,7 @@ class Transaction(Base, Export): description: Mapped[Optional[str]] amount: Mapped[money] - split: Mapped[bool] = mapped_column(init=False) + split: Mapped[bool] = mapped_column(init=False, default=False) type: Mapped[str] = mapped_column(init=False) From abff76ad4e496da19527cfc087fc5859965c355d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 21:22:07 +0000 Subject: [PATCH 53/66] [Interactive] Finish split command --- pfbudget/__main__.py | 43 ++++++++++++++++++++++++++++++++++++ pfbudget/core/categorizer.py | 3 +++ 2 files changed, 46 insertions(+) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index b8f312e..df8dd3a 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,3 +1,6 @@ +from decimal import Decimal +from typing import Sequence + from pfbudget.cli.runnable import argparser from pfbudget.common.types import Operation from pfbudget.core.manager import Manager @@ -26,6 +29,10 @@ def interactive(manager: Manager): next = False quit = True + case "split": + manager.action(Operation.Split, split(transaction, categories)) + next = False + case "tag": tag = input("tag: ") if tag not in [t.name for t in tags]: @@ -48,10 +55,46 @@ def interactive(manager: Manager): ) next = False + session.commit() if quit: break +def split( + original: type.Transaction, categories: Sequence[type.Category] +) -> list[type.Transaction]: + + total = original.amount + splitted: list[type.Transaction] = [] + + while True: + if abs(sum(t.amount for t in splitted)) > abs(total): + print( + "The total amount from the splitted transactions exceeds the original transaction amount, please try again..." + ) + splitted.clear() + + if sum(t.amount for t in splitted) == total: + break + + while (category := input("New transaction category: ")) not in [ + c.name for c in categories + ]: + print(f"{category} is not a valid category") + + amount = input("amount: ") + + split = type.Transaction(original.date, original.description, Decimal(amount)) + split.category = type.TransactionCategory( + category, type.CategorySelector(type.Selector_T.manual) + ) + + splitted.append(split) + + splitted.insert(0, original) + return splitted + + if __name__ == "__main__": argparser = argparser() args = vars(argparser.parse_args()) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 1e0e60a..c15ebe0 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -35,6 +35,7 @@ class Categorizer: @Timer(name="nullify") def _nullify(self, transactions: Sequence[t.BankTransaction]): + print(f"Nullifying {len(transactions)} transactions") count = 0 matching = [] for transaction in transactions: @@ -72,6 +73,7 @@ class Categorizer: transactions: Sequence[t.BankTransaction], categories: Sequence[t.Category], ): + print(f"Categorizing {len(transactions)} transactions") d = {} for category in [c for c in categories if c.rules]: for rule in category.rules: @@ -112,6 +114,7 @@ class Categorizer: def _rule_based_tags( self, transactions: Sequence[t.BankTransaction], tags: Sequence[t.Tag] ): + print(f"Tagging {len(transactions)} transactions") d = {} for tag in [t for t in tags if len(t.rules) > 0]: for rule in tag.rules: From f943374adeb300ed51766971daf2b9cf09556da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 23:39:46 +0000 Subject: [PATCH 54/66] [Fix] Splitted transactions do not have category Adds uncategorized method to the DB client to retrieve transactions w/o a category AND not splitted. --- pfbudget/__main__.py | 2 +- pfbudget/db/client.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index df8dd3a..2d42e32 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -16,7 +16,7 @@ def interactive(manager: Manager): tags = session.get(type.Tag) print(f"Available tags: {[t.name for t in tags]}") - transactions = session.get(type.Transaction, ~type.Transaction.category.has()) + transactions = session.uncategorized() print(f"{len(transactions)} transactions left to categorize") for transaction in sorted(transactions): diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index cf42b0f..7d1fbe5 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -2,6 +2,7 @@ from dataclasses import asdict from sqlalchemy import create_engine, delete, select, update from sqlalchemy.dialects.postgresql import insert from sqlalchemy.orm import Session +from sqlalchemy.sql.expression import false from typing import Sequence, Type, TypeVar from pfbudget.db.model import ( @@ -9,6 +10,7 @@ from pfbudget.db.model import ( CategoryGroup, CategorySchedule, Link, + Transaction, ) @@ -57,6 +59,22 @@ class DbClient: return self.__session.scalars(stmt).all() + def uncategorized(self) -> Sequence[Transaction]: + """Selects all valid uncategorized transactions + At this moment that includes: + - Categories w/o category + - AND non-split categories + + Returns: + Sequence[Transaction]: transactions left uncategorized + """ + stmt = ( + select(Transaction) + .where(~Transaction.category.has()) + .where(Transaction.split == false()) + ) + return self.__session.scalars(stmt).all() + def add(self, rows: list): self.__session.add_all(rows) From 2da721d53c82b7735259af4b3e3576148d451b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 23:41:42 +0000 Subject: [PATCH 55/66] [Interactive] Adds skip command --- pfbudget/__main__.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 2d42e32..e0fbce8 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -24,7 +24,11 @@ def interactive(manager: Manager): quit = False next = True while next: - match (input("(/split/tag/note/quit): ")): + match (input("(/split/tag/note/skip/quit): ")): + case "skip": + next = False + continue + case "quit" | "exit": next = False quit = True From da348c4ffb3aaae08c0b6dc9597be9b093c3ec8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 23:43:09 +0000 Subject: [PATCH 56/66] [Fix] Splitted categories were not being commited --- pfbudget/core/manager.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index ba7c6f2..f620220 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -197,12 +197,20 @@ class Manager: assert len(originals) == 1, ">1 transactions matched {original.id}!" originals[0].split = True - transactions = [ - SplitTransaction( - originals[0].date, t.description, t.amount, originals[0].id + transactions = [] + for t in params[1:]: + if originals[0].date != t.date: + t.date = originals[0].date + print( + f"{t.date} is different from original date {originals[0].date}, using original" + ) + + splitted = SplitTransaction( + t.date, t.description, t.amount, originals[0].id ) - for t in params[1:] - ] + splitted.category = t.category + transactions.append(splitted) + session.add(transactions) case Operation.Export: From 7453ffbd3aaea2e44a3ea49b18f791bb1424f20e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sun, 29 Jan 2023 23:43:56 +0000 Subject: [PATCH 57/66] [Interactive] Adds new format for tags Now tags can be defined along side categories by starting with :. E.g. eating out:work:today will classify the transaction with the eating out category and work and today tags. --- pfbudget/__main__.py | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index e0fbce8..c7ab0fe 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -24,7 +24,7 @@ def interactive(manager: Manager): quit = False next = True while next: - match (input("(/split/tag/note/skip/quit): ")): + match (input("((:tag)/split/note/skip/quit): ")): case "skip": next = False continue @@ -34,29 +34,37 @@ def interactive(manager: Manager): quit = True case "split": - manager.action(Operation.Split, split(transaction, categories)) + manager.action(Operation.Split, split(transaction, categories, tags)) next = False - case "tag": - tag = input("tag: ") - if tag not in [t.name for t in tags]: - session.add([type.Tag(tag)]) - - transaction.tags.add(type.TransactionTag(tag)) - case "note": note = input("note: ") transaction.note = type.Note(note) case other: - if other not in [c.name for c in categories]: - print(f"{other} is not a valid category") + if len(li := other.split(":")) > 1: + _category = li[0] + _tags = li[1:] + else: + _category = other + _tags = [] + + if _category not in [c.name for c in categories]: + print(f"{other} doesn't have a valid category") continue transaction.category = type.TransactionCategory( - other, + _category, type.CategorySelector(type.Selector_T.manual), ) + + for tag in _tags: + if tag not in [t.name for t in tags]: + session.add([type.Tag(tag)]) + tags = session.get(type.Tag) + + transaction.tags.add(type.TransactionTag(tag)) + next = False session.commit() @@ -65,7 +73,9 @@ def interactive(manager: Manager): def split( - original: type.Transaction, categories: Sequence[type.Category] + original: type.Transaction, + categories: Sequence[type.Category], + tags: Sequence[type.Tag], ) -> list[type.Transaction]: total = original.amount From 23eb2c80bdfb61cb05ca35f46ad64821bcb84338 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 30 Jan 2023 22:24:23 +0000 Subject: [PATCH 58/66] [Interactive] Defines an Interactive class Affords a cleaner coding over the function. Renames the runnable.py into what it actually is, the argparser.py. --- pfbudget/__main__.py | 106 +----------------- pfbudget/cli/{runnable.py => argparser.py} | 0 pfbudget/cli/interactive.py | 120 +++++++++++++++++++++ pfbudget/db/model.py | 9 +- 4 files changed, 130 insertions(+), 105 deletions(-) rename pfbudget/cli/{runnable.py => argparser.py} (100%) create mode 100644 pfbudget/cli/interactive.py diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index c7ab0fe..228a83b 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,114 +1,14 @@ from decimal import Decimal from typing import Sequence -from pfbudget.cli.runnable import argparser +from pfbudget.cli.argparser import argparser +from pfbudget.cli.interactive import Interactive from pfbudget.common.types import Operation from pfbudget.core.manager import Manager import pfbudget.db.model as type from pfbudget.utils.utils import parse_args_period -def interactive(manager: Manager): - with manager.db.session() as session: - - categories = session.get(type.Category) - print(f"Available categories: {[c.name for c in categories]}") - tags = session.get(type.Tag) - print(f"Available tags: {[t.name for t in tags]}") - - transactions = session.uncategorized() - print(f"{len(transactions)} transactions left to categorize") - - for transaction in sorted(transactions): - print(f"{transaction}") - quit = False - next = True - while next: - match (input("((:tag)/split/note/skip/quit): ")): - case "skip": - next = False - continue - - case "quit" | "exit": - next = False - quit = True - - case "split": - manager.action(Operation.Split, split(transaction, categories, tags)) - next = False - - case "note": - note = input("note: ") - transaction.note = type.Note(note) - - case other: - if len(li := other.split(":")) > 1: - _category = li[0] - _tags = li[1:] - else: - _category = other - _tags = [] - - if _category not in [c.name for c in categories]: - print(f"{other} doesn't have a valid category") - continue - - transaction.category = type.TransactionCategory( - _category, - type.CategorySelector(type.Selector_T.manual), - ) - - for tag in _tags: - if tag not in [t.name for t in tags]: - session.add([type.Tag(tag)]) - tags = session.get(type.Tag) - - transaction.tags.add(type.TransactionTag(tag)) - - next = False - - session.commit() - if quit: - break - - -def split( - original: type.Transaction, - categories: Sequence[type.Category], - tags: Sequence[type.Tag], -) -> list[type.Transaction]: - - total = original.amount - splitted: list[type.Transaction] = [] - - while True: - if abs(sum(t.amount for t in splitted)) > abs(total): - print( - "The total amount from the splitted transactions exceeds the original transaction amount, please try again..." - ) - splitted.clear() - - if sum(t.amount for t in splitted) == total: - break - - while (category := input("New transaction category: ")) not in [ - c.name for c in categories - ]: - print(f"{category} is not a valid category") - - amount = input("amount: ") - - split = type.Transaction(original.date, original.description, Decimal(amount)) - split.category = type.TransactionCategory( - category, type.CategorySelector(type.Selector_T.manual) - ) - - splitted.append(split) - - splitted.insert(0, original) - return splitted - - if __name__ == "__main__": argparser = argparser() args = vars(argparser.parse_args()) @@ -125,7 +25,7 @@ if __name__ == "__main__": params = [] match (op): case Operation.ManualCategorization: - interactive(Manager(db, verbosity)) + Interactive(Manager(db, verbosity)).start() exit() case Operation.Parse: diff --git a/pfbudget/cli/runnable.py b/pfbudget/cli/argparser.py similarity index 100% rename from pfbudget/cli/runnable.py rename to pfbudget/cli/argparser.py diff --git a/pfbudget/cli/interactive.py b/pfbudget/cli/interactive.py new file mode 100644 index 0000000..f0c3c7e --- /dev/null +++ b/pfbudget/cli/interactive.py @@ -0,0 +1,120 @@ +import decimal + +from ..core.manager import Manager +from ..db.model import ( + Category, + CategorySelector, + Note, + Selector_T, + SplitTransaction, + Tag, + Transaction, + TransactionCategory, + TransactionTag, +) + + +class Interactive: + help = "category(:tag)/split/note:/skip/quit" + selector = Selector_T.manual + + def __init__(self, manager: Manager) -> None: + self.manager = manager + + with self.manager.db.session() as session: + self.categories = session.get(Category) + self.tags = session.get(Tag) + session.expunge_all() + + def intro(self) -> None: + print( + f"Welcome! Available categories are {[c.name for c in self.categories]} and" + f" currently existing tags are {[t.name for t in self.tags]}" + ) + + def start(self) -> None: + self.intro() + + with self.manager.db.session() as session: + uncategorized = session.uncategorized() + n = len(uncategorized) + print(f"{n} left to categorize") + + i = 0 + new = [] + next = uncategorized[i] + print(next) + while (command := input("$ ")) != "quit": + match command: + case "help": + print(self.help) + + case "skip": + i += 1 + + case "quit": + break + + case "split": + new = self.split(next) + session.add(new) + + case other: + if not other: + print(self.help) + continue + + if other.startswith("note:"): + # TODO adding notes to a splitted transaction won't allow categorization + next.note = Note(other[len("note:") :].strip()) + else: + ct = other.split(":") + if (category := ct[0]) not in [ + c.name for c in self.categories + ]: + print(self.help, self.categories) + + tags = [] + if len(ct) > 1: + tags = ct[1:] + + next.category = TransactionCategory( + category, CategorySelector(self.selector) + ) + for tag in tags: + if tag not in [t.name for t in self.tags]: + session.add([Tag(tag)]) + self.tags = session.get(Tag) + + next.tags.add(TransactionTag(tag)) + + i += 1 + + session.commit() + + next = uncategorized[i] if len(new) == 0 else new.pop() + print(next) + + def split(self, original: Transaction) -> list[SplitTransaction]: + total = original.amount + new = [] + + done = False + while not done: + if abs(sum(t.amount for t in new)) > abs(total): + print("Overflow, try again") + new.clear() + continue + + if sum(t.amount for t in new) == total: + done = True + break + + amount = decimal.Decimal(input("amount: ")) + new.append( + SplitTransaction( + original.date, original.description, amount, original.id + ) + ) + + return new diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index fa2a33e..dcd1969 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -96,7 +96,9 @@ class Transaction(Base, Export): type: Mapped[str] = mapped_column(init=False) category: Mapped[Optional[TransactionCategory]] = relationship(init=False) - note: Mapped[Optional[Note]] = relationship(init=False) + note: Mapped[Optional[Note]] = relationship( + cascade="all, delete-orphan", init=False, passive_deletes=True + ) tags: Mapped[set[TransactionTag]] = relationship(init=False) __mapper_args__ = {"polymorphic_on": "type", "polymorphic_identity": "transaction"} @@ -175,7 +177,10 @@ class Category(Base, Export): ) def __repr__(self) -> str: - return f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)}, schedule={self.schedule})" + return ( + f"Category(name={self.name}, group={self.group}, #rules={len(self.rules)}," + f" schedule={self.schedule})" + ) @property def format(self) -> dict[str, Any]: From f7df033d5834930c093a6c47e43b0e982ddad60e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Mon, 6 Feb 2023 22:10:53 +0000 Subject: [PATCH 59/66] Add start date rule Rename date to end. --- .../952de57a3c43_start_end_date_rule.py | 32 +++++++++++ pfbudget/__main__.py | 13 +++-- pfbudget/cli/argparser.py | 3 +- pfbudget/db/model.py | 53 +++++++++++-------- 4 files changed, 70 insertions(+), 31 deletions(-) create mode 100644 alembic/versions/952de57a3c43_start_end_date_rule.py diff --git a/alembic/versions/952de57a3c43_start_end_date_rule.py b/alembic/versions/952de57a3c43_start_end_date_rule.py new file mode 100644 index 0000000..e9590b8 --- /dev/null +++ b/alembic/versions/952de57a3c43_start_end_date_rule.py @@ -0,0 +1,32 @@ +"""Start/End date rule + +Revision ID: 952de57a3c43 +Revises: 18572111d9ff +Create Date: 2023-02-06 21:57:57.545327+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "952de57a3c43" +down_revision = "18572111d9ff" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "rules", sa.Column("start", sa.Date(), nullable=True), schema="transactions" + ) + op.alter_column("rules", column_name="date", new_column_name="end", schema="transactions") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column("rules", column_name="end", new_column_name="date", schema="transactions") + op.drop_column("rules", "start", schema="transactions") + # ### end Alembic commands ### diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 228a83b..0132d84 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -1,6 +1,3 @@ -from decimal import Decimal -from typing import Sequence - from pfbudget.cli.argparser import argparser from pfbudget.cli.interactive import Interactive from pfbudget.common.types import Operation @@ -146,12 +143,13 @@ if __name__ == "__main__": ] case Operation.RuleAdd: - keys = {"category", "date", "description", "bank", "min", "max"} + keys = {"category", "start", "end", "description", "regex", "bank", "min", "max"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ type.CategoryRule( - args["date"][0] if args["date"] else None, + args["start"][0] if args["start"] else None, + args["end"][0] if args["end"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, args["bank"][0] if args["bank"] else None, @@ -197,12 +195,13 @@ if __name__ == "__main__": params = [type.Tag(tag) for tag in args["tag"]] case Operation.TagRuleAdd: - keys = {"tag", "date", "description", "bank", "min", "max"} + keys = {"tag", "start", "end", "description", "regex", "bank", "min", "max"} assert args.keys() >= keys, f"missing {args.keys() - keys}" params = [ type.TagRule( - args["date"][0] if args["date"] else None, + args["start"][0] if args["start"] else None, + args["end"][0] if args["end"] else None, args["description"][0] if args["description"] else None, args["regex"][0] if args["regex"] else None, args["bank"][0] if args["bank"] else None, diff --git a/pfbudget/cli/argparser.py b/pfbudget/cli/argparser.py index 9029920..1e47aa6 100644 --- a/pfbudget/cli/argparser.py +++ b/pfbudget/cli/argparser.py @@ -374,7 +374,8 @@ def tag_rule(parser: argparse.ArgumentParser): def rules(parser: argparse.ArgumentParser): - parser.add_argument("--date", nargs=1, type=dt.date.fromisoformat) + parser.add_argument("--start", nargs=1, type=dt.date.fromisoformat) + parser.add_argument("--end", nargs=1, type=dt.date.fromisoformat) parser.add_argument("--description", nargs=1, type=str) parser.add_argument("--regex", nargs=1, type=str) parser.add_argument("--bank", nargs=1, type=str) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index dcd1969..eb2d712 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -335,7 +335,8 @@ class Rule(Base, Export): __tablename__ = "rules" id: Mapped[idpk] = mapped_column(init=False) - date: Mapped[Optional[dt.date]] + start: Mapped[Optional[dt.date]] + end: Mapped[Optional[dt.date]] description: Mapped[Optional[str]] regex: Mapped[Optional[str]] bank: Mapped[Optional[str]] @@ -349,32 +350,34 @@ class Rule(Base, Export): "polymorphic_on": "type", } - def matches(self, transaction: BankTransaction) -> bool: - if ( - (self.date and self.date < transaction.date) - or ( - self.description - and transaction.description - and self.description not in transaction.description - ) - or ( - self.regex - and transaction.description - and not re.compile(self.regex, re.IGNORECASE).search( - transaction.description - ) - ) - or (self.bank and self.bank != transaction.bank) - or (self.min and self.min > transaction.amount) - or (self.max and self.max < transaction.amount) - ): - return False - return True + def matches(self, t: BankTransaction) -> bool: + valid = None + if self.regex: + valid = re.compile(self.regex, re.IGNORECASE) + + ops = ( + Rule.exists(self.start, lambda r: r < t.date), + Rule.exists(self.end, lambda r: r > t.date), + Rule.exists(self.description, lambda r: r == t.description), + Rule.exists( + valid, + lambda r: r.search(t.description) if t.description else False, + ), + Rule.exists(self.bank, lambda r: r == t.bank), + Rule.exists(self.min, lambda r: r < t.amount), + Rule.exists(self.max, lambda r: r > t.amount), + ) + + if all(ops): + return True + + return False @property def format(self) -> dict[str, Any]: return dict( - date=self.date, + start=self.start, + end=self.end, description=self.description, regex=self.regex, bank=self.bank, @@ -383,6 +386,10 @@ class Rule(Base, Export): type=self.type, ) + @staticmethod + def exists(r, op) -> bool: + return op(r) if r is not None else True + class CategoryRule(Rule): __table_args__ = {"schema": "category"} From 6f68d971ee285bb899f640ceb7b0b547e2135293 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Sat, 11 Feb 2023 22:48:04 +0000 Subject: [PATCH 60/66] Clear up forge/dismantle logic --- pfbudget/__main__.py | 2 +- pfbudget/core/manager.py | 41 ++++++++++++++++++++++++++++++++++++++-- pfbudget/db/client.py | 5 ++++- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 0132d84..98fcb65 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -237,7 +237,7 @@ if __name__ == "__main__": keys = {"original", "links"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [type.Link(args["original"][0], link) for link in args["links"]] + params = [args["original"][0], args["links"]] case ( Operation.Export diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index f620220..27df90b 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -17,6 +17,7 @@ from pfbudget.db.model import ( MoneyTransaction, Nordigen, Rule, + Selector_T, SplitTransaction, Tag, TagRule, @@ -168,8 +169,32 @@ class Manager: session.remove_by_name(CategoryGroup, params) case Operation.Forge: + if not ( + isinstance(params[0], int) + and all(isinstance(p, int) for p in params[1]) + ): + raise TypeError("f{params} are not transaction ids") + with self.db.session() as session: - session.add(params) + original = session.get(Transaction, Transaction.id, params[0])[0] + links = session.get(Transaction, Transaction.id, params[1]) + + if not original.category: + original.category = self.askcategory(original) + + for link in links: + if ( + not link.category + or link.category.name != original.category.name + ): + print( + f"{link} category will change to" + f" {original.category.name}" + ) + link.category = original.category + + tobelinked = [Link(original.id, link.id) for link in links] + session.add(tobelinked) case Operation.Dismantle: assert all(isinstance(param, Link) for param in params) @@ -202,7 +227,8 @@ class Manager: if originals[0].date != t.date: t.date = originals[0].date print( - f"{t.date} is different from original date {originals[0].date}, using original" + f"{t.date} is different from original date" + f" {originals[0].date}, using original" ) splitted = SplitTransaction( @@ -326,6 +352,17 @@ class Manager: def parse(self, filename: Path, args: dict): return parse_data(filename, args) + def askcategory(self, transaction: Transaction): + selector = CategorySelector(Selector_T.manual) + + with self.db.session() as session: + categories = session.get(Category) + + while True: + category = input(f"{transaction}: ") + if category in [c.name for c in categories]: + return TransactionCategory(category, selector) + @staticmethod def dump(fn, sequence): with open(fn, "wb") as f: diff --git a/pfbudget/db/client.py b/pfbudget/db/client.py index 7d1fbe5..4c52820 100644 --- a/pfbudget/db/client.py +++ b/pfbudget/db/client.py @@ -51,7 +51,10 @@ class DbClient: def get(self, type: Type[T], column=None, values=None) -> Sequence[T]: if column is not None: if values: - stmt = select(type).where(column.in_(values)) + if isinstance(values, Sequence): + stmt = select(type).where(column.in_(values)) + else: + stmt = select(type).where(column == values) else: stmt = select(type).where(column) else: From dd724b6c287c089587ae9e79a1148e2cbb78a800 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 23 Feb 2023 23:21:54 +0000 Subject: [PATCH 61/66] Export in .csv Importing is not supported, since there's no way to represent a Null field in .csv --- pfbudget/__main__.py | 4 +-- pfbudget/cli/argparser.py | 25 ++++++++++--------- pfbudget/core/manager.py | 51 +++++++++++++++++++++++++-------------- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 98fcb65..7a28ef5 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -253,9 +253,9 @@ if __name__ == "__main__": | Operation.ExportCategoryGroups | Operation.ImportCategoryGroups ): - keys = {"file"} + keys = {"file", "format"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = args["file"] + params = [args["file"][0], args["format"][0]] Manager(db, verbosity).action(op, params) diff --git a/pfbudget/cli/argparser.py b/pfbudget/cli/argparser.py index 1e47aa6..298aa04 100644 --- a/pfbudget/cli/argparser.py +++ b/pfbudget/cli/argparser.py @@ -63,7 +63,7 @@ def argparser() -> argparse.ArgumentParser: # Exports transactions to .csv file export = subparsers.add_parser("export") export.set_defaults(op=Operation.Export) - export_args(export) + file_options(export) pimport = subparsers.add_parser("import") pimport.set_defaults(op=Operation.Import) @@ -215,11 +215,11 @@ def bank(parser: argparse.ArgumentParser): export = commands.add_parser("export") export.set_defaults(op=Operation.ExportBanks) - export_args(export) + file_options(export) pimport = commands.add_parser("import") pimport.set_defaults(op=Operation.ImportBanks) - export_args(pimport) + file_options(pimport) def nordigen(parser: argparse.ArgumentParser): @@ -276,11 +276,11 @@ def category(parser: argparse.ArgumentParser): export = commands.add_parser("export") export.set_defaults(op=Operation.ExportCategories) - export_args(export) + file_options(export) pimport = commands.add_parser("import") pimport.set_defaults(op=Operation.ImportCategories) - export_args(pimport) + file_options(pimport) def category_group(parser: argparse.ArgumentParser): @@ -296,11 +296,11 @@ def category_group(parser: argparse.ArgumentParser): export = commands.add_parser("export") export.set_defaults(op=Operation.ExportCategoryGroups) - export_args(export) + file_options(export) pimport = commands.add_parser("import") pimport.set_defaults(op=Operation.ImportCategoryGroups) - export_args(pimport) + file_options(pimport) def category_rule(parser: argparse.ArgumentParser): @@ -324,11 +324,11 @@ def category_rule(parser: argparse.ArgumentParser): export = commands.add_parser("export") export.set_defaults(op=Operation.ExportCategoryRules) - export_args(export) + file_options(export) pimport = commands.add_parser("import") pimport.set_defaults(op=Operation.ImportCategoryRules) - export_args(pimport) + file_options(pimport) def tags(parser: argparse.ArgumentParser): @@ -366,11 +366,11 @@ def tag_rule(parser: argparse.ArgumentParser): export = commands.add_parser("export") export.set_defaults(op=Operation.ExportTagRules) - export_args(export) + file_options(export) pimport = commands.add_parser("import") pimport.set_defaults(op=Operation.ImportTagRules) - export_args(pimport) + file_options(pimport) def rules(parser: argparse.ArgumentParser): @@ -397,5 +397,6 @@ def link(parser: argparse.ArgumentParser): dismantle.add_argument("links", nargs="+", type=int) -def export_args(parser: argparse.ArgumentParser): +def file_options(parser: argparse.ArgumentParser): parser.add_argument("file", nargs=1, type=str) + parser.add_argument("format", nargs=1, default="pickle") diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 27df90b..227a8f4 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -1,3 +1,4 @@ +import csv from pathlib import Path import pickle import webbrowser @@ -241,11 +242,11 @@ class Manager: case Operation.Export: with self.db.session() as session: - self.dump(params[0], sorted(session.get(Transaction))) + self.dump(params[0], params[1], sorted(session.get(Transaction))) case Operation.Import: transactions = [] - for row in self.load(params[0]): + for row in self.load(params[0], params[1]): match row["type"]: case "bank": transaction = BankTransaction( @@ -278,11 +279,11 @@ class Manager: case Operation.ExportBanks: with self.db.session() as session: - self.dump(params[0], session.get(Bank)) + self.dump(params[0], params[1], session.get(Bank)) case Operation.ImportBanks: banks = [] - for row in self.load(params[0]): + for row in self.load(params[0], params[1]): bank = Bank(row["name"], row["BIC"], row["type"]) if row["nordigen"]: bank.nordigen = Nordigen(**row["nordigen"]) @@ -294,10 +295,10 @@ class Manager: case Operation.ExportCategoryRules: with self.db.session() as session: - self.dump(params[0], session.get(CategoryRule)) + self.dump(params[0], params[1], session.get(CategoryRule)) case Operation.ImportCategoryRules: - rules = [CategoryRule(**row) for row in self.load(params[0])] + rules = [CategoryRule(**row) for row in self.load(params[0], params[1])] if self.certify(rules): with self.db.session() as session: @@ -305,10 +306,10 @@ class Manager: case Operation.ExportTagRules: with self.db.session() as session: - self.dump(params[0], session.get(TagRule)) + self.dump(params[0], params[1], session.get(TagRule)) case Operation.ImportTagRules: - rules = [TagRule(**row) for row in self.load(params[0])] + rules = [TagRule(**row) for row in self.load(params[0], params[1])] if self.certify(rules): with self.db.session() as session: @@ -316,12 +317,12 @@ class Manager: case Operation.ExportCategories: with self.db.session() as session: - self.dump(params[0], session.get(Category)) + self.dump(params[0], params[1], session.get(Category)) case Operation.ImportCategories: # rules = [Category(**row) for row in self.load(params[0])] categories = [] - for row in self.load(params[0]): + for row in self.load(params[0], params[1]): category = Category(row["name"], row["group"]) if len(row["rules"]) > 0: # Only category rules could have been created with a rule @@ -340,10 +341,12 @@ class Manager: case Operation.ExportCategoryGroups: with self.db.session() as session: - self.dump(params[0], session.get(CategoryGroup)) + self.dump(params[0], params[1], session.get(CategoryGroup)) case Operation.ImportCategoryGroups: - groups = [CategoryGroup(**row) for row in self.load(params[0])] + groups = [ + CategoryGroup(**row) for row in self.load(params[0], params[1]) + ] if self.certify(groups): with self.db.session() as session: @@ -364,14 +367,26 @@ class Manager: return TransactionCategory(category, selector) @staticmethod - def dump(fn, sequence): - with open(fn, "wb") as f: - pickle.dump([e.format for e in sequence], f) + def dump(fn, format, sequence): + if format == "pickle": + with open(fn, "wb") as f: + pickle.dump([e.format for e in sequence], f) + elif format == "csv": + with open(fn, "w", newline="") as f: + csv.writer(f).writerows([e.format.values() for e in sequence]) + else: + print("format not well specified") @staticmethod - def load(fn): - with open(fn, "rb") as f: - return pickle.load(f) + def load(fn, format): + if format == "pickle": + with open(fn, "rb") as f: + return pickle.load(f) + elif format == "csv": + raise Exception("CSV import not supported") + else: + print("format not well specified") + return [] @staticmethod def certify(imports: list) -> bool: From 1a774e3769d3328ea7037a91824cdc841fb326d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 23 Feb 2023 23:23:19 +0000 Subject: [PATCH 62/66] Adds get all transactions operation --- pfbudget/common/types.py | 1 + pfbudget/core/manager.py | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/pfbudget/common/types.py b/pfbudget/common/types.py index 0fe3efe..d999009 100644 --- a/pfbudget/common/types.py +++ b/pfbudget/common/types.py @@ -6,6 +6,7 @@ from enum import Enum, auto class Operation(Enum): Init = auto() + Transactions = auto() Parse = auto() Download = auto() Categorize = auto() diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 227a8f4..9435e90 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -34,14 +34,23 @@ class Manager: self._db = db self._verbosity = verbosity - def action(self, op: Operation, params: list): + def action(self, op: Operation, params=None): if self._verbosity > 0: print(f"op={op}, params={params}") + if params is None: + params = [] + match (op): case Operation.Init: pass + case Operation.Transactions: + with self.db.session() as session: + transactions = session.get(Transaction) + ret = [t.format for t in transactions] + return ret + case Operation.Parse: # Adapter for the parse_data method. Can be refactored. args = {"bank": params[1], "creditcard": params[2], "category": None} From ed2dda63e949198e20c7aa672c9cc7954da4194e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Thu, 23 Feb 2023 23:24:01 +0000 Subject: [PATCH 63/66] Allows using rules for the nullying step --- pfbudget/core/categorizer.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index c15ebe0..4194379 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -9,7 +9,7 @@ class Categorizer: options = {} def __init__(self): - self.options["null_days"] = 4 + self.options["null_days"] = 3 def rules( self, @@ -28,14 +28,20 @@ class Categorizer: tags (Sequence[Tag]): currently available tags """ - self._nullify(transactions) + try: + null = next(cat for cat in categories if cat.name == "null") + print("Nullifying") + self._nullify(transactions, null) + + categories = [cat for cat in categories if cat.name != "null"] + except StopIteration: + print("Null category not defined") self._rule_based_categories(transactions, categories) self._rule_based_tags(transactions, tags) @Timer(name="nullify") - def _nullify(self, transactions: Sequence[t.BankTransaction]): - print(f"Nullifying {len(transactions)} transactions") + def _nullify(self, transactions: Sequence[t.BankTransaction], null: t.Category): count = 0 matching = [] for transaction in transactions: @@ -46,11 +52,13 @@ class Categorizer: transaction.date - timedelta(days=self.options["null_days"]) <= cancel.date <= transaction.date + timedelta(days=self.options["null_days"]) - and transaction not in matching - and cancel not in matching and cancel != transaction and cancel.bank != transaction.bank and cancel.amount == -transaction.amount + and transaction not in matching + and cancel not in matching + and all(r.matches(transaction) for r in null.rules) + and all(r.matches(cancel) for r in null.rules) ) ): transaction.category = t.TransactionCategory( @@ -65,7 +73,7 @@ class Categorizer: count += 2 break - print(f"Nullified {count} transactions") + print(f"Nullified {count} of {len(transactions)} transactions") @Timer(name="categoryrules") def _rule_based_categories( @@ -87,12 +95,14 @@ class Categorizer: continue # passed all conditions, assign category - if ( - transaction.category - and transaction.category.name == category.name - ): + if transaction.category: + if transaction.category.name == category.name: + continue + if ( - input(f"Overwrite {transaction} with {category}? (y/n)") + input( + f"Overwrite {transaction} with {category.name}? (y/n)" + ) == "y" ): transaction.category.name = category.name From 6b26651d22ec857b82fa9c1fdf672c3cd187eea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Wed, 12 Apr 2023 18:19:24 +0100 Subject: [PATCH 64/66] Option eua now takes id instead of bank name --- pfbudget/cli/argparser.py | 2 +- pfbudget/input/nordigen.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pfbudget/cli/argparser.py b/pfbudget/cli/argparser.py index 298aa04..80f732d 100644 --- a/pfbudget/cli/argparser.py +++ b/pfbudget/cli/argparser.py @@ -129,7 +129,7 @@ def argparser() -> argparse.ArgumentParser: # Nordigen requisition id requisition = subparsers.add_parser("eua") requisition.set_defaults(op=Operation.RequisitionId) - requisition.add_argument("name", nargs=1, type=str) + requisition.add_argument("id", nargs=1, type=str) requisition.add_argument("country", nargs=1, type=str) # Download through the Nordigen API diff --git a/pfbudget/input/nordigen.py b/pfbudget/input/nordigen.py index a28027b..eeca3b3 100644 --- a/pfbudget/input/nordigen.py +++ b/pfbudget/input/nordigen.py @@ -86,8 +86,7 @@ class NordigenInput(Input): print(f"New access token: {token}") return token - def requisition(self, institution: str, country: str = "PT"): - id = self._client.institution.get_institution_id_by_name(country, institution) + def requisition(self, id: str, country: str = "PT"): requisition = self._client.initialize_session( redirect_uri=self.redirect_url, institution_id=id, From 2cbf00a9390b35105959d76e2f4dddf145c7255f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Wed, 12 Apr 2023 18:20:57 +0100 Subject: [PATCH 65/66] CLI now allow --no-nulls flag on categorize Null categorization, which is a major time slog, may now be disabled. --- pfbudget/__main__.py | 6 ++++++ pfbudget/cli/argparser.py | 5 ++++- pfbudget/core/categorizer.py | 17 ++++++++++------- pfbudget/core/manager.py | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 7a28ef5..a3fe169 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -25,6 +25,12 @@ if __name__ == "__main__": Interactive(Manager(db, verbosity)).start() exit() + case Operation.Categorize: + keys = {"no_nulls"} + assert args.keys() >= keys, f"missing {args.keys() - keys}" + + params = [args["no_nulls"]] + case Operation.Parse: keys = {"path", "bank", "creditcard"} assert args.keys() >= keys, f"missing {args.keys() - keys}" diff --git a/pfbudget/cli/argparser.py b/pfbudget/cli/argparser.py index 80f732d..e60cbfa 100644 --- a/pfbudget/cli/argparser.py +++ b/pfbudget/cli/argparser.py @@ -78,7 +78,10 @@ def argparser() -> argparse.ArgumentParser: # Automatic/manual categorization categorize = subparsers.add_parser("categorize").add_subparsers(required=True) - categorize.add_parser("auto").set_defaults(op=Operation.Categorize) + auto = categorize.add_parser("auto") + auto.set_defaults(op=Operation.Categorize) + auto.add_argument("--no-nulls", action="store_false") + categorize.add_parser("manual").set_defaults(op=Operation.ManualCategorization) """ diff --git a/pfbudget/core/categorizer.py b/pfbudget/core/categorizer.py index 4194379..b8b99df 100644 --- a/pfbudget/core/categorizer.py +++ b/pfbudget/core/categorizer.py @@ -16,6 +16,7 @@ class Categorizer: transactions: Sequence[t.BankTransaction], categories: Sequence[t.Category], tags: Sequence[t.Tag], + nullify: bool = True ): """Overarching categorization tool @@ -28,14 +29,16 @@ class Categorizer: tags (Sequence[Tag]): currently available tags """ - try: - null = next(cat for cat in categories if cat.name == "null") - print("Nullifying") - self._nullify(transactions, null) + if nullify: + try: + null = next(cat for cat in categories if cat.name == "null") + print("Nullifying") + self._nullify(transactions, null) - categories = [cat for cat in categories if cat.name != "null"] - except StopIteration: - print("Null category not defined") + except StopIteration: + print("Null category not defined") + + categories = [cat for cat in categories if cat.name != "null"] self._rule_based_categories(transactions, categories) self._rule_based_tags(transactions, tags) diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index 9435e90..45b1982 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -97,7 +97,7 @@ class Manager: ) categories = session.get(Category) tags = session.get(Tag) - Categorizer().rules(uncategorized, categories, tags) + Categorizer().rules(uncategorized, categories, tags, params[0]) case Operation.BankMod: with self.db.session() as session: From 5af05c2fa4c6c295e80fffe2417b50fb498edd97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Wed, 12 Apr 2023 18:23:24 +0100 Subject: [PATCH 66/66] Print the transaction ID --- pfbudget/db/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pfbudget/db/model.py b/pfbudget/db/model.py index eb2d712..68d2fa6 100644 --- a/pfbudget/db/model.py +++ b/pfbudget/db/model.py @@ -106,6 +106,7 @@ class Transaction(Base, Export): @property def format(self) -> dict[str, Any]: return dict( + id=self.id, date=self.date, description=self.description, amount=self.amount,