From 1cfc8dbe38fb9d2cb40b2ec3f9d0b5fbd40fb24a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lu=C3=ADs=20Murta?= Date: Fri, 26 May 2023 23:58:44 +0100 Subject: [PATCH] Create EUA and request user access permission Move the token creation/renewal code into the NordigenClient and remove external access. The NordigenClient now also takes the DB client in the constructor. While creating the unit test, noticed that the nordigen mocks for downloaded transactions could match the simple transactions mock, thus helping in the online download command. Also created the nordigen.NordigenClient mock, with the used methods mocked and raising a NotImplement when a new endpoint is requested. --- pfbudget/__main__.py | 4 +- pfbudget/cli/argparser.py | 3 +- pfbudget/core/manager.py | 29 +++++---- pfbudget/extract/nordigen.py | 112 +++++++++++++++++++++++------------ tests/mocks/nordigen.py | 61 ++++++++++++++++++- tests/mocks/transactions.py | 19 +++--- tests/test_psd2.py | 7 ++- 7 files changed, 170 insertions(+), 65 deletions(-) diff --git a/pfbudget/__main__.py b/pfbudget/__main__.py index 3f6c05b..e63db2c 100644 --- a/pfbudget/__main__.py +++ b/pfbudget/__main__.py @@ -38,10 +38,10 @@ if __name__ == "__main__": params = [args["path"], args["bank"], args["creditcard"]] case Operation.RequisitionId: - keys = {"name", "country"} + keys = {"bank"} assert args.keys() >= keys, f"missing {args.keys() - keys}" - params = [args["name"][0], args["country"][0]] + params = [args["bank"][0]] case Operation.Download: keys = {"all", "banks", "interval", "start", "end", "year", "dry_run"} diff --git a/pfbudget/cli/argparser.py b/pfbudget/cli/argparser.py index b27abed..d9df835 100644 --- a/pfbudget/cli/argparser.py +++ b/pfbudget/cli/argparser.py @@ -133,8 +133,7 @@ def argparser() -> argparse.ArgumentParser: # PSD2 requisition id requisition = subparsers.add_parser("eua") requisition.set_defaults(op=Operation.RequisitionId) - requisition.add_argument("id", nargs=1, type=str) - requisition.add_argument("country", nargs=1, type=str) + requisition.add_argument("bank", nargs=1, type=str) # Download through the PSD2 API download = subparsers.add_parser("download", parents=[period]) diff --git a/pfbudget/core/manager.py b/pfbudget/core/manager.py index dee4fbb..7fd3b08 100644 --- a/pfbudget/core/manager.py +++ b/pfbudget/core/manager.py @@ -79,7 +79,7 @@ class Manager: else: banks = self.database.select(Bank, Bank.nordigen) - extractor = PSD2Extractor(Manager.nordigen_client()) + extractor = PSD2Extractor(self.nordigen_client()) transactions = [] for bank in banks: @@ -122,16 +122,26 @@ class Manager: case Operation.PSD2Del: self.database.delete(NordigenBank, NordigenBank.name, params) - case Operation.Token: - Manager.nordigen_client().new_token() - case Operation.RequisitionId: - link, _ = Manager.nordigen_client().requisition(params[0], params[1]) - print(f"Opening {link} to request access to {params[0]}") + bank_name = params[0] + bank = self.database.select(Bank, (lambda: Bank.name == bank_name))[0] + + if not bank.nordigen or not bank.nordigen.bank_id: + raise ValueError(f"{bank} doesn't have a Nordigen ID") + + link, req_id = self.nordigen_client().new_requisition( + bank.nordigen.bank_id + ) + + self.database.update( + NordigenBank, + [{"name": bank.nordigen.name, "requisition_id": req_id}], + ) + webbrowser.open(link) case Operation.PSD2CountryBanks: - banks = Manager.nordigen_client().country_banks(params[0]) + banks = self.nordigen_client().country_banks(params[0]) print(banks) case ( @@ -387,6 +397,5 @@ class Manager: self._database = Client(self._db, echo=self._verbosity > 2) return self._database - @staticmethod - def nordigen_client() -> NordigenClient: - return NordigenClient(NordigenCredentialsManager.default) + def nordigen_client(self) -> NordigenClient: + return NordigenClient(NordigenCredentialsManager.default, self.database) diff --git a/pfbudget/extract/nordigen.py b/pfbudget/extract/nordigen.py index 716c5b2..2e0bec5 100644 --- a/pfbudget/extract/nordigen.py +++ b/pfbudget/extract/nordigen.py @@ -1,12 +1,17 @@ from dataclasses import dataclass +import datetime as dt import dotenv import json import nordigen import os import requests import time +from typing import Optional, Tuple import uuid +from pfbudget.db.client import Client +from pfbudget.db.model import Nordigen + from .exceptions import CredentialsError, DownloadError dotenv.load_dotenv() @@ -16,38 +21,35 @@ dotenv.load_dotenv() class NordigenCredentials: id: str key: str - token: str = "" def valid(self) -> bool: - return self.id and self.key + return len(self.id) != 0 and len(self.key) != 0 class NordigenClient: redirect_url = "https://murta.dev" - def __init__(self, credentials: NordigenCredentials): - super().__init__() - + def __init__(self, credentials: NordigenCredentials, client: Client): if not credentials.valid(): raise CredentialsError - self._client = nordigen.NordigenClient( + self.__client = nordigen.NordigenClient( secret_key=credentials.key, secret_id=credentials.id, timeout=5 ) - - if credentials.token: - self._client.token = credentials.token + self.__client.token = self.__token(client) def download(self, requisition_id): try: - requisition = self._client.requisition.get_requisition_by_id(requisition_id) + requisition = self.__client.requisition.get_requisition_by_id( + requisition_id + ) print(requisition) except requests.HTTPError as e: raise DownloadError(e) transactions = {} for acc in requisition["accounts"]: - account = self._client.account_api(acc) + account = self.__client.account_api(acc) retries = 0 while retries < 3: @@ -71,43 +73,75 @@ class NordigenClient: with open("json/" + bank.name + ".json", "w") as f: json.dump(downloaded, f) - def new_token(self): - return self._client.generate_token() + def new_requisition( + self, + institution_id: str, + max_historical_days: Optional[int] = None, + access_valid_for_days: Optional[int] = None, + ) -> Tuple[str, str]: + kwargs = { + "max_historical_days": max_historical_days, + "access_valid_for_days": access_valid_for_days, + } + kwargs = {k: v for k, v in kwargs.items() if v is not None} - def refresh_token(self, token: str): - return self._client.exchange_token(token) - - def requisition(self, id: str, country: str = "PT"): - requisition = self._client.initialize_session( - redirect_uri=self.redirect_url, - institution_id=id, - reference_id=str(uuid.uuid4()), + req = self.__client.initialize_session( + self.redirect_url, institution_id, str(uuid.uuid4()), **kwargs ) - return requisition.link, requisition.requisition_id + return req.link, req.requisition_id def country_banks(self, country: str): - return self._client.institution.get_institutions(country) + return self.__client.institution.get_institutions(country) - # def __token(self): - # if token := os.environ.get("TOKEN"): - # return token - # else: - # token = self._client.generate_token() - # print(f"New access token: {token}") - # return token["access"] + def __token(self, client: Client) -> str: + with client.session as session: + token = session.select(Nordigen) - @property - def token(self): - return self._client.token + def datetime(seconds: int) -> dt.datetime: + return dt.datetime.now() + dt.timedelta(seconds=seconds) - @token.setter - def token(self, value: str): - self._client.token = value + if not len(token): + print("First time nordigen token setup") + new = self.__client.generate_token() + session.insert( + [ + Nordigen( + "access", + new["access"], + datetime(new["access_expires"]), + ), + Nordigen( + "refresh", + new["refresh"], + datetime(new["refresh_expires"]), + ), + ] + ) + + return new["access"] + + else: + access = next(t for t in token if t.type == "access") + refresh = next(t for t in token if t.type == "refresh") + + if access.expires > dt.datetime.now(): + pass + elif refresh.expires > dt.datetime.now(): + new = self.__client.exchange_token(refresh.token) + access.token = new["access"] + access.expires = datetime(new["access_expires"]) + else: + new = self.__client.generate_token() + access.token = new["access"] + access.expires = datetime(new["access_expires"]) + refresh.token = new["refresh"] + refresh.expires = datetime(new["refresh_expires"]) + + return access.token class NordigenCredentialsManager: default = NordigenCredentials( - os.environ.get("SECRET_ID"), - os.environ.get("SECRET_KEY"), - os.environ.get("TOKEN"), + os.environ.get("SECRET_ID", ""), + os.environ.get("SECRET_KEY", ""), ) diff --git a/tests/mocks/nordigen.py b/tests/mocks/nordigen.py index 8ef8ee1..75a6f08 100644 --- a/tests/mocks/nordigen.py +++ b/tests/mocks/nordigen.py @@ -1,3 +1,8 @@ +from typing import Any, Dict, List, Optional +import nordigen +from nordigen.types.http_enums import HTTPMethod +from nordigen.types.types import RequisitionDto, TokenType + from pfbudget.extract.nordigen import NordigenCredentials @@ -13,6 +18,7 @@ accounts_id = { "owner_name": "string", } +# The downloaded transactions match the simple and simple_transformed mocks accounts_id_transactions = { "transactions": { "booked": [ @@ -84,4 +90,57 @@ requisitions_id = { "redirect_immediate": False, } -credentials = NordigenCredentials("ID", "KEY", "TOKEN") +credentials = NordigenCredentials("ID", "KEY") + + +class MockNordigenClient(nordigen.NordigenClient): + def __init__( + self, + secret_key: str = "ID", + secret_id: str = "KEY", + timeout: int = 10, + base_url: str = "https://ob.nordigen.com/api/v2", + ) -> None: + super().__init__(secret_key, secret_id, timeout, base_url) + + def generate_token(self) -> TokenType: + return { + "access": "access_token", + "refresh": "refresh_token", + "access_expires": 86400, + "refresh_expires": 2592000, + } + + def exchange_token(self, refresh_token: str) -> TokenType: + assert len(refresh_token) > 0, "invalid refresh token" + return { + "access": "access_token", + "refresh": "refresh_token", + "access_expires": 86400, + "refresh_expires": 2592000, + } + + def request( + self, + method: HTTPMethod, + endpoint: str, + data: Optional[Dict[str, Any]] = None, + headers: Optional[Dict[str, Any]] = None, + ) -> Any: + if endpoint == "requisitions/" + "requisition_id" + "/": + return requisitions_id + elif endpoint == "accounts/" + id + "/transactions/": + return accounts_id_transactions + else: + raise NotImplementedError(endpoint) + + def initialize_session( + self, + redirect_uri: str, + institution_id: str, + reference_id: str, + max_historical_days: int = 90, + access_valid_for_days: int = 90, + access_scope: List[str] | None = None, + ) -> RequisitionDto: + return RequisitionDto("http://random", "requisition_id") diff --git a/tests/mocks/transactions.py b/tests/mocks/transactions.py index 5564c30..0024a53 100644 --- a/tests/mocks/transactions.py +++ b/tests/mocks/transactions.py @@ -12,22 +12,25 @@ from pfbudget.db.model import ( TransactionTag, ) +# The simple and simple_transformed match the nordigen mocks simple = [ - Transaction(date(2023, 1, 1), "", Decimal("-10")), - Transaction(date(2023, 1, 2), "", Decimal("-50")), + BankTransaction(date(2023, 1, 14), "string", Decimal("328.18"), bank="bank"), + BankTransaction(date(2023, 2, 14), "string", Decimal("947.26"), bank="bank"), ] simple_transformed = [ - Transaction( - date(2023, 1, 1), + BankTransaction( + date(2023, 1, 14), "", - Decimal("-10"), + Decimal("328.18"), + bank="bank", category=TransactionCategory("category#1", CategorySelector.algorithm), ), - Transaction( - date(2023, 1, 2), + BankTransaction( + date(2023, 2, 14), "", - Decimal("-50"), + Decimal("947.26"), + bank="bank", category=TransactionCategory("category#2", CategorySelector.algorithm), ), ] diff --git a/tests/test_psd2.py b/tests/test_psd2.py index b1bf764..86321f9 100644 --- a/tests/test_psd2.py +++ b/tests/test_psd2.py @@ -4,6 +4,7 @@ from typing import Any, Optional import pytest import requests +from mocks.client import MockClient import mocks.nordigen as mock from pfbudget.db.model import AccountType, Bank, BankTransaction, NordigenBank @@ -58,8 +59,8 @@ def mock_requests(monkeypatch: pytest.MonkeyPatch): @pytest.fixture def extractor() -> Extractor: - credentials = NordigenCredentials("ID", "KEY", "TOKEN") - return PSD2Extractor(NordigenClient(credentials)) + credentials = NordigenCredentials("ID", "KEY") + return PSD2Extractor(NordigenClient(credentials, MockClient())) @pytest.fixture @@ -72,7 +73,7 @@ class TestExtractPSD2: def test_empty_credentials(self): cred = NordigenCredentials("", "") with pytest.raises(CredentialsError): - NordigenClient(cred) + NordigenClient(cred, MockClient()) def test_no_psd2_bank(self, extractor: Extractor): with pytest.raises(BankError):