Upgrade for beancount3/beangulp

.pre-commit-config.yaml

@@ -52,4 +52,5 @@ repos:
   rev: v1.10.1
   hooks:
   -   id: mypy
-      args: [--install-types, --non-interactive, --ignore-missing-imports]
+      args: [--install-types, --non-interactive, --ignore-missing-imports, --disallow-incomplete-defs]
+#      args: [--install-types, --non-interactive, --ignore-missing-imports, --disallow-untyped-defs]

setup.cfg

@@ -38,7 +38,9 @@ package_dir =
 # install_requires = numpy; scipy
 install_requires =
     importlib-metadata; python_version<"3.8"
-    beancount>=2,<3
+    beancount>=3
+    beangulp
+    beanprice
     bitstampclient
     mt-940
     pyyaml
@@ -49,6 +51,7 @@ install_requires =
     blockcypher
     imap-tools
     undictify
+    rsa
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 # tests_require = pytest; pytest-cov
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4

src/tariochbctools/importers/bcge/importer.py

@@ -1,22 +1,23 @@
 import re
+from typing import Any
 
 from tariochbctools.importers.general import mt940importer
 
 
-def strip_newline(string):
+def strip_newline(string: str) -> str:
     return string.replace("\n", "").replace("\r", "")
 
 
 class BCGEImporter(mt940importer.Importer):
-    def prepare_payee(self, trxdata):
+    def prepare_payee(self, trxdata: dict[str, Any]) -> str:
         transaction_details = strip_newline(trxdata["transaction_details"])
         payee = re.search(r"ORDP/([^/]+)", transaction_details)
         if payee is None:
             return ""
         else:
             return payee.group(1)
 
-    def prepare_narration(self, trxdata):
+    def prepare_narration(self, trxdata: dict[str, Any]) -> str:
         transaction_details = strip_newline(trxdata["transaction_details"])
         extra_details = strip_newline(trxdata["extra_details"])
         beneficiary = re.search(r"/BENM/([^/]+)", transaction_details)

src/tariochbctools/importers/bitst/importer.py

@@ -1,36 +1,38 @@
 from datetime import date
 from os import path
+from typing import Any
 
+import beangulp
 import bitstamp.client
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import MISSING, D
-from beancount.ingest import importer
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Bitstamp."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("bitstamp.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("bitstamp.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def extract(self, file, existing_entries):
-        self.priceLookup = PriceLookup(existing_entries, "CHF")
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        self.priceLookup = PriceLookup(existing, "CHF")
 
-        config = yaml.safe_load(file.contents())
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         self.client = bitstamp.client.Trading(
             username=config["username"], key=config["key"], secret=config["secret"]
         )
         self.currencies = config["currencies"]
-        self.account = config["account"]
+        self._account = config["account"]
         self.otherExpensesAccount = config["otherExpensesAccount"]
         self.capGainAccount = config["capGainAccount"]
 
@@ -46,7 +48,7 @@ def extract(self, file, existing_entries):
 
         return result
 
-    def fetchSingle(self, trx):
+    def fetchSingle(self, trx: dict[str, Any]) -> data.Transaction:
         id = int(trx["id"])
         type = int(trx["type"])
         date = parse(trx["datetime"]).date()
@@ -67,12 +69,13 @@ def fetchSingle(self, trx):
 
         if type == 0:
             narration = "Deposit"
-            cost = data.Cost(
-                self.priceLookup.fetchPriceAmount(posCcy, date), "CHF", None, None
-            )
+            if posCcy:
+                cost = data.Cost(
+                    self.priceLookup.fetchPriceAmount(posCcy, date), "CHF", None, None
+                )
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     cost,
                     None,
@@ -84,7 +87,7 @@ def fetchSingle(self, trx):
             narration = "Withdrawal"
             postings = [
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     None,
                     None,
@@ -94,14 +97,15 @@ def fetchSingle(self, trx):
             ]
         elif type == 2:
             fee = D(trx["fee"])
-            if posCcy.lower() + "_" + negCcy.lower() in trx:
+            if posCcy and negCcy and posCcy.lower() + "_" + negCcy.lower() in trx:
                 feeCcy = negCcy
                 negAmt -= fee
             else:
                 feeCcy = posCcy
                 posAmt -= fee
 
-            rateFiatCcy = self.priceLookup.fetchPriceAmount(feeCcy, date)
+            if feeCcy:
+                rateFiatCcy = self.priceLookup.fetchPriceAmount(feeCcy, date)
             if feeCcy == posCcy:
                 posCcyCost = None
                 posCcyPrice = amount.Amount(rateFiatCcy, "CHF")
@@ -119,15 +123,15 @@ def fetchSingle(self, trx):
 
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     posCcyCost,
                     posCcyPrice,
                     None,
                     None,
                 ),
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     negCcyCost,
                     negCcyPrice,

src/tariochbctools/importers/blockchain/importer.py

@@ -1,28 +1,29 @@
 from os import path
 
+import beangulp
 import blockcypher
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Blockchain data."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("blockchain.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("blockchain.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Entries:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         baseCcy = config["base_ccy"]
-        priceLookup = PriceLookup(existing_entries, baseCcy)
+        priceLookup = PriceLookup(existing, baseCcy)
 
         entries = []
         for address in self.config["addresses"]:

src/tariochbctools/importers/cembrastatement/importer.py

@@ -1,26 +1,31 @@
+import datetime
 import re
-from datetime import datetime, timedelta
+from datetime import timedelta
 
+import beangulp
 import camelot
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
-from beancount.ingest.importers.mixins import identifier
 
 
-class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Cembra Card Statement PDF files."""
 
-    def __init__(self, regexps, account):
-        identifier.IdentifyMixin.__init__(self, matchers=[("filename", regexps)])
-        self.account = account
+    def __init__(self, filepattern: str, account: data.Account):
+        self._filepattern = filepattern
+        self._account = account
         self.currency = "CHF"
 
-    def file_account(self, file):
-        return self.account
+    def identify(self, filepath: str) -> bool:
+        return re.search(self._filepattern, filepath) is not None
 
-    def createEntry(self, file, date, amt, text):
-        meta = data.new_metadata(file.name, 0)
+    def account(self, filepath: str) -> data.Account:
+        return self._account
+
+    def createEntry(
+        self, filepath: str, date: datetime.date, amt: data.Decimal, text: str
+    ) -> data.Transaction:
+        meta = data.new_metadata(filepath, 0)
         return data.Transaction(
             meta,
             date,
@@ -30,19 +35,21 @@ def createEntry(self, file, date, amt, text):
             data.EMPTY_SET,
             data.EMPTY_SET,
             [
-                data.Posting(self.account, amt, None, None, None, None),
+                data.Posting(self._account, amt, None, None, None, None),
             ],
         )
 
-    def createBalanceEntry(self, file, date, amt):
-        meta = data.new_metadata(file.name, 0)
-        return data.Balance(meta, date, self.account, amt, None, None)
+    def createBalanceEntry(
+        self, filepath: str, date: datetime.date, amt: data.Decimal
+    ) -> data.Balance:
+        meta = data.new_metadata(filepath, 0)
+        return data.Balance(meta, date, self._account, amt, None, None)
 
-    def extract(self, file, existing_entries):
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
         entries = []
 
         tables = camelot.read_pdf(
-            file.name, pages="2-end", flavor="stream", table_areas=["50,700,560,50"]
+            filepath, pages="2-end", flavor="stream", table_areas=["50,700,560,50"]
         )
         for table in tables:
             df = table.df
@@ -63,40 +70,49 @@ def extract(self, file, existing_entries):
 
                 # Transaction entry
                 try:
-                    book_date = datetime.strptime(book_date, "%d.%m.%Y").date()
+                    book_date = datetime.datetime.strptime(book_date, "%d.%m.%Y").date()
                 except Exception:
                     book_date = None
 
                 if book_date:
                     amount = self.getAmount(debit, credit)
 
                     if amount:
-                        entries.append(self.createEntry(file, book_date, amount, text))
+                        entries.append(
+                            self.createEntry(filepath, book_date, amount, text)
+                        )
                     continue
 
                 # Balance entry
                 try:
-                    book_date = re.search(
-                        r"Saldo per (\d\d\.\d\d\.\d\d\d\d) zu unseren Gunsten CHF", text
-                    ).group(1)
-                    book_date = datetime.strptime(book_date, "%d.%m.%Y").date()
-                    # add 1 day: cembra provides balance at EOD, but beancount checks it at SOD
-                    book_date = book_date + timedelta(days=1)
+                    m = re.search(
+                        r"Saldo per (\d\d\.\d\d\.\d\d\d\d) zu unseren Gunsten CHF",
+                        text,
+                    )
+                    if m:
+                        book_date = m.group(1)
+                        book_date = datetime.datetime.strptime(
+                            book_date, "%d.%m.%Y"
+                        ).date()
+                        # add 1 day: cembra provides balance at EOD, but beancount checks it at SOD
+                        book_date = book_date + timedelta(days=1)
                 except Exception:
                     book_date = None
 
                 if book_date:
                     amount = self.getAmount(debit, credit)
 
                     if amount:
-                        entries.append(self.createBalanceEntry(file, book_date, amount))
+                        entries.append(
+                            self.createBalanceEntry(filepath, book_date, amount)
+                        )
 
         return entries
 
-    def cleanDecimal(self, formattedNumber):
+    def cleanDecimal(self, formattedNumber: str) -> data.Decimal:
         return D(formattedNumber.replace("'", ""))
 
-    def getAmount(self, debit, credit):
+    def getAmount(self, debit: str, credit: str) -> data.Amount:
         amt = -self.cleanDecimal(debit) if debit else self.cleanDecimal(credit)
         if amt:
             return amount.Amount(amt, self.currency)

src/tariochbctools/importers/general/mailAdapterImporter.py

@@ -2,24 +2,26 @@
 from os import path
 
 import yaml
-from beancount.ingest import cache, importer
+from beancount.core import data
+from beangulp import Importer
 from imap_tools import MailBox
 
 
-class MailAdapterImporter(importer.ImporterProtocol):
+class MailAdapterImporter(Importer):
     """An importer adapter that fetches file from mails and then calls another importer."""
 
-    def __init__(self, importers):
+    def __init__(self, importers: list[Importer]):
         self.importers = importers
 
-    def identify(self, file):
-        return "mail.yaml" == path.basename(file.name)
+    def identify(self, filepath: str) -> bool:
+        return "mail.yaml" == path.basename(filepath)
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
 
         with MailBox(config["host"]).login(
             config["user"], config["password"], initial_folder=config["folder"]
@@ -33,13 +35,10 @@ def extract(self, file, existing_entries):
                         with open(attFileName, "wb") as attFile:
                             attFile.write(att.payload)
                             attFile.flush()
-                            fileMemo = cache.get_file(attFileName)
 
                             for delegate in self.importers:
-                                if delegate.identify(fileMemo):
-                                    newEntries = delegate.extract(
-                                        fileMemo, existing_entries
-                                    )
+                                if delegate.identify(attFileName):
+                                    newEntries = delegate.extract(attFileName, existing)
                                     result.extend(newEntries)
                                     processed = True