Upgrade for beancount3/beangulp

.pre-commit-config.yaml

@@ -52,4 +52,4 @@ repos:
   rev: v1.10.1
   hooks:
   -   id: mypy
-      args: [--install-types, --non-interactive, --ignore-missing-imports]
+      args: [--install-types, --non-interactive, --ignore-missing-imports, --disallow-incomplete-defs]

setup.cfg

@@ -38,7 +38,9 @@ package_dir =
 # install_requires = numpy; scipy
 install_requires =
     importlib-metadata; python_version<"3.8"
-    beancount>=2,<3
+    beancount>=3
+    beangulp
+    beanprice
     bitstampclient
     mt-940
     pyyaml
@@ -49,6 +51,7 @@ install_requires =
     blockcypher
     imap-tools
     undictify
+    rsa
 # The usage of test_requires is discouraged, see `Dependency Management` docs
 # tests_require = pytest; pytest-cov
 # Require a specific Python version, e.g. Python 2.7 or >= 3.4

src/tariochbctools/importers/bitst/importer.py

@@ -1,36 +1,37 @@
 from datetime import date
 from os import path
 
+import beangulp
 import bitstamp.client
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import MISSING, D
-from beancount.ingest import importer
 from dateutil.parser import parse
 from dateutil.relativedelta import relativedelta
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Bitstamp."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("bitstamp.yaml")
+    def identify(self, filepath):
+        return path.basename(filepath).endswith("bitstamp.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath):
         return ""
 
-    def extract(self, file, existing_entries):
-        self.priceLookup = PriceLookup(existing_entries, "CHF")
+    def extract(self, filepath, existing):
+        self.priceLookup = PriceLookup(existing, "CHF")
 
-        config = yaml.safe_load(file.contents())
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         self.client = bitstamp.client.Trading(
             username=config["username"], key=config["key"], secret=config["secret"]
         )
         self.currencies = config["currencies"]
-        self.account = config["account"]
+        self._account = config["account"]
         self.otherExpensesAccount = config["otherExpensesAccount"]
         self.capGainAccount = config["capGainAccount"]
 
@@ -72,7 +73,7 @@ def fetchSingle(self, trx):
             )
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     cost,
                     None,
@@ -84,7 +85,7 @@ def fetchSingle(self, trx):
             narration = "Withdrawal"
             postings = [
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     None,
                     None,
@@ -119,15 +120,15 @@ def fetchSingle(self, trx):
 
             postings = [
                 data.Posting(
-                    self.account + ":" + posCcy,
+                    self._account + ":" + posCcy,
                     amount.Amount(posAmt, posCcy),
                     posCcyCost,
                     posCcyPrice,
                     None,
                     None,
                 ),
                 data.Posting(
-                    self.account + ":" + negCcy,
+                    self._account + ":" + negCcy,
                     amount.Amount(negAmt, negCcy),
                     negCcyCost,
                     negCcyPrice,

src/tariochbctools/importers/blockchain/importer.py

@@ -1,28 +1,29 @@
 from os import path
 
+import beangulp
 import blockcypher
 import yaml
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
 
 from tariochbctools.importers.general.priceLookup import PriceLookup
 
 
-class Importer(importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Blockchain data."""
 
-    def identify(self, file):
-        return path.basename(file.name).endswith("blockchain.yaml")
+    def identify(self, filepath: str) -> bool:
+        return path.basename(filepath).endswith("blockchain.yaml")
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Entries:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
         self.config = config
         baseCcy = config["base_ccy"]
-        priceLookup = PriceLookup(existing_entries, baseCcy)
+        priceLookup = PriceLookup(existing, baseCcy)
 
         entries = []
         for address in self.config["addresses"]:

src/tariochbctools/importers/cembrastatement/importer.py

@@ -1,26 +1,28 @@
 import re
 from datetime import datetime, timedelta
 
+import beangulp
 import camelot
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
-from beancount.ingest.importers.mixins import identifier
 
 
-class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for Cembra Card Statement PDF files."""
 
-    def __init__(self, regexps, account):
-        identifier.IdentifyMixin.__init__(self, matchers=[("filename", regexps)])
-        self.account = account
+    def __init__(self, filepattern: str, account: data.Account):
+        self._filepattern = filepattern
+        self._account = account
         self.currency = "CHF"
 
-    def file_account(self, file):
-        return self.account
+    def identify(self, filepath: str) -> bool:
+        return re.search(self._filepattern, filepath) is not None
 
-    def createEntry(self, file, date, amt, text):
-        meta = data.new_metadata(file.name, 0)
+    def account(self, filepath):
+        return self._account
+
+    def createEntry(self, filepath, date, amt, text):
+        meta = data.new_metadata(filepath, 0)
         return data.Transaction(
             meta,
             date,
@@ -30,19 +32,19 @@ def createEntry(self, file, date, amt, text):
             data.EMPTY_SET,
             data.EMPTY_SET,
             [
-                data.Posting(self.account, amt, None, None, None, None),
+                data.Posting(self._account, amt, None, None, None, None),
             ],
         )
 
-    def createBalanceEntry(self, file, date, amt):
-        meta = data.new_metadata(file.name, 0)
-        return data.Balance(meta, date, self.account, amt, None, None)
+    def createBalanceEntry(self, filepath, date, amt):
+        meta = data.new_metadata(filepath, 0)
+        return data.Balance(meta, date, self._account, amt, None, None)
 
-    def extract(self, file, existing_entries):
+    def extract(self, filepath, existing):
         entries = []
 
         tables = camelot.read_pdf(
-            file.name, pages="2-end", flavor="stream", table_areas=["50,700,560,50"]
+            filepath, pages="2-end", flavor="stream", table_areas=["50,700,560,50"]
         )
         for table in tables:
             df = table.df
@@ -71,7 +73,9 @@ def extract(self, file, existing_entries):
                     amount = self.getAmount(debit, credit)
 
                     if amount:
-                        entries.append(self.createEntry(file, book_date, amount, text))
+                        entries.append(
+                            self.createEntry(filepath, book_date, amount, text)
+                        )
                     continue
 
                 # Balance entry
@@ -89,7 +93,9 @@ def extract(self, file, existing_entries):
                     amount = self.getAmount(debit, credit)
 
                     if amount:
-                        entries.append(self.createBalanceEntry(file, book_date, amount))
+                        entries.append(
+                            self.createBalanceEntry(filepath, book_date, amount)
+                        )
 
         return entries
 

src/tariochbctools/importers/general/mailAdapterImporter.py

@@ -2,24 +2,26 @@
 from os import path
 
 import yaml
-from beancount.ingest import cache, importer
+from beancount.core import data
+from beangulp import Importer
 from imap_tools import MailBox
 
 
-class MailAdapterImporter(importer.ImporterProtocol):
+class MailAdapterImporter(Importer):
     """An importer adapter that fetches file from mails and then calls another importer."""
 
     def __init__(self, importers):
         self.importers = importers
 
-    def identify(self, file):
-        return "mail.yaml" == path.basename(file.name)
+    def identify(self, filepath: str) -> bool:
+        return "mail.yaml" == path.basename(filepath)
 
-    def file_account(self, file):
+    def account(self, filepath: str) -> data.Account:
         return ""
 
-    def extract(self, file, existing_entries):
-        config = yaml.safe_load(file.contents())
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
+        with open(filepath) as file:
+            config = yaml.safe_load(file)
 
         with MailBox(config["host"]).login(
             config["user"], config["password"], initial_folder=config["folder"]
@@ -33,13 +35,10 @@ def extract(self, file, existing_entries):
                         with open(attFileName, "wb") as attFile:
                             attFile.write(att.payload)
                             attFile.flush()
-                            fileMemo = cache.get_file(attFileName)
 
                             for delegate in self.importers:
-                                if delegate.identify(fileMemo):
-                                    newEntries = delegate.extract(
-                                        fileMemo, existing_entries
-                                    )
+                                if delegate.identify(attFileName):
+                                    newEntries = delegate.extract(attFileName, existing)
                                     result.extend(newEntries)
                                     processed = True
 

src/tariochbctools/importers/general/mt940importer.py

@@ -1,37 +1,35 @@
+import re
+
+import beangulp
 import mt940
 from beancount.core import amount, data
 from beancount.core.number import D
-from beancount.ingest import importer
-from beancount.ingest.importers.mixins import identifier
 
 
-class Importer(identifier.IdentifyMixin, importer.ImporterProtocol):
+class Importer(beangulp.Importer):
     """An importer for MT940 files."""
 
-    def __init__(self, regexps, account):
-        identifier.IdentifyMixin.__init__(self, matchers=[("filename", regexps)])
-        self.account = account
-
-    def identify(self, file):
-        if file.mimetype() != "text/plain":
-            return False
+    def __init__(self, filepattern: str, account: data.Account):
+        self._filepattern = filepattern
+        self._account = account
 
-        return super().identify(file)
+    def identify(self, filepath: str) -> bool:
+        return re.search(self._filepattern, filepath) is not None
 
-    def file_account(self, file):
-        return self.account
+    def account(self, filepath: str) -> data.Account:
+        return self._account
 
-    def extract(self, file, existing_entries):
+    def extract(self, filepath: str, existing: data.Entries) -> data.Entries:
         entries = []
-        transactions = mt940.parse(file.contents())
+        transactions = mt940.parse(filepath)
         for trx in transactions:
             trxdata = trx.data
             ref = trxdata["bank_reference"]
             if ref:
                 metakv = {"ref": ref}
             else:
                 metakv = None
-            meta = data.new_metadata(file.name, 0, metakv)
+            meta = data.new_metadata(filepath, 0, metakv)
             if "entry_date" in trxdata:
                 date = trxdata["entry_date"]
             else:
@@ -46,7 +44,7 @@ def extract(self, file, existing_entries):
                 data.EMPTY_SET,
                 [
                     data.Posting(
-                        self.account,
+                        self._account,
                         amount.Amount(
                             D(trxdata["amount"].amount), trxdata["amount"].currency
                         ),

src/tariochbctools/importers/general/priceLookup.py

@@ -1,18 +1,18 @@
 from datetime import date
 
-from beancount.core import amount, prices
+from beancount.core import amount, data, prices
 from beancount.core.number import D
 
 
 class PriceLookup:
-    def __init__(self, existing_entries, baseCcy: str):
-        if existing_entries:
-            self.priceMap = prices.build_price_map(existing_entries)
+    def __init__(self, existing: data.Entries, baseCcy: str):
+        if existing:
+            self.priceMap = prices.build_price_map(existing)
         else:
             self.priceMap = None
         self.baseCcy = baseCcy
 
-    def fetchPriceAmount(self, instrument: str, date: date):
+    def fetchPriceAmount(self, instrument: str, date: date) -> data.Amount:
         if self.priceMap:
             price = prices.get_price(
                 self.priceMap, tuple([instrument, self.baseCcy]), date
@@ -21,7 +21,7 @@ def fetchPriceAmount(self, instrument: str, date: date):
         else:
             return D(1)
 
-    def fetchPrice(self, instrument: str, date: date):
+    def fetchPrice(self, instrument: str, date: date) -> data.Amount:
         if instrument == self.baseCcy:
             return None