update

monzo_utils/lib/db.py

@@ -97,6 +97,10 @@ def one(self, table, **kwargs):
         return False
 
 
+    def all(self, table, **kwargs):
+        return self.query('select', table, kwargs)
+
+
     def find(self, table):
         self.query_table = table
         self.sel = []

monzo_utils/lib/db_driver/dynamodb.py

@@ -8,7 +8,8 @@
 CACHE_WHOLE_TABLES = [
     'provider',
     'account',
-    'pot'
+    'pot',
+    'payments'
 ]
 
 SECONDARY_INDICES = {
@@ -50,6 +51,45 @@ def select(self, table, expression, limit=None, orderby=None, orderdir=None, del
                 sys.stdout.write('.')
                 sys.stdout.flush()
 
+            if type(expression[key]) == list:
+                batch = []
+                results = []
+
+                for value in expression[key]:
+                    batch.append({
+                        key: { 'S': value }
+                    })
+
+                    if len(batch) >= 100:
+                        resp = self.dbd.batch_get_item(
+                            RequestItems={
+                                f"{self.prefix}_{table}": {
+                                    'Keys': batch
+                                }
+                            }
+                        )
+
+                        if 'Responses' in resp and f"{self.prefix}_{table}" in resp['Responses']:
+                            for item in resp['Responses'][f"{self.prefix}_{table}"]:
+                                results.append(self.to_obj(item))
+
+                        batch = []
+
+                if len(batch) > 0:
+                    resp = self.dbd.batch_get_item(
+                        RequestItems={
+                            f"{self.prefix}_{table}": {
+                                'Keys': batch
+                            }
+                        }
+                    )
+
+                    if 'Responses' in resp and f"{self.prefix}_{table}" in resp['Responses']:
+                        for item in resp['Responses'][f"{self.prefix}_{table}"]:
+                            results.append(self.to_obj(item))
+
+                return results
+
             resp = self.dbd.get_item(
                 TableName=f"{self.prefix}_{table}",
                 Key={

monzo_utils/lib/monzo_payments.py

@@ -23,6 +23,7 @@
 from monzo_utils.model.transaction import Transaction
 from monzo_utils.model.flex_summary import FlexSummary
 from monzo_utils.model.state import State
+from monzo_utils.model.payments import Payments
 from govuk_bank_holidays.bank_holidays import BankHolidays
 from calendar import monthrange
 
@@ -39,6 +40,7 @@ def __init__(self, account_name, output_json=False, abbreviate=False):
         self.total_this_month = 0
         self.next_month = 0
         self.next_month_bills_pot = 0
+        self.cache = {}
 
         self.config = self.load_config()
         self.validate_config()
@@ -468,6 +470,19 @@ def get_last_salary_date(self):
         else:
             account = self.account
 
+        if 'last_salary_transaction' in self.cache:
+            last_salary_transaction = self.cache['last_salary_transaction']
+
+            return last_salary_transaction.date
+
+        last_salary_transaction = Payments.one(key='last_salary_transaction')
+
+        if last_salary_transaction:
+            last_salary_transaction = Transaction.one(id=last_salary_transaction.transaction_id)
+
+            self.cache['last_salary_transaction'] = last_salary_transaction
+            return last_salary_transaction.date
+
         last_salary_transaction = account.last_salary_transaction(
             description=self.config['salary_description'],
             salary_minimum=self.config['salary_minimum'] if 'salary_minimum' in self.config else 1000,
@@ -478,6 +493,14 @@ def get_last_salary_date(self):
             sys.stderr.write("failed to find last salary transaction.\n")
             sys.exit(1)
 
+        payments = Payments()
+        payments.update({
+            'key': 'last_salary_transaction',
+            'transaction_id': last_salary_transaction.id,
+            'account_id': account.id
+        })
+        payments.save()
+
         last_salary_date = last_salary_transaction.date
 
         return last_salary_date

monzo_utils/lib/monzo_sync.py

@@ -21,6 +21,7 @@
 from monzo_utils.model.transaction import Transaction
 from monzo_utils.model.counterparty import Counterparty
 from monzo_utils.model.transaction_metadata import TransactionMetadata
+from monzo_utils.model.payments import Payments
 from monzo.exceptions import MonzoAuthenticationError, MonzoServerError, MonzoHTTPError, MonzoPermissionsError
 from boto3.dynamodb.conditions import Key
 
@@ -408,9 +409,9 @@ def add_transaction(self, account, mo_transaction, pot_account_ids, pot_id=None)
 
         transaction.update(transaction_data)
 
-        transaction.save()
+        changed = transaction.save()
 
-        return transaction
+        return transaction, changed
 
 
     def get_or_create_provider(self, provider_name):
@@ -474,11 +475,16 @@ def sync_account(self, mo_account, days):
 
         Log().info(f'syncing transactions for account: {account.name}')
 
-        pot_account_ids, total = self.sync_account_transactions(account, pot_lookup, days)
+        pot_account_ids, total, changed = self.sync_account_transactions(account, pot_lookup, days)
 
         Log().info(f'syncing pot transactions for account: {account.name}')
 
-        self.sync_account_pot_transactions(account, pot_account_ids, pot_lookup, total, days)
+        pot_changed = self.sync_account_pot_transactions(account, pot_account_ids, pot_lookup, total, days)
+
+        if changed or pot_changed:
+            Log().info(f'transaction changes detected, dropping cached payments for {account.name}')
+
+            self.drop_payments_cache(account)
 
 
     def sync_account_transactions(self, account, pot_lookup, days):
@@ -502,15 +508,22 @@ def sync_account_transactions(self, account, pot_lookup, days):
 
         pot_account_ids = {}
 
+        changed = False
+
         for mo_transaction in mo_transactions:
-            transaction = self.add_transaction(account, mo_transaction, pot_account_ids)
+            transaction, was_changed = self.add_transaction(account, mo_transaction, pot_account_ids)
+
+            if was_changed and transaction.money_out:
+                changed = True
 
             total += 1
 
-        return pot_account_ids, total
+        return pot_account_ids, total, changed
 
 
     def sync_account_pot_transactions(self, account, pot_account_ids, pot_lookup, total, days):
+        changed = False
+
         for pot_account_id in pot_account_ids:
             if pot_lookup[pot_account_ids[pot_account_id]].deleted:
                 continue
@@ -520,12 +533,17 @@ def sync_account_pot_transactions(self, account, pot_account_ids, pot_lookup, to
             mo_pot_transactions = self.api.transactions(pot_account_id, days=days)
 
             for mo_pot_transaction in mo_pot_transactions:
-                transaction = self.add_transaction(account, mo_pot_transaction, pot_account_ids, pot_lookup[pot_account_ids[pot_account_id]].id)
+                transaction, was_changed = self.add_transaction(account, mo_pot_transaction, pot_account_ids, pot_lookup[pot_account_ids[pot_account_id]].id)
+
+                if was_changed and transaction.money_out:
+                    changed = True
 
                 total += 1
 
         Log().info(f"account {account.name} synced {total} transactions")
 
+        return changed
+
 
     def sync_account_pots(self, account):
         mo_pots = self.api.pots(account_id=account.account_id)
@@ -582,3 +600,34 @@ def get_or_create_account(self, mo_account, account_config):
         account.save()
 
         return account
+
+
+    def drop_payments_cache(self, account):
+        resp = Payments.search(filter_expression='account_id = :account_id', attr_values={':account_id': {'S': account.id}})
+
+        batch = []
+
+        for item in resp:
+            batch.append({
+                'DeleteRequest': {
+                    'Key': {
+                        'key': {'S': item.key}
+                    }
+                }
+            })
+
+            if len(batch) >= 25:
+                self.db.driver.dbd.batch_write_item(
+                    RequestItems={
+                        f"banking_payments": batch
+                    }
+                )
+
+                batch = []
+
+        if len(batch) >0:
+            self.db.driver.dbd.batch_write_item(
+                RequestItems={
+                    f"banking_payments": batch
+                }
+            )

monzo_utils/model/base.py

@@ -18,13 +18,25 @@ def one(cls, **kwargs):
         row = DB().one(table, **kwargs)
 
         if row:
-            table = re.sub(r'(?<!^)(?=[A-Z])', '_', cls.__name__).lower()
-
             return getattr(importlib.import_module(f"monzo_utils.model.{table}"), cls.__name__)(row)
 
         return None
 
 
+    @classmethod
+    def all(cls, **kwargs):
+        table = re.sub(r'(?<!^)(?=[A-Z])', '_', cls.__name__).lower()
+
+        resp = DB().all(table, **kwargs)
+
+        results = []
+
+        for row in resp:
+            results.append(getattr(importlib.import_module(f"monzo_utils.model.{table}"), cls.__name__)(row))
+
+        return results
+
+
     @classmethod
     def find(cls, param1=None, param2=None, param3=None, limit=None, orderby=None, orderdir=None, filter_expression=None, attr_names=None, attr_values=None, key_condition_expression=None):
         table = re.sub(r'(?<!^)(?=[A-Z])', '_', cls.__name__).lower()
@@ -178,13 +190,15 @@ def update(self, attributes):
 
     def save(self):
         if self.state['modified'] is False:
-            return
+            return False
 
         if self.id:
             DB().update(self.table, self.id, self.attributes)
         else:
             self.id = DB().create(self.table, self.attributes.copy())
 
+        return True
+
 
     def delete(self):
         DB().delete(self.table, self.primary_key, getattr(self, self.primary_key))

monzo_utils/model/finance.py

@@ -3,6 +3,7 @@
 from monzo_utils.model.transaction import Transaction
 from monzo_utils.model.provider import Provider
 from monzo_utils.model.account import Account
+from monzo_utils.model.payments import Payments
 
 class Finance(Payment):
 
@@ -62,6 +63,16 @@ def all_finance_transactions(self):
         if 'single_payment' in self.payment_config and self.payment_config['single_payment']:
             filter_expression, attr_names, attr_values, key_condition_expression = self.get_transaction_where_condition(amounts=False)
 
+            all_transactions_hash = self.hash('all_finance_transactions', filter_expression, attr_names, attr_values, key_condition_expression, self.account.id)
+
+            all_transactions = Payments().one(key=all_transactions_hash)
+
+            if all_transactions is not None:
+                all_finance_transactions = Transaction.all(id=all_transactions.transaction_ids.split(','))
+
+                self.cache['all_finance_transactions'] = all_finance_transactions
+                return all_finance_transactions
+
             self.cache['all_finance_transactions'] = []
 
             account_ids = [self.account.id]
@@ -86,14 +97,35 @@ def all_finance_transactions(self):
                     key_condition_expression=key_condition_expression
                 )
 
+            transaction_ids = []
             for transaction in transactions:
                 if 'monthly_day' in self.payment_config and transaction.date.day != self.payment_config['monthly_day']:
                     continue
 
                 self.cache['all_finance_transactions'].append(transaction)
+                transaction_ids.append(transaction.id)
+
+            all_transactions = Payments()
+            all_transactions.update({
+                'key': all_transactions_hash,
+                'transaction_ids': ','.join(transaction_ids),
+                'account_id': self.account.id
+            })
+            all_transactions.save()
+
         else:
             filter_expression, attr_names, attr_values, key_condition_expression = self.get_transaction_where_condition(amounts)
 
+            all_transactions_hash = self.hash('all_finance_transactions', filter_expression, attr_names, attr_values, key_condition_expression, self.account.id)
+
+            all_transactions = Payments().one(key=all_transactions_hash)
+
+            if all_transactions is not None:
+                all_finance_transactions = Transaction.all(id=all_transactions.transaction_ids.split(','))
+
+                self.cache['all_finance_transactions'] = all_finance_transactions
+                return all_finance_transactions
+
             self.cache['all_finance_transactions'] = []
 
             account_ids = [self.account.id]
@@ -118,12 +150,24 @@ def all_finance_transactions(self):
                     key_condition_expression=key_condition_expression
                 )
 
+            transaction_ids = []
+
             for transaction in transactions:
                 if 'monthly_day' in self.payment_config and transaction.date.day != self.payment_config['monthly_day']:
                     continue
 
                 self.cache['all_finance_transactions'].append(transaction)
 
+                transaction_ids.append(transaction.id)
+
+            all_transactions = Payments()
+            all_transactions.update({
+                'key': all_transactions_hash,
+                'transaction_ids': ','.join(transaction_ids),
+                'account_id': self.account.id
+            })
+            all_transactions.save()
+
         return self.cache['all_finance_transactions']