Merge branch 'release/4.30.0'

mt940/__about__.py

@@ -6,7 +6,7 @@
 statistics and manipulation.
 '''.strip().split())
 __email__ = '[email protected]'
-__version__ = '4.29.0'
+__version__ = '4.30.0'
 __license__ = 'BSD'
 __copyright__ = 'Copyright 2015 Rick van Hattem (wolph)'
 __url__ = 'https://github.com/WoLpH/mt940'

mt940/models.py

@@ -261,7 +261,10 @@ class Transactions(abc.Sequence):
         pre_related_reference=[],
         post_related_reference=[],
         pre_statement=[processors.date_fixup_pre_processor],
-        post_statement=[processors.date_cleanup_post_processor],
+        post_statement=[
+            processors.date_cleanup_post_processor,
+            processors.transactions_to_transaction('transaction_reference'),
+        ],
         pre_statement_number=[],
         post_statement_number=[],
         pre_non_swift=[],
@@ -282,6 +285,12 @@ class Transactions(abc.Sequence):
         pre_sum_debit_entries=[],
         post_sum_debit_entries=[])
 
+    def __getstate__(self):  # pragma: no cover
+        # Processors are not always safe to dump so ignore them entirely
+        state = self.__dict__.copy()
+        del state['processors']
+        return state
+
     def __init__(self, processors=None, tags=None):
         self.processors = self.DEFAULT_PROCESSORS.copy()
         self.tags = Transactions.defaultTags().copy()

mt940/processors.py

@@ -46,7 +46,8 @@ def mBank_set_transaction_code(transactions, tag, tag_dict, *args):
     processing
     """
     tag_dict['transaction_code'] = int(
-        tag_dict[tag.slug].split(';')[0].split(' ', 1)[0])
+        tag_dict[tag.slug].split(';')[0].split(' ', 1)[0]
+    )
 
     return tag_dict
 
@@ -67,8 +68,10 @@ def mBank_set_iph_id(transactions, tag, tag_dict, *args):
     return tag_dict
 
 
-tnr_re = re.compile(r'TNR:[ \n](?P<tnr>\d+\.\d+)',
-                    flags=re.MULTILINE | re.UNICODE)
+tnr_re = re.compile(
+    r'TNR:[ \n](?P<tnr>\d+\.\d+)',
+    flags=re.MULTILINE | re.UNICODE
+)
 
 
 def mBank_set_tnr(transactions, tag, tag_dict, *args):
@@ -205,8 +208,8 @@ def _parse_mt940_gvcodes(purpose):
     return result
 
 
-def transaction_details_post_processor(transactions, tag, tag_dict, result,
-                                       space=False):
+def transaction_details_post_processor(
+        transactions, tag, tag_dict, result, space=False):
     '''Parse the extra details in some transaction formats such as the 60-65
     keys.
 
@@ -226,8 +229,10 @@ def transaction_details_post_processor(transactions, tag, tag_dict, result,
 
         purpose = result.get('purpose')
 
-        if purpose and any(gvk in purpose for gvk in GVC_KEYS
-                           if gvk != ''):  # pragma: no branch
+        if purpose and any(
+            gvk in purpose for gvk in GVC_KEYS
+            if gvk != ''
+        ):  # pragma: no branch
             result.update(_parse_mt940_gvcodes(result['purpose']))
 
         del result['transaction_details']
@@ -236,4 +241,29 @@ def transaction_details_post_processor(transactions, tag, tag_dict, result,
 
 
 transaction_details_post_processor_with_space = functools.partial(
-    transaction_details_post_processor, space=True)
+    transaction_details_post_processor, space=True
+)
+
+
+def transactions_to_transaction(*keys):
+    '''Copy the global transactions details to the transaction.
+
+    Args:
+        *keys (str): the keys to copy to the transaction
+    '''
+    def _transactions_to_transaction(transactions, tag, tag_dict, result):
+        '''Copy the global transactions details to the transaction.
+
+        Args:
+            transactions (mt940.models.Transactions): list of transactions
+            tag (mt940.tags.Tag): tag
+            tag_dict (dict): dict with the raw tag details
+            result (dict): the resulting tag dict
+        '''
+        for key in keys:
+            if key in transactions.data:
+                result[key] = transactions.data[key]
+
+        return result
+
+    return _transactions_to_transaction

mt940/tags.py

@@ -84,7 +84,6 @@
 
     print('MT940 requires the `enum34` package', file=sys.stderr)
 
-
     class enum(object):
         @staticmethod
         def unique(*args, **kwargs):
@@ -109,7 +108,7 @@ def parse(self, transactions, value):
         match = self.re.match(value)
         if match:  # pragma: no branch
             self.logger.debug(
-                'matched (%d) %r against %r, got: %s',
+                'matched (%d) %r against "%s", got: %s',
                 len(value), value, self.pattern,
                 match.groupdict()
             )

mt940_tests/betterplace/amount_formats.yml

@@ -9,53 +9,11 @@ data:
     - !!binary |
       B9oDEg==
     status: C
-processors:
-  post_account_identification: []
-  post_available_balance: []
-  post_closing_balance: []
-  post_date_time_indication: []
-  post_final_closing_balance: []
-  post_final_opening_balance: []
-  post_floor_limit_indicator: []
-  post_forward_available_balance: []
-  post_intermediate_closing_balance: []
-  post_intermediate_opening_balance: []
-  post_non_swift: []
-  post_opening_balance: []
-  post_related_reference: []
-  post_statement:
-  - !!python/name:mt940.processors.date_cleanup_post_processor ''
-  post_statement_number: []
-  post_sum_credit_entries: []
-  post_sum_debit_entries: []
-  post_transaction_details:
-  - !!python/name:mt940.processors.transaction_details_post_processor ''
-  post_transaction_reference_number: []
-  pre_account_identification: []
-  pre_available_balance: []
-  pre_closing_balance: []
-  pre_date_time_indication: []
-  pre_final_closing_balance: []
-  pre_final_opening_balance: []
-  pre_floor_limit_indicator: []
-  pre_forward_available_balance: []
-  pre_intermediate_closing_balance: []
-  pre_intermediate_opening_balance: []
-  pre_non_swift: []
-  pre_opening_balance: []
-  pre_related_reference: []
-  pre_statement:
-  - !!python/name:mt940.processors.date_fixup_pre_processor ''
-  pre_statement_number: []
-  pre_sum_credit_entries: []
-  pre_sum_debit_entries: []
-  pre_transaction_details: []
-  pre_transaction_reference_number: []
 tags:
   13: !!python/object:mt940.tags.DateTimeIndication
     re: !!python/object/apply:re._compile
-    - "^\n    (?P<year>\\d{2})\n    (?P<month>\\d{2})\n    (?P<day>\\d{2})\n    (?P<hour>\\\
-      d{2})\n    (?P<minute>\\d{2})\n    (\\+(?P<offset>\\d{4})|)\n    "
+    - "^\n    (?P<year>\\d{2})\n    (?P<month>\\d{2})\n    (?P<day>\\d{2})\n    (?P<hour>\\d{2})\n
+      \   (?P<minute>\\d{2})\n    (\\+(?P<offset>\\d{4})|)\n    "
     - 98
   20: !!python/object:mt940.tags.TransactionReferenceNumber
     re: !!python/object/apply:re._compile
@@ -71,34 +29,35 @@ tags:
     - 98
   28: !!python/object:mt940.tags.StatementNumber
     re: !!python/object/apply:re._compile
-    - "\n    (?P<statement_number>\\d{1,5})  # 5n\n    (?:/?(?P<sequence_number>\\\
-      d{1,5}))?  # [/5n]\n    $"
+    - "\n    (?P<statement_number>\\d{1,5})  # 5n\n    (?:/?(?P<sequence_number>\\d{1,5}))?
+      \ # [/5n]\n    $"
     - 98
   60: !!python/object:mt940.tags.OpeningBalance
     re: &id001 !!python/object/apply:re._compile
-    - "^\n    (?P<status>[DC])  # 1!a Debit/Credit\n    (?P<year>\\d{2})  # 6!n Value\
-      \ Date (YYMMDD)\n    (?P<month>\\d{2})\n    (?P<day>\\d{2})\n    (?P<currency>.{3})\
-      \  # 3!a Currency\n    (?P<amount>[0-9,]{0,16})  # 15d Amount (includes decimal\
-      \ sign, so 16)\n    "
+    - "^\n    (?P<status>[DC])  # 1!a Debit/Credit\n    (?P<year>\\d{2})  # 6!n Value
+      Date (YYMMDD)\n    (?P<month>\\d{2})\n    (?P<day>\\d{2})\n    (?P<currency>.{3})
+      \ # 3!a Currency\n    (?P<amount>[0-9,]{0,16})  # 15d Amount (includes decimal
+      sign, so 16)\n    "
     - 98
   60M: !!python/object:mt940.tags.IntermediateOpeningBalance
     re: *id001
   60F: !!python/object:mt940.tags.FinalOpeningBalance
     re: *id001
   61: !!python/object:mt940.tags.Statement
     re: !!python/object/apply:re._compile
-    - "^\n    (?P<year>\\d{2})  # 6!n Value Date (YYMMDD)\n    (?P<month>\\d{2})\n\
-      \    (?P<day>\\d{2})\n    (?P<entry_month>\\d{2})?  # [4!n] Entry Date (MMDD)\n\
-      \    (?P<entry_day>\\d{2})?\n    (?P<status>R?[DC])  # 2a Debit/Credit Mark\n\
-      \    (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd character of the currency\n\
-      \                            # code, if needed)\n    \\n? # apparently some\
-      \ banks (sparkassen) incorporate newlines here\n    (?P<amount>[\\d,]{1,15})\
-      \  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9 ]{3})?  # 1!a3!c Transaction Type Identification\
-      \ Code\n    # We need the (slow) repeating negative lookahead to search for\
-      \ // so we\n    # don't acciddntly include the bank reference in the customer\
-      \ reference.\n    (?P<customer_reference>((?!//)[^\\n]){0,16})  # 16x Customer\
-      \ Reference\n    (//(?P<bank_reference>.{0,23}))?  # [//23x] Bank Reference\n\
-      \    (\\n?(?P<extra_details>.{0,34}))?  # [34x] Supplementary Details\n    $"
+    - "^\n    (?P<year>\\d{2})  # 6!n Value Date (YYMMDD)\n    (?P<month>\\d{2})\n
+      \   (?P<day>\\d{2})\n    (?P<entry_month>\\d{2})?  # [4!n] Entry Date (MMDD)\n
+      \   (?P<entry_day>\\d{2})?\n    (?P<status>R?[DC])  # 2a Debit/Credit Mark\n
+      \   (?P<funds_code>[A-Z])? # [1!a] Funds Code (3rd character of the currency\n
+      \                           # code, if needed)\n    [\\n ]? # apparently some
+      banks (sparkassen) incorporate newlines here\n    # cuscal can also send a space
+      here as well\n    (?P<amount>[\\d,]{1,15})  # 15d Amount\n    (?P<id>[A-Z][A-Z0-9
+      ]{3})?  # 1!a3!c Transaction Type Identification Code\n    # We need the (slow)
+      repeating negative lookahead to search for // so we\n    # don't acciddntly
+      include the bank reference in the customer reference.\n    (?P<customer_reference>((?!//)[^\\n]){0,16})
+      \ # 16x Customer Reference\n    (//(?P<bank_reference>.{0,23}))?  # [//23x]
+      Bank Reference\n    (\\n?(?P<extra_details>.{0,34}))?  # [34x] Supplementary
+      Details\n    $"
     - 98
   62: !!python/object:mt940.tags.ClosingBalance
     re: *id001
@@ -112,24 +71,24 @@ tags:
     re: *id001
   86: !!python/object:mt940.tags.TransactionDetails
     re: !!python/object/apply:re._compile
-    - "\n    (?P<transaction_details>(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n\
-      \    "
+    - "\n    (?P<transaction_details>(([\\s\\S]{0,65}\\r?\\n?){0,8}[\\s\\S]{0,65}))\n
+      \   "
     - 98
   34: !!python/object:mt940.tags.FloorLimitIndicator
     re: !!python/object/apply:re._compile
-    - "^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n    (?P<status>[DC ]?)  # 2a\
-      \ Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})  # 15d Amount (includes decimal\
-      \ sign, so 16)\n    $"
+    - "^\n    (?P<currency>[A-Z]{3})  # 3!a Currency\n    (?P<status>[DC ]?)  # 2a
+      Debit/Credit Mark\n    (?P<amount>[0-9,]{0,16})  # 15d Amount (includes decimal
+      sign, so 16)\n    $"
     - 98
   NS: !!python/object:mt940.tags.NonSwift
     re: !!python/object/apply:re._compile
-    - "\n    (?P<non_swift>\n        (\n            (\\d{2}.{0,})\n            (\\\
-      n\\d{2}.{0,})*\n        )|(\n            [^\\n]*\n        )\n    )\n    $"
+    - "\n    (?P<non_swift>\n        (\n            (\\d{2}.{0,})\n            (\\n\\d{2}.{0,})*\n
+      \       )|(\n            [^\\n]*\n        )\n    )\n    $"
     - 98
   90: !!python/object:mt940.tags.SumEntries
     re: &id002 !!python/object/apply:re._compile
-    - "^\n    (?P<number>\\d*)\n    (?P<currency>.{3})  # 3!a Currency\n    (?P<amount>[\\\
-      d,]{1,15})  # 15d Amount\n    "
+    - "^\n    (?P<number>\\d*)\n    (?P<currency>.{3})  # 3!a Currency\n    (?P<amount>[\\d,]{1,15})
+      \ # 15d Amount\n    "
     - 98
   90D: !!python/object:mt940.tags.SumDebitEntries
     re: *id002