Fix locale for mapping ubs_ch_fr (#64)

* Fixed common encoding code for de/fr/it_CH. Added test data

* Forgot to add test data ^^

* More doc

* travis: add language-pack-fr for ubs-ch-fr mapping. Upgrade (dev)-requirements for Python-3.9. ubs-ch-fr: fix description

* ubs-ch-fr: fix lint issuesm fix multiple bank tr header in test files

* Contributing doc fix (mkvirtualenv)

Co-authored-by: Marco Emilio Poleggi <marcoep@***>
Co-authored-by: Reuben Cummings <reubano@***>

.travis.yml

@@ -29,6 +29,12 @@ branches:
   except:
     - /^v[0-9]/
 
+# For ubs-ch-fr mapping
+addons:
+  apt:
+    packages:
+      - language-pack-fr
+
 before_install:
   - pip install -U pip
   - pip install wheel

README.md

@@ -143,6 +143,17 @@ optional arguments:
 
 	csv2ofx -m yoodlee file.csv
 
+
+#### Special cases
+
+Some banks, like *UBS Switzerland*, may provide CSV exports that are not
+readily tractable by csv2ofx because of extra header or trailing lines,
+redundant or unwanted columns. These input files can be preprocessed with the
+shipped `utilz/csvtrim` shell script. F.i., with mapping `ubs-ch-fr`:
+
+    csvtrim untrimmed.csv | csv2ofx -m ubs-ch-fr
+
+
 ## CUSTOMIZATION
 
 ### Code modification
@@ -251,7 +262,7 @@ cd csv2ofx
 2. Setup a new [virtualenv](http://www.virtualenv.org/en/latest/index.html)
 
 ```bash
-mkvirtualenv --no-site-packages csv2ofx
+mkvirtualenv -i pkutils csv2ofx
 activate csv2ofx
 python setup.py develop
 pip install -r dev-requirements.txt

csv2ofx/mappings/ubs-ch-fr.py

@@ -23,10 +23,10 @@
 """
 from operator import itemgetter
 
-# Financial numbers are expressed as "2'045.56" -- TO-DO switch to Babel?
+# Financial numbers are expressed as "2'045.56" in de/fr/it_CH (utf8 has some
+# glitches, so we go for the default one)
 import locale
-
-locale.setlocale(locale.LC_ALL, "fr_CH.UTF-8")
+locale.setlocale(locale.LC_NUMERIC, 'fr_CH')
 
 __author__ = 'Marco "sphakka" Poleggi'
 
@@ -38,13 +38,18 @@ def fixdate(ds):
     return ".".join((dmy[1], dmy[0], dmy[2]))
 
 
-def map_desc(tr):
-    description = tr["Description 2"] or tr["Description 1"]
-    return description + (": " + tr["Description 3"] if tr["Description 3"] else "")
+def map_descr(tr):
+    descr = tr["Description 2"] or tr["Description 1"]
+    return descr + (": " + tr["Description 3"] if tr["Description 3"] else "")
+
+
+def map_class(tr):
+    clss = tr["Description 1"]
+    return clss + (": " + tr["Description 2"] if tr["Description 2"] else "")
 
 
 def map_payee(tr):
-    return tr["Description 3"] if tr.get("Débit") != "" else tr["Description 2"]
+    return tr["Description 3"] if tr.get("Débit") else tr["Description 2"]
 
 
 mapping = {
@@ -60,12 +65,12 @@ def map_payee(tr):
     "amount": lambda tr: locale.atof(tr["Débit"] or tr["Crédit"]),
     # debits show _your_ notes in "Desc 2", whereas credits report the
     # _payee_. Thus a better "class" value comes from "Desc 1" + "Desc 2"
-    "class": lambda tr: tr["Description 1"] + (": " + tr["Description 2"] or ""),
+    "class": map_class,
     "notes": itemgetter("Description 2"),
     # switch day/month (maybe file a bug: always inverted when ambiguous like
     # '01.02.2018')
     "date": lambda tr: fixdate(tr["Date de valeur"]),
-    "desc": map_desc,
+    "desc": map_descr,
     "payee": map_payee,
     "check_num": itemgetter("N° de transaction"),
     "balance": lambda tr: locale.atof(tr["Solde"]),

csv2ofx/utilz/csvtrim

@@ -26,8 +26,10 @@ Usage:
 where
 
   CSV_FILE:  path to an existing file or '-' for stdin
-  FIELDS:    cut-style list of fields to keep. Default to '$dfields'
-  SEPARATOR: a single (escaped) character. Default to '$dseparator'
+  FIELDS:    cut-style list of fields to keep. Default: '$dfields'
+  SEPARATOR: a single (escaped) character. Default: '$dseparator'
+
+(default values are for exports from UBS CH (DE/FR/IT))
 
 e.g.
 

data/converted/ubs-ch-fr.qif

@@ -0,0 +1,25 @@
+!Account
+N0123 45678901.23A
+TBank
+^
+!Type:Bank
+NA01234BC01234567
+D2019-03-31
+LSolde prix prestations
+MSolde prix prestations
+T-10.00
+^
+N3456789ZT1234567
+D2019-02-28
+PASSOCIATION FOO-BAR
+LVirement postal: ASSOCIATION FOO-BAR
+MASSOCIATION FOO-BAR: BVD DE QUELQUE-PART 1, 1201 GENEVE, CH ASSOCIATION FOO-BAR
+T240.00
+^
+N9979360TI2115087
+D2019-04-27
+PQuuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur
+LOrdre e-banking: REMB-CASH
+MREMB-CASH: Quuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur REMB-CASH
+T-200.00
+^

data/test/ubs-ch-fr_trimmed.csv

@@ -0,0 +1,4 @@
+Produit;Monn.;Description;Date de valeur;Description 1;Description 2;Description 3;N° de transaction;Débit;Crédit;Solde
+0123 45678901.23A;CHF;Compte personnel UBS;31.03.2019;Solde prix prestations;;;A01234BC01234567;10.00;;11'373.94
+0123 45678901.23A;CHF;Compte personnel UBS;28.02.2019;Virement postal;ASSOCIATION FOO-BAR;BVD DE QUELQUE-PART 1, 1201 GENEVE, CH;3456789ZT1234567;;240.00;11'613.94
+0123 45678901.23A;CHF;Compte personnel UBS;27.04.2019;Ordre e-banking;REMB-CASH;Quuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur;9979360TI2115087;200.00;;11'413.94

dev-requirements.txt

@@ -1,7 +1,7 @@
 pip>20.0
 wheel>=0.29.0
 black>=19.3b0,<22.0
-coverage>=4.0.3,<6.0.0
+coverage>=4.0.3,<6.3.0
 flake8>=2.5.1,<5.0.0
 flake8-black>=0.1.1,<0.3.0
 nose>=1.3.7,<2.0.0

tests/test.py

@@ -147,6 +147,11 @@ def gen_test(raw):
             "pcmastercard.csv",
             "pcmastercard.ofx",
         ),
+        (
+            # N.B. input file obtained by pre-processing with
+            #    bin/csvtrim ubs-ch-fr.csv > ubs-ch-fr_trimmed.csv
+            ["-oq", "-m ubs-ch-fr"], "ubs-ch-fr_trimmed.csv", "ubs-ch-fr.qif"
+        ),
         (
             ["-o", "-m outbank", "-e 20190301", SERVER_DATE],
             "outbank.csv",