diff --git a/.travis.yml b/.travis.yml index 5b61dcc..05de432 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,12 @@ branches: except: - /^v[0-9]/ +# For ubs-ch-fr mapping +addons: + apt: + packages: + - language-pack-fr + before_install: - pip install -U pip - pip install wheel diff --git a/README.md b/README.md index 72bc2e9..e59541d 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,17 @@ optional arguments: csv2ofx -m yoodlee file.csv + +#### Special cases + +Some banks, like *UBS Switzerland*, may provide CSV exports that are not +readily tractable by csv2ofx because of extra header or trailing lines, +redundant or unwanted columns. These input files can be preprocessed with the +shipped `utilz/csvtrim` shell script. F.i., with mapping `ubs-ch-fr`: + + csvtrim untrimmed.csv | csv2ofx -m ubs-ch-fr + + ## CUSTOMIZATION ### Code modification @@ -251,7 +262,7 @@ cd csv2ofx 2. Setup a new [virtualenv](http://www.virtualenv.org/en/latest/index.html) ```bash -mkvirtualenv --no-site-packages csv2ofx +mkvirtualenv -i pkutils csv2ofx activate csv2ofx python setup.py develop pip install -r dev-requirements.txt diff --git a/csv2ofx/mappings/ubs-ch-fr.py b/csv2ofx/mappings/ubs-ch-fr.py index 03bb6cb..c743413 100644 --- a/csv2ofx/mappings/ubs-ch-fr.py +++ b/csv2ofx/mappings/ubs-ch-fr.py @@ -23,10 +23,10 @@ """ from operator import itemgetter -# Financial numbers are expressed as "2'045.56" -- TO-DO switch to Babel? +# Financial numbers are expressed as "2'045.56" in de/fr/it_CH (utf8 has some +# glitches, so we go for the default one) import locale - -locale.setlocale(locale.LC_ALL, "fr_CH.UTF-8") +locale.setlocale(locale.LC_NUMERIC, 'fr_CH') __author__ = 'Marco "sphakka" Poleggi' @@ -38,13 +38,18 @@ def fixdate(ds): return ".".join((dmy[1], dmy[0], dmy[2])) -def map_desc(tr): - description = tr["Description 2"] or tr["Description 1"] - return description + (": " + tr["Description 3"] if tr["Description 3"] else "") +def map_descr(tr): + descr = tr["Description 2"] or tr["Description 1"] + return descr + (": " + tr["Description 3"] if tr["Description 3"] else "") + + +def map_class(tr): + clss = tr["Description 1"] + return clss + (": " + tr["Description 2"] if tr["Description 2"] else "") def map_payee(tr): - return tr["Description 3"] if tr.get("Débit") != "" else tr["Description 2"] + return tr["Description 3"] if tr.get("Débit") else tr["Description 2"] mapping = { @@ -60,12 +65,12 @@ def map_payee(tr): "amount": lambda tr: locale.atof(tr["Débit"] or tr["Crédit"]), # debits show _your_ notes in "Desc 2", whereas credits report the # _payee_. Thus a better "class" value comes from "Desc 1" + "Desc 2" - "class": lambda tr: tr["Description 1"] + (": " + tr["Description 2"] or ""), + "class": map_class, "notes": itemgetter("Description 2"), # switch day/month (maybe file a bug: always inverted when ambiguous like # '01.02.2018') "date": lambda tr: fixdate(tr["Date de valeur"]), - "desc": map_desc, + "desc": map_descr, "payee": map_payee, "check_num": itemgetter("N° de transaction"), "balance": lambda tr: locale.atof(tr["Solde"]), diff --git a/csv2ofx/utilz/csvtrim b/csv2ofx/utilz/csvtrim index ec908b1..68ea3a0 100755 --- a/csv2ofx/utilz/csvtrim +++ b/csv2ofx/utilz/csvtrim @@ -26,8 +26,10 @@ Usage: where CSV_FILE: path to an existing file or '-' for stdin - FIELDS: cut-style list of fields to keep. Default to '$dfields' - SEPARATOR: a single (escaped) character. Default to '$dseparator' + FIELDS: cut-style list of fields to keep. Default: '$dfields' + SEPARATOR: a single (escaped) character. Default: '$dseparator' + +(default values are for exports from UBS CH (DE/FR/IT)) e.g. diff --git a/data/converted/ubs-ch-fr.qif b/data/converted/ubs-ch-fr.qif new file mode 100644 index 0000000..a20267f --- /dev/null +++ b/data/converted/ubs-ch-fr.qif @@ -0,0 +1,25 @@ +!Account +N0123 45678901.23A +TBank +^ +!Type:Bank +NA01234BC01234567 +D2019-03-31 +LSolde prix prestations +MSolde prix prestations +T-10.00 +^ +N3456789ZT1234567 +D2019-02-28 +PASSOCIATION FOO-BAR +LVirement postal: ASSOCIATION FOO-BAR +MASSOCIATION FOO-BAR: BVD DE QUELQUE-PART 1, 1201 GENEVE, CH ASSOCIATION FOO-BAR +T240.00 +^ +N9979360TI2115087 +D2019-04-27 +PQuuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur +LOrdre e-banking: REMB-CASH +MREMB-CASH: Quuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur REMB-CASH +T-200.00 +^ diff --git a/data/test/default.csv b/data/test/default.csv old mode 100755 new mode 100644 diff --git a/data/test/ubs-ch-fr_trimmed.csv b/data/test/ubs-ch-fr_trimmed.csv new file mode 100644 index 0000000..330c204 --- /dev/null +++ b/data/test/ubs-ch-fr_trimmed.csv @@ -0,0 +1,4 @@ +Produit;Monn.;Description;Date de valeur;Description 1;Description 2;Description 3;N° de transaction;Débit;Crédit;Solde +0123 45678901.23A;CHF;Compte personnel UBS;31.03.2019;Solde prix prestations;;;A01234BC01234567;10.00;;11'373.94 +0123 45678901.23A;CHF;Compte personnel UBS;28.02.2019;Virement postal;ASSOCIATION FOO-BAR;BVD DE QUELQUE-PART 1, 1201 GENEVE, CH;3456789ZT1234567;;240.00;11'613.94 +0123 45678901.23A;CHF;Compte personnel UBS;27.04.2019;Ordre e-banking;REMB-CASH;Quuz-baz SàrL, CH - 1203 GENEVE, E-Banking CHF intérieur;9979360TI2115087;200.00;;11'413.94 diff --git a/dev-requirements.txt b/dev-requirements.txt index 6142359..801be00 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,7 +1,7 @@ pip>20.0 wheel>=0.29.0 black>=19.3b0,<22.0 -coverage>=4.0.3,<6.0.0 +coverage>=4.0.3,<6.3.0 flake8>=2.5.1,<5.0.0 flake8-black>=0.1.1,<0.3.0 nose>=1.3.7,<2.0.0 diff --git a/tests/test.py b/tests/test.py index 79f4b13..ff4befa 100755 --- a/tests/test.py +++ b/tests/test.py @@ -147,6 +147,11 @@ def gen_test(raw): "pcmastercard.csv", "pcmastercard.ofx", ), + ( + # N.B. input file obtained by pre-processing with + # bin/csvtrim ubs-ch-fr.csv > ubs-ch-fr_trimmed.csv + ["-oq", "-m ubs-ch-fr"], "ubs-ch-fr_trimmed.csv", "ubs-ch-fr.qif" + ), ( ["-o", "-m outbank", "-e 20190301", SERVER_DATE], "outbank.csv",