Skip to content

Commit

Permalink
Improve Swissquote Bank AG PDF-Importer
Browse files Browse the repository at this point in the history 
Add missing testparts
Add missing testCases
Optimize regEx
Optimize testcases (Standardization)
Rename testfiles (Standardization)
  • Loading branch information
@Nirus2000 @buchen
Nirus2000 authored and buchen committed Apr 18, 2022
1 parent 72178b4 commit 90261d1
Show file tree
Hide file tree
Showing 12 changed files with 268 additions and 191 deletions.
0 .../pdf/swissquote/SwissquoteDividende01.txt → ...tatransfer/pdf/swissquote/Dividende01.txt
View file
File renamed without changes.
0 .../pdf/swissquote/SwissquoteDividende02.txt → ...tatransfer/pdf/swissquote/Dividende02.txt
View file
File renamed without changes.
0 .../pdf/swissquote/SwissquoteDividende03.txt → ...tatransfer/pdf/swissquote/Dividende03.txt
View file
File renamed without changes.
0 ...df/swissquote/SwissquoteKapitalgewinn.txt → ...tatransfer/pdf/swissquote/Dividende04.txt
View file
File renamed without changes.
0 ...sfer/pdf/swissquote/SwissquoteGebuehr.txt → ...tatransfer/pdf/swissquote/Gebuehren01.txt
View file
File renamed without changes.
0 ...nsfer/pdf/swissquote/SwissquoteKauf01.txt → ...io/datatransfer/pdf/swissquote/Kauf01.txt
View file
File renamed without changes.
0 ...nsfer/pdf/swissquote/SwissquoteKauf02.txt → ...io/datatransfer/pdf/swissquote/Kauf02.txt
View file
File renamed without changes.
0 ...nsfer/pdf/swissquote/SwissquoteKauf03.txt → ...io/datatransfer/pdf/swissquote/Kauf03.txt
View file
File renamed without changes.
332 changes: 216 additions & 116 deletions ...ts/src/name/abuchen/portfolio/datatransfer/pdf/swissquote/SwissquotePDFExtractorTest.java
View file

Large diffs are not rendered by default.

0 ...er/pdf/swissquote/SwissquoteVerkauf01.txt → ...datatransfer/pdf/swissquote/Verkauf01.txt
View file
File renamed without changes.
0 ...er/pdf/swissquote/SwissquoteVerkauf02.txt → ...datatransfer/pdf/swissquote/Verkauf02.txt
View file
File renamed without changes.
127 changes: 52 additions & 75 deletions ...abuchen.portfolio/src/name/abuchen/portfolio/datatransfer/pdf/SwissquotePDFExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public SwissquotePDFExtractor(Client client)

addBuySellTransaction();
addDividendsTransaction();
addFeeTransaction();
addAccountFeesTransaction();
}

@Override
Expand Down Expand Up @@ -60,49 +60,51 @@ private void addBuySellTransaction()
t.setType(PortfolioTransaction.Type.SELL);
}
})

// APPLE ORD ISIN: US0378331005 NASDAQ New York
// 15 193 USD 2'895.00
.section("name", "isin", "shares", "currency")
.section("name", "isin", "currency")
.match("^(?<name>.*) ISIN: (?<isin>[\\w]{12}) .*$")
.match("^(?<shares>['.,\\d]+) ['.,\\d]+ (?<currency>[\\w]{3}) (['.,\\d]+)$")
.assign((t, v) -> {
t.setShares(asShares(v.get("shares")));
t.setSecurity(getOrCreateSecurity(v));
})
.match("^[\\.'\\d]+ [\\.'\\d]+ (?<currency>[\\w]{3}) [\\.'\\d]+$")
.assign((t, v) -> t.setSecurity(getOrCreateSecurity(v)))

// 15 193 USD 2'895.00
.section("shares")
.match("^(?<shares>[\\.'\\d]+) [\\.'\\d]+ [\\w]{3} [\\.'\\d]+$")
.assign((t, v) -> t.setShares(asShares(v.get("shares"))))

// Betrag belastet auf Kontonummer 99999901, Valutadatum 07.08.2019
// Betrag gutgeschrieben auf Ihrer Kontonummer 99999900, Valutadatum 07.02.2018
.section("date")
.match("^Betrag (belastet|gutgeschrieben) (auf|auf Ihrer) Kontonummer([\\s]+)? [\\d]+,([\\s]+)? Valutadatum (?<date>\\d+.\\d+.\\d{4})$")
.match("^Betrag (belastet|gutgeschrieben) (auf|auf Ihrer) Kontonummer([\\s]+)? [\\d]+, ([\\s]+)?Valutadatum (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$")
.assign((t, v) -> t.setDate(asDate(v.get("date"))))

// Zu Ihren Lasten USD 2'900.60
// Zu Ihren Gunsten CHF 8'198.70
.section("currency", "amount")
.match("^(Zu Ihren Lasten|Zu Ihren Gunsten) (?<currency>[\\w]{3}) (?<amount>['.,\\d]+)$")
.match("^Zu Ihren (Lasten|Gunsten) (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$")
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(v.get("currency"));
})

// Total DKK 37'301.50
// Wechselkurs 15.0198
.section("fxcurrency", "fxamount", "exchangeRate").optional()
.match("^Total (?<fxcurrency>[\\w]{3}) ([-\\s]+)?(?<fxamount>['.,\\d]+)$")
.match("^Wechselkurs (?<exchangeRate>['.,\\d]+)$")
.section("fxCurrency", "fxAmount", "exchangeRate").optional()
.match("^Total (?<fxCurrency>[\\w]{3}) ([\\s]+)?(?<fxAmount>[\\.'\\d]+)$")
.match("^Wechselkurs (?<exchangeRate>[\\.'\\d]+)$")
.assign((t, v) -> {
// read the forex currency, exchange rate and gross
// amount in forex currency
String forex = asCurrencyCode(v.get("fxcurrency"));
String forex = asCurrencyCode(v.get("fxCurrency"));
if (t.getPortfolioTransaction().getSecurity().getCurrencyCode().equals(forex))
{
BigDecimal exchangeRate = asExchangeRate(v.get("exchangeRate"));
BigDecimal reverseRate = BigDecimal.ONE.divide(exchangeRate, 10,
RoundingMode.HALF_DOWN);

// gross given in forex currency
long fxAmount = asAmount(v.get("fxamount"));
long fxAmount = asAmount(v.get("fxAmount"));
long amount = reverseRate.multiply(BigDecimal.valueOf(fxAmount))
.setScale(0, RoundingMode.HALF_DOWN).longValue();

Expand All @@ -117,20 +119,18 @@ private void addBuySellTransaction()
// Total DKK 35'410.5
// Wechselkurs 14.9827
// CHF 5'305.45
.section("amount", "currency", "exchangeRate", "forexCurrency", "forexAmount").optional()
.match("^Total (?<forexCurrency>\\w{3}+) (?<forexAmount>[\\d+',.]*)$")
.match("^Wechselkurs (?<exchangeRate>[\\d+',.]*)$")
.match("^(?<currency>\\w{3}+) (?<amount>[\\d+',.]*)$")
.section("amount", "currency", "exchangeRate", "fxCurrency", "fxAmount").optional()
.match("^Total (?<fxCurrency>[\\w]{3}) (?<fxAmount>[\\.'\\d]+)$")
.match("^Wechselkurs (?<exchangeRate>[\\.'\\d]+)$")
.match("^(?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$")
.assign((t, v) -> {
Money forex = Money.of(asCurrencyCode(v.get("forexCurrency")),
asAmount(v.get("forexAmount")));
Money forex = Money.of(asCurrencyCode(v.get("fxCurrency")), asAmount(v.get("fxAmount")));
Money gross = Money.of(asCurrencyCode(v.get("currency")), asAmount(v.get("amount")));

BigDecimal exchangeRate = asExchangeRate(v.get("exchangeRate"));
type.getCurrentContext().put("exchangeRate", exchangeRate.toPlainString());

if (forex.getCurrencyCode()
.equals(t.getPortfolioTransaction().getSecurity().getCurrencyCode()))
if (forex.getCurrencyCode().equals(t.getPortfolioTransaction().getSecurity().getCurrencyCode()))
{
Unit unit;
// Swissquote sometimes uses scaled exchanges
Expand Down Expand Up @@ -167,62 +167,38 @@ private void addDividendsTransaction()
Block block = new Block("^(Dividende|Kapitalgewinn) Unsere Referenz:(.*)$");
type.addBlock(block);
Transaction<AccountTransaction> pdfTransaction = new Transaction<AccountTransaction>()

.subject(() -> {
AccountTransaction entry = new AccountTransaction();
entry.setType(AccountTransaction.Type.DIVIDENDS);
return entry;
});
.subject(() -> {
AccountTransaction entry = new AccountTransaction();
entry.setType(AccountTransaction.Type.DIVIDENDS);
return entry;
});

pdfTransaction
// HARVEST CAPITAL CREDIT ORD ISIN: US41753F1093NKN: 350
// Anzahl 350
// Dividende 0.08 USD
.section("name", "isin", "shares", "currency")
.section("name", "isin", "currency")
.match("^(?<name>.*) ISIN: (?<isin>[\\w]{12}).*$")
.match("^Anzahl (?<shares>['.,\\d]+)$")
.match("^(Dividende|Kapitalgewinn) (['.,\\d]+) (?<currency>[\\w]{3})$")
.assign((t, v) -> {
t.setShares(asShares(v.get("shares")));
t.setSecurity(getOrCreateSecurity(v));
})
.match("^(Dividende|Kapitalgewinn) ([\\.'\\d]+) (?<currency>[\\w]{3})$")
.assign((t, v) -> t.setSecurity(getOrCreateSecurity(v)))

// Anzahl 350
.section("shares")
.match("^Anzahl (?<shares>[\\.'\\d]+)$")
.assign((t, v) -> t.setShares(asShares(v.get("shares"))))

// Ausführungsdatum 19.06.2019
.section("date")
.match("^Ausf.hrungsdatum (?<date>\\d+.\\d+.\\d{4})")
.match("^Ausf.hrungsdatum (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})")
.assign((t, v) -> t.setDateTime(asDate(v.get("date"))))

// Total USD 19.60
.section("currency", "amount").optional()
.match("^Total (?<currency>[\\w]{3}) (?<amount>[.,\\d]+)")
.section("currency", "amount")
.match("^Total (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)")
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(v.get("currency"));
})

// Total USD 13.52
// Wechselkurs USD / CHF : 0.94242
.section("amount", "currency", "forexCurrency", "exchangeRate").optional()
.match("^Total (?<currency>[\\w]{3}) (?<amount>['.,\\d]+)")
.match("^Wechselkurs [\\w]{3} \\/ (?<forexCurrency>[\\w]{3}) : (?<exchangeRate>['.,\\d]+)")
.assign((t, v) -> {
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));

BigDecimal exchangeRate = asExchangeRate(v.get("exchangeRate")).setScale(10,
RoundingMode.HALF_DOWN);
type.getCurrentContext().put("exchangeRate", exchangeRate.toPlainString());

BigDecimal inverseRate = BigDecimal.ONE.divide(exchangeRate, 10, RoundingMode.HALF_DOWN);

Money forex = Money.of(asCurrencyCode(v.get("forexCurrency")),
Math.round(t.getAmount() / inverseRate.doubleValue()));
Unit unit = new Unit(Unit.Type.GROSS_VALUE, t.getMonetaryAmount(), forex, inverseRate);

if (unit.getForex().getCurrencyCode().equals(t.getSecurity().getCurrencyCode()))
t.addUnit(unit);
})

.wrap(TransactionItem::new);

addTaxesSectionsTransaction(pdfTransaction, type);
Expand All @@ -231,12 +207,12 @@ private void addDividendsTransaction()
block.set(pdfTransaction);
}

private void addFeeTransaction()
private void addAccountFeesTransaction()
{
DocumentType type = new DocumentType("Depotgebühren");
DocumentType type = new DocumentType("Depotgeb.hren");
this.addDocumentTyp(type);

Block block = new Block("^Depotgebühren Unsere Referenz(.*)$");
Block block = new Block("^Depotgeb.hren Unsere Referenz(.*)$");
type.addBlock(block);
block.set(new Transaction<AccountTransaction>()

Expand All @@ -247,12 +223,13 @@ private void addFeeTransaction()
})

.section("date", "amount", "currency")
.match("^Valutadatum (?<date>\\d+.\\d+.\\d{4}+)$")
.match("^Betrag belastet (?<currency>\\w{3}+) (?<amount>[\\d+',.]*)$")
.match("^Valutadatum (?<date>[\\d]{2}\\.[\\d]{2}\\.[\\d]{4})$")
.match("^Betrag belastet (?<currency>[\\w]{3}) (?<amount>[\\.'\\d]+)$")
.assign((t, v) -> {
t.setDateTime(asDate(v.get("date")));
t.setAmount(asAmount(v.get("amount")));
t.setCurrencyCode(asCurrencyCode(v.get("currency")));
t.setNote("Depotgebühren");
})

.wrap(TransactionItem::new));
Expand All @@ -263,22 +240,22 @@ private <T extends Transaction<?>> void addTaxesSectionsTransaction(T transactio
transaction
// Abgabe (Eidg. Stempelsteuer) USD 4.75
.section("currency", "tax").optional()
.match("^Abgabe \\(Eidg. Stempelsteuer\\) (?<currency>[\\w]{3}) (?<tax>['.,\\d]+)$")
.match("^Abgabe \\(Eidg\\. Stempelsteuer\\) (?<currency>[\\w]{3}) (?<tax>[\\.'\\d]+)$")
.assign((t, v) -> processTaxEntries(t, v, type))

// Quellensteuer 15.00% (US) USD 4.20
.section("currency", "withHoldingTax").optional()
.match("^Quellensteuer ['.,\\d]+% \\(.*\\) (?<currency>[\\w]{3}) (?<withHoldingTax>['.,\\d]+)$")
.match("^Quellensteuer [\\.'\\d]+% \\(.*\\) (?<currency>[\\w]{3}) (?<withHoldingTax>[\\.'\\d]+)$")
.assign((t, v) -> processWithHoldingTaxEntries(t, v, "withHoldingTax", type))

// Zusätzlicher Steuerrückbehalt 15% USD 4.20
.section("currency", "tax").optional()
.match("^Zus.tzlicher Steuerr.ckbehalt ['.,\\d]+% (?<currency>[\\w]{3}) (?<tax>['.,\\d]+)$")
.match("^Zus.tzlicher Steuerr.ckbehalt [\\.'\\d]+% (?<currency>[\\w]{3}) (?<tax>[\\.'\\d]+)$")
.assign((t, v) -> processTaxEntries(t, v, type))

// Verrechnungssteuer 35% (CH) CHF 63.88
.section("currency", "tax").optional()
.match("^Verrechnungssteuer ['.,\\d]+% \\(.*\\) (?<currency>[\\w]{3}) (?<tax>['.,\\d]+)$")
.match("^Verrechnungssteuer [\\.'\\d]+% \\(.*\\) (?<currency>[\\w]{3}) (?<tax>[\\.'\\d]+)$")
.assign((t, v) -> processTaxEntries(t, v, type));
}

Expand All @@ -287,12 +264,12 @@ private <T extends Transaction<?>> void addFeesSectionsTransaction(T transaction
transaction
// Kommission Swissquote Bank AG USD 0.85
.section("currency", "fee").optional()
.match("^Kommission Swissquote Bank AG (?<currency>[\\w]{3}) (?<fee>['.,\\d]+)$")
.match("^Kommission Swissquote Bank AG (?<currency>[\\w]{3}) (?<fee>[\\.'\\d]+)$")
.assign((t, v) -> processFeeEntries(t, v, type))

// Börsengebühren CHF 1.00
.section("currency", "fee").optional()
.match("^B.rsengeb.hren (?<currency>[\\w]{3}) (?<fee>['.,\\d]+)$")
.match("^B.rsengeb.hren (?<currency>[\\w]{3}) (?<fee>[\\.'\\d]+)$")
.assign((t, v) -> processFeeEntries(t, v, type));
}

Expand Down

0 comments on commit 90261d1

Please sign in to comment.