From 5de3ad73e5d1c43e572c23f92d423c513831bbd7 Mon Sep 17 00:00:00 2001 From: Artem Hluvchynskyi Date: Sun, 13 May 2018 21:00:17 +0300 Subject: [PATCH] New batch of flags and fields estate_has_hidden_cost: resolves #22 family_member_did_not_provide_information_flag: resolves #23 corprights_abroad_flag: resolves #24 income_prize_fact and incomes.prize: resolves #25 bank_accounts_abroad_flag: resolves #26 assets_lend_3rd_parties_flag: resolves #27 cash_flag_500k: resolves #28 income_presents_to_total_flag_50: resolves #29 In addition to the above, the version of CouchDB is updated to 2.1.1 in the Dockerfile. --- couchdb/Dockerfile | 2 +- data/profiles/aggregated.json | 6 +- utils/Dockerfile | 6 +- views/aggregated/step_11_incomes.es6 | 6 +- views/aggregated/step_12_assets.es6 | 8 ++- views/red_flags.es6 | 91 ++++++++++++++++++++++++++-- 6 files changed, 107 insertions(+), 12 deletions(-) diff --git a/couchdb/Dockerfile b/couchdb/Dockerfile index a3fef1a..de6d2f7 100644 --- a/couchdb/Dockerfile +++ b/couchdb/Dockerfile @@ -1,4 +1,4 @@ -FROM klaemo/couchdb:2.0.0 +FROM apache/couchdb:2.1.1 # Clean original build artefacts and compile couch-chakra RUN rm -rf /etc/apt/sources.list.d/* &&\ echo 'deb http://ftp.debian.org/debian jessie-backports main' >> /etc/apt/sources.list diff --git a/data/profiles/aggregated.json b/data/profiles/aggregated.json index 1882ef7..eab7ce5 100644 --- a/data/profiles/aggregated.json +++ b/data/profiles/aggregated.json @@ -58,7 +58,7 @@ "view": "aggregated.step_11_incomes", "output": "{datadir}/export/step_11_incomes_agg.csv", "columns": [ - "id", "incomes.declarant", "incomes.family", "incomes.total", "incomes.has_hidden", + "id", "incomes.declarant", "incomes.family", "incomes.total", "incomes.prize", "incomes.has_hidden", "incomes.has_foreign", "incomes.family_ratio" ] }, @@ -66,7 +66,7 @@ "view": "aggregated.step_12_assets", "output": "{datadir}/export/step_12_assets_agg.csv", "columns": [ - "id", "assets.declarant", "assets.family", "assets.total", "assets.has_hidden", + "id", "assets.declarant", "assets.family", "assets.total", "assets.lent", "assets.has_hidden", "assets.has_foreign", "assets.family_ratio" ] }, @@ -77,6 +77,8 @@ "id", "assets_to_income_flag", "income_presents_to_total_flag", "expenses_to_inc_and_assets_flag", "liabilities_to_inc_and_assets_flag", "cash_flag", "garage_wo_car_flag", "house_no_land_flag", "lux_cars_flag", "lux_cars_flag_v2", "vehicle_purch_no_cost_flag", "estate_purch_no_cost_flag", + "estate_has_hidden_cost", "corprights_abroad_flag", "bank_accounts_abroad_flag", "income_prize_fact", + "assets_lend_3rd_parties", "cash_flag_500k", "income_presents_to_total_flag_50", "family_member_did_not_provide_information_flag", "expenses.total", "liabilities.total", "assets.cash.total", "incomes.presents.all" ] } diff --git a/utils/Dockerfile b/utils/Dockerfile index c73aea3..601e616 100644 --- a/utils/Dockerfile +++ b/utils/Dockerfile @@ -5,7 +5,7 @@ COPY requirements.txt /usr/local/dragnet WORKDIR /usr/local/dragnet RUN pip3 install -r requirements.txt COPY . /usr/local/dragnet -RUN mkdir /mnt/dragnet_data -RUN mkdir /mnt/declarations_bank -RUN mkdir /usr/local/dragnet/views +RUN mkdir -p /mnt/dragnet_data \ + && mkdir -p /mnt/declarations_bank \ + && mkdir -p /usr/local/dragnet/views VOLUME ["/mnt/dragnet_data", "/mnt/declarations_bank", "/usr/local/dragnet/views"] diff --git a/views/aggregated/step_11_incomes.es6 b/views/aggregated/step_11_incomes.es6 index c25b2c3..3274cf6 100644 --- a/views/aggregated/step_11_incomes.es6 +++ b/views/aggregated/step_11_incomes.es6 @@ -6,6 +6,7 @@ let declarant_income = 0.0, family_income = 0.0, total_income = 0.0, + total_prizes = 0.0, has_hidden = false, has_foreign = false; for (let key in nacp_doc.step_11) { @@ -25,6 +26,8 @@ declarant_income += income_doc.sizeIncome; else if (String(income_doc.person) in (nacp_doc.step_2 || {})) family_income += income_doc.sizeIncome; + if (income_doc.dnt_objectType_encoded == 'prize') + total_prizes += income_doc.sizeIncome; total_income += income_doc.sizeIncome; } @@ -32,5 +35,6 @@ if (total_income != 0.0) family_ratio = family_income / total_income; - emit(doc._id, [doc.doc_uuid, declarant_income, family_income, total_income, has_hidden, has_foreign, family_ratio]); + emit(doc._id, [doc.doc_uuid, declarant_income, family_income, total_income, total_prizes, has_hidden, has_foreign, + family_ratio]); } \ No newline at end of file diff --git a/views/aggregated/step_12_assets.es6 b/views/aggregated/step_12_assets.es6 index 3f2b498..2d0fae5 100644 --- a/views/aggregated/step_12_assets.es6 +++ b/views/aggregated/step_12_assets.es6 @@ -192,6 +192,7 @@ let declarant_assets = 0.0, family_assets = 0.0, total_assets = 0.0, + lent_assets = 0.0, has_hidden = false, has_foreign = false; for (let key in nacp_doc.step_12) { @@ -226,6 +227,10 @@ declarant_assets += val; else if (String(assets_doc.person) in (nacp_doc.step_2 || {})) family_assets += val; + + if (assets_doc.dnt_objectType_encoded == 'lend') + lent_assets += val; + total_assets += val; } @@ -233,5 +238,6 @@ if (total_assets != 0.0) family_ratio = family_assets / total_assets; - emit(doc._id, [doc.doc_uuid, declarant_assets, family_assets, total_assets, has_hidden, has_foreign, family_ratio]); + emit(doc._id, [doc.doc_uuid, declarant_assets, family_assets, total_assets, lent_assets, has_hidden, has_foreign, + family_ratio]); } \ No newline at end of file diff --git a/views/red_flags.es6 b/views/red_flags.es6 index ffd5e5b..fec2d86 100644 --- a/views/red_flags.es6 +++ b/views/red_flags.es6 @@ -211,27 +211,40 @@ // Thresholds const assets_to_income_fraction = 10, income_presents_to_total_fraction = 0.75, + income_presents_to_total_fraction_50 = 0.5, expenses_to_other_fraction = 3, liabilities_to_other_fraction = 2, - total_cash_threshold = 5000000; + total_cash_threshold = 5000000, + total_cash_threshold_500k = 500000, + lent_assets_threshold = 300000; // Flags let assets_to_income = false, expenses_to_inc_and_assets = false, liabilities_to_inc_and_assets = false, + loan_shark = false, income_presents_to_total = false, + income_presents_to_total_50 = false, + income_has_prizes = false, jar_of_cash = false, + jar_of_pocket_cash = false, garage_wo_car = false, has_luxury_cars = false, has_luxury_cars_v2 = false, vehicle_purch_no_cost = false, estate_purch_no_cost = false, - house_no_land = false; + house_no_land = false, + estate_has_hidden_cost = false, + corprights_has_foreign = false, + has_foreign_bank_acc = false, + hidden_in_family = false; // Helper values let total_income = 0.0, total_presents = 0.0, + total_prizes = 0.0, total_expenses = 0.0, total_assets = 0.0, + lent_assets = 0.0, total_cash = 0.0, total_liabilities = 0.0, has_garage = false, @@ -293,6 +306,22 @@ const owning_date = estate_doc.owningDate.split('.'); if (owning_date.length == 3 && owning_date[2] == nacp_doc.step_0.declarationYear1 && !estate_doc.costDate) estate_purch_no_cost = true; + + if (estate_doc.dnt_costDate_hidden && estate_doc.dnt_costAssessment_hidden) + estate_has_hidden_cost = true; + + if (String(estate_doc.person) in (nacp_doc.step_2 || {}) && (estate_doc.dnt_costDate_hidden || estate_doc.dnt_costAssessment_hidden)) + hidden_in_family = true; + } + } + if (nacp_doc.step_5) { + for (let key in nacp_doc.step_5) { + const valuables_doc = nacp_doc.step_5[key]; + if (typeof(valuables_doc) != 'object') + continue; + + if (String(valuables_doc.person) in (nacp_doc.step_2 || {}) && valuables_doc.dnt_costDateUse_hidden) + hidden_in_family = true; } } if (nacp_doc.step_6) { @@ -308,6 +337,9 @@ if (owning_date.length == 3 && owning_date[2] == nacp_doc.step_0.declarationYear1 && !vehicle_doc.costDate) vehicle_purch_no_cost = true; + if (String(vehicle_doc.person) in (nacp_doc.step_2 || {}) && (vehicle_doc.dnt_costDate_hidden || vehicle_doc.dnt_graduationYear_hidden)) + hidden_in_family = true; + let full_name = ''; if (vehicle_doc.brand) full_name += vehicle_doc.brand; @@ -327,15 +359,42 @@ } } } + if (nacp_doc.step_7) { + for (let key in nacp_doc.step_7) { + const securities_doc = nacp_doc.step_7[key]; + if (typeof(securities_doc) != 'object') + continue; + + if (String(securities_doc.person) in (nacp_doc.step_2 || {}) && (securities_doc.dnt_amount_hidden || securities_doc.dnt_cost_hidden)) + hidden_in_family = true; + } + } + if (nacp_doc.step_8) { + for (let key in nacp_doc.step_8) { + const corp_doc = nacp_doc.step_8[key]; + if (typeof(corp_doc) != 'object') + continue; + if (corp_doc.country != '1') + corprights_has_foreign = true; + if (String(corp_doc.person) in (nacp_doc.step_2 || {}) && corp_doc.dnt_cost_hidden) + hidden_in_family = true; + } + } if (nacp_doc.step_11) { for (let key in nacp_doc.step_11) { const income_doc = nacp_doc.step_11[key]; if (typeof(income_doc) != 'object') continue; + + if (String(income_doc.person) in (nacp_doc.step_2 || {}) && income_doc.dnt_sizeIncome_hidden) + hidden_in_family = true; + if (income_doc.sizeIncome === undefined) continue; if (present_types.indexOf(income_doc.dnt_objectType_encoded) != -1) total_presents += income_doc.sizeIncome; + if (income_doc.dnt_objectType_encoded == 'prize') + total_prizes += income_doc.sizeIncome; total_income += income_doc.sizeIncome; } } @@ -344,6 +403,13 @@ const assets_doc = nacp_doc.step_12[key]; if (typeof(assets_doc) != 'object') continue; + + if (assets_doc.dnt_objectType_encoded == 'bank' && assets_doc.dnt_is_foreign) + has_foreign_bank_acc = true; + + if (String(assets_doc.person) in (nacp_doc.step_2 || {}) && assets_doc.dnt_sizeAssets_hidden) + hidden_in_family = true; + if (assets_doc.sizeAssets === undefined) continue; @@ -353,6 +419,8 @@ if (assets_doc.dnt_objectType_encoded == 'cash') total_cash += val; + else if (assets_doc.dnt_objectType_encoded == 'lend') + lent_assets += val; total_assets += val; } } @@ -361,6 +429,10 @@ const liability_doc = nacp_doc.step_13[key]; if (typeof(liability_doc) != 'object') continue; + + if (String(liability_doc.person) in (nacp_doc.step_2 || {}) && liability_doc.dnt_sizeObligation_hidden) + hidden_in_family = true; + if (liability_doc.sizeObligation === undefined) continue; @@ -376,6 +448,10 @@ const expense_doc = nacp_doc.step_14[key]; if (typeof(expense_doc) != 'object') continue; + + if (String(expense_doc.person) in (nacp_doc.step_2 || {}) && expense_doc.dnt_costAmount_hidden) + hidden_in_family = true; + if (expense_doc.costAmount === undefined) continue; if (expense_doc.type != 1) @@ -386,7 +462,9 @@ if (total_income != 0) { assets_to_income = (total_assets / total_income) > assets_to_income_fraction; - income_presents_to_total = (total_presents / total_income) > income_presents_to_total_fraction; + const presents_fraction = total_presents / total_income; + income_presents_to_total = presents_fraction > income_presents_to_total_fraction; + income_presents_to_total_50 = presents_fraction > income_presents_to_total_fraction_50; } const total_inc_and_assets = total_income + total_assets; if (total_inc_and_assets != 0) { @@ -394,11 +472,16 @@ liabilities_to_inc_and_assets = (total_liabilities / total_inc_and_assets) > liabilities_to_other_fraction; } jar_of_cash = total_cash > total_cash_threshold; + jar_of_pocket_cash = total_cash > total_cash_threshold_500k; garage_wo_car = has_garage && !has_vehicle; house_no_land = has_house && !has_land; + income_has_prizes = total_prizes > 0; + loan_shark = lent_assets > lent_assets_threshold; emit(doc._id, [doc.doc_uuid, assets_to_income, income_presents_to_total, expenses_to_inc_and_assets, liabilities_to_inc_and_assets, jar_of_cash, garage_wo_car, house_no_land, has_luxury_cars, - has_luxury_cars_v2, vehicle_purch_no_cost, estate_purch_no_cost, + has_luxury_cars_v2, vehicle_purch_no_cost, estate_purch_no_cost, estate_has_hidden_cost, + corprights_has_foreign, has_foreign_bank_acc, income_has_prizes, loan_shark, jar_of_pocket_cash, + income_presents_to_total_50, hidden_in_family, total_expenses, total_liabilities, total_cash, total_presents]); }