diff --git a/README.md b/README.md index 6b76eb92..340c9049 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ [Prototype Phase Slides Link](https://docs.google.com/presentation/d/1pRIDMuRsB5ZIwAhkmasCoDJcMUB6IIQHSfBdcfv4s7E/edit#slide=id.g192ae5ba20d_2_0) - +[Pre-Final Phase Slides Link](https://docs.google.com/presentation/d/1wihsksSqfJSjYV3IzQX-mVXwEKKzURQ54BUnT3h4LDI/edit#slide=id.p) ## Explorative Analysis Any missing values, outliers, bias? diff --git a/backend/requirements.txt b/backend/requirements.txt index 04cd3bc1..fc4b464a 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -12,6 +12,7 @@ dill==0.3.6 dnspython==2.2.1 ecdsa==0.18.0 email-validator==1.3.0 +fairlearn==0.8.0 fastapi==0.88.0 fastapi-pagination==0.11.1 h11==0.14.0 diff --git a/backend/src/common/BasePipeline.py b/backend/src/common/BasePipeline.py index f202609a..04fb6c51 100644 --- a/backend/src/common/BasePipeline.py +++ b/backend/src/common/BasePipeline.py @@ -29,6 +29,10 @@ def __init__(self, model_file_name, model_training=False, **kwargs): def explain(self, transaction_sample: np.ndarray, ) -> dict: pass + @abstractmethod + def get_influential_features(self, transaction_sample: np.ndarray, ) -> list: + pass + def load_explainer(self, explainer_file_name): dir = os.path.dirname(os.path.abspath(__file__)) fname = os.path.join(dir, explainer_file_name) diff --git a/backend/src/common/RFClassifierPipeline.py b/backend/src/common/RFClassifierPipeline.py index 649a92ed..608ee1f7 100644 --- a/backend/src/common/RFClassifierPipeline.py +++ b/backend/src/common/RFClassifierPipeline.py @@ -49,6 +49,28 @@ def get_explainable_group(feature_name): explanability_scores[explainable_group] += score return explanability_scores raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method") + + def get_influential_features(self, transaction_sample: np.ndarray, ) -> list: + def get_feature_name(feature_array_exp): + for element in feature_array_exp: + if element in INPUT_FEATURES: + return element + return feature_array_exp[0] + + if self.explainer is not None: + predict_fn_rf = lambda x: self.pipeline.predict_proba(x).astype(float) + exp = self.explainer.explain_instance(transaction_sample, predict_fn_rf, num_features=100) + influential_features = [] + for feature in exp.as_list(): + feature_name = get_feature_name(feature[0].split(" ")) + score = feature[1] + if score >= 0: + influential_features.append(feature_name) + if len(influential_features) >= 5: + break + return influential_features + raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method") + if __name__ == '__main__': diff --git a/backend/src/metadata/service.py b/backend/src/metadata/service.py index 31e442d2..1094f66e 100644 --- a/backend/src/metadata/service.py +++ b/backend/src/metadata/service.py @@ -64,6 +64,14 @@ def get_explainability_scores(psp_reference: int, explainer_name: str) -> dict: explanability_scores = pipeline.explain(transaction_sample) return explanability_scores +def get_explainability_features(psp_reference: int, explainer_name: str) -> list: + X_test, _ = load_test_data_by_psp_ref(psp_reference) + transaction_sample = X_test.values[0] + pipeline = explainer_factory(explainer_name=explainer_name) + explanability_features = pipeline.get_influential_features(transaction_sample) + return explanability_features + + def get_classifier_metrics(classifier_name: str, threshold: float = 0.5) -> dict: X_test, y_test, A_test = load_test_data(sensitive_features_included=True) diff --git a/backend/src/transactions/controller.py b/backend/src/transactions/controller.py index ac2ef8e8..99eaf0de 100644 --- a/backend/src/transactions/controller.py +++ b/backend/src/transactions/controller.py @@ -37,6 +37,14 @@ def explain_transaction( explainability_scores = metadata_service.get_explainability_scores(psp_reference=psp_reference, explainer_name=explainer_name) return explainability_scores +@transaction_app.get("/{psp_reference}/explainability_features", response_model=list, + description="Get most influential features") +def get_influential_features( + psp_reference: int, + explainer_name: ExplainerEnum = Query(ExplainerEnum.random_forest_lime) +): + explainability_features = metadata_service.get_explainability_features(psp_reference=psp_reference, explainer_name=explainer_name) + return explainability_features @transaction_app.post("", response_model=schemas.ReadTransaction) def create_transaction(transaction: schemas.CreateTransaction, db: Session = Depends(get_db)): diff --git a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html index f643e9a8..8bd2858f 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html +++ b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html @@ -1,6 +1,6 @@
@@ -59,6 +59,15 @@ + + + Fairness Metrics + + + + + +
diff --git a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss index 55e521b1..746a8f61 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss +++ b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss @@ -6,6 +6,21 @@ } } +.column { + float: left; + width: 47%; + margin-top: 1%; + margin-left: 1.5%; + margin-right: 1.5%; + margin-bottom: 1%; +} +/* Clear floats after the columns */ +.row:after { + content: ""; + display: table; + clear: both; +} + span { cursor: pointer; diff --git a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts index 340b6c03..a6789734 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts +++ b/dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts @@ -1,7 +1,8 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnInit, ChangeDetectorRef } from '@angular/core'; import { UntypedFormControl, UntypedFormGroup } from '@angular/forms'; import { MetricsService } from './metrics.service' import { DashboardChartsData, IChartProps } from './dashboard-charts-data'; +import { Chart } from 'chart.js'; interface ModelRates { @@ -11,6 +12,13 @@ interface ModelRates { auc: number; block_rate: number; fraud_rate:number; + fairness: FairnessMatrics; +} + +interface FairnessMatrics { + balanced_accuracy: JSON; + false_positive_rate: JSON; + false_negative_rate: JSON; } interface ModelCosts{ @@ -32,8 +40,6 @@ interface Metrics { styleUrls: ['dashboard.component.scss'] }) export class DashboardComponent implements OnInit { - constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService) { - } public mainChart: IChartProps = {}; public chart: Array = []; @@ -55,11 +61,44 @@ export class DashboardComponent implements OnInit { public limitChargebackRario: number = 0.03; public over3percent = false; + public charts: any; + public data_balanced_accuracy = { + labels: [""], + datasets: [ + { + label: 'Balanced Accuracy', + backgroundColor: '#f87979', + data: [0] + } + ] + }; + + public values_fn_rate: number[] = []; + public data_fn_rate = { + labels: [""], + datasets: [ + { + label: 'False Negative Rate', + backgroundColor: '#66ccff', + data: [0] + } + ] + }; + + constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService, private cdr: ChangeDetectorRef) { + } + + ngOnInit(): void { this.initCharts(); this.updateMetrics(); } + ngAfterViewInit() { + this.charts = (document.getElementById('fn_rate'))._chart; + this.charts + } + updateMetrics(): void { this.updateAccuracy(); this.updateRevenue(); @@ -69,10 +108,18 @@ export class DashboardComponent implements OnInit { this.metricsService.getAccuracyMetrics(this.current_threshold).subscribe( (rates: any) => { this.model_rates = rates as ModelRates; + this.updateFairnessMetrics(this.model_rates.fairness); } ) } + updateFairnessMetrics(fairness: FairnessMatrics): void { + this.data_balanced_accuracy.labels = Object.keys(fairness.balanced_accuracy); + this.data_balanced_accuracy.datasets[0].data = Object.values(fairness.balanced_accuracy); + this.data_fn_rate.labels = Object.keys(fairness.false_negative_rate); + this.data_fn_rate.datasets[0].data = Object.values(fairness.false_negative_rate); + } + updateRevenue(): void { this.metricsService.getStoreCosts(this.current_threshold).subscribe( (rates: any) => { @@ -81,8 +128,8 @@ export class DashboardComponent implements OnInit { this.checkInconsistencies(); if (this.firstTime){ this.createOriginalMetricWidget(); - this.dataLoaded = Promise.resolve(true); this.loading = Promise.resolve(false); + this.dataLoaded = Promise.resolve(true); this.firstTime = false; } } @@ -122,12 +169,11 @@ export class DashboardComponent implements OnInit { } checkInconsistencies() { - if (this.current_metrics_widget.chargeback_costs/this.current_metrics_widget.total_revenue > this.limitChargebackRario){ + if (this.current_metrics_widget.fraud_rate > this.limitChargebackRario){ this.over3percent = true } else { this.over3percent = false } - } initCharts(): void { diff --git a/dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts b/dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts index 394c423b..74860511 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts +++ b/dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts @@ -25,6 +25,7 @@ import { ExplanationComponent } from './explainability/explanation/explanation.c import { DetailedChartsComponent } from './detailed-charts/detailed-charts.component'; import { HttpClientModule } from '@angular/common/http'; import { HistoricalTransactionsComponent } from './historical-transactions/historical-transactions.component'; +import { FairnessComponent } from './fairness/fairness.component'; @NgModule({ imports: [ @@ -51,7 +52,7 @@ import { HistoricalTransactionsComponent } from './historical-transactions/histo NgxChartsModule, HttpClientModule ], - declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent] + declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent, FairnessComponent] }) export class DashboardModule { } diff --git a/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html b/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html index 77658a3b..183d2a50 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html +++ b/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html @@ -1,16 +1,26 @@
-
+
Transaction Risk Score: {{ risk_score | number:'1.4-4' }}
-
-

+

+
+
There is no signifiant set of attributes whose value is out of the usual behaviour. Therefore this transaction has been labeled as accepted. +
+
+
+
Most influential predictors:

- Both the IP and the amount spent in the purchase are unlikely according to the historical patterns of this user. The payment has been blocked because it's likely to have suffered an Account Takeover Attack.
-

+ +
    +
  • {{ explanation }}
  • +
+
+ +
diff --git a/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.ts b/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.ts index 0bb0c0e8..b8d7fb6b 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.ts +++ b/dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.ts @@ -13,6 +13,10 @@ export class ExplanationComponent implements OnInit{ @Input() psp_reference: bigint = BigInt(0); @Input() risk_score: number = 0; + public accepted :boolean = this.risk_score < 0.5; + + public verbose_explanation: string[] = []; + public general_explanations: string[] = []; chartRadarData = { labels: ['General Evidences Score', 'IP Score', 'Card Behaviour Score', 'Amount Spent Score', 'E-Mail Score'], @@ -49,7 +53,9 @@ export class ExplanationComponent implements OnInit{ }; ngOnInit(): void { - this.createRadarChart() + this.createRadarChart(); + this.getMostInfluentialFeatures(); + this.accepted = this.risk_score < 0.5; } createRadarChart(): void { @@ -60,10 +66,78 @@ export class ExplanationComponent implements OnInit{ ) } + getMostInfluentialFeatures(): void { + this.explanationService.getInfluentialFeatures(Number(this.psp_reference)).subscribe( + (influential_features: any[]) => { + this.createReadableExplanation(influential_features) + } + ) + } + fillExplanationScores(explanationScores: any) { const data = [explanationScores["general_evidences"], explanationScores["ip_risk"], explanationScores["risk_card_behaviour"], explanationScores["risk_card_amount"], explanationScores["email_risk"]] this.chartRadarData.datasets[0].data = data; } + createReadableExplanation(influential_features: string[]){ + this.verbose_explanation = []; + for (let sentence of influential_features){ + this.createOneSentence(sentence); + } + if (this.general_explanations.length > 0){ + let features = "" + for (let explanation of this.general_explanations){ + features += explanation; + features += " and " + } + let last_sentence = "General evidences such as (" + features.slice(0, -5) + ") have been decisive to block this transaction"; + this.verbose_explanation.push(last_sentence); + } + } + + createOneSentence(sentence: string): void { + let explanation: string = ""; + if (this.risk_score > 0.5) { + const token0 = sentence.split("_")[0] + const N = sentence.split("_").slice(-2)[0].slice(0, -3).toString(); + const window = sentence.split("_").slice(-1).toString() + if (window == "window") { + if (token0 == "card"){ + const type = sentence.split("_")[1] + if (type == "avg"){ + explanation = "The amount spent with this CARD during the last " + N + " days."; + }else if (type == "nb"){ + explanation = "The number of transactions made with this CARD during the last "+ N +" days it's been higher than usual."; + } + } else if (token0 == "email"){ + const type = sentence.split("_")[2] + if (type == "risk"){ + explanation = "There have been transactions made with this EMAIL during the last "+ N +" days that have been fraudulent."; + }else if (type == "nb"){ + explanation = "The number of transactions made with this EMAIL during the last "+ N +" days it's been higher than usual."; + } + } else if (token0 == "ip"){ + const type = sentence.split("_")[2] + if (type == "risk"){ + explanation = "There have been transactions made with this IP during the last "+ N +" days that have been fraudulent."; + }else if (type == "nb"){ + explanation = "The number of transactions made with this IP during the last "+ N +" days it's been higher than usual."; + } + } + } else if (sentence == "diff_tx_time_in_hours" || sentence == "is_night" || sentence == "is_weekend"){ + explanation = "The time in the day this transaction has been made differs from what is usual."; + } else if (sentence == "same_country" || sentence == "is_diff_previous_ip_country"){ + explanation = "This transaction has been made in another country, which is unusual for this user."; + } else if (sentence == "is_credit"){ + this.general_explanations.push("having used a credit card") + } else if (sentence == "shopper_interaction_POS"){ + this.general_explanations.push("being an online transaction") + } + } + if (explanation != ""){ + this.verbose_explanation.push(explanation); + } + } + } diff --git a/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.html b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.html new file mode 100644 index 00000000..a451db9b --- /dev/null +++ b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.html @@ -0,0 +1,10 @@ +
+
+
+ +
+
+ +
+
+
\ No newline at end of file diff --git a/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.scss b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.scss new file mode 100644 index 00000000..f9da28fb --- /dev/null +++ b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.scss @@ -0,0 +1,14 @@ +.column { + float: left; + width: 47%; + margin-top: 1%; + margin-left: 1.5%; + margin-right: 1.5%; + margin-bottom: 1%; + } + /* Clear floats after the columns */ + .row:after { + content: ""; + display: table; + clear: both; + } \ No newline at end of file diff --git a/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.spec.ts b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.spec.ts new file mode 100644 index 00000000..e412efed --- /dev/null +++ b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.spec.ts @@ -0,0 +1,23 @@ +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { FairnessComponent } from './fairness.component'; + +describe('FairnessComponent', () => { + let component: FairnessComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + await TestBed.configureTestingModule({ + declarations: [ FairnessComponent ] + }) + .compileComponents(); + + fixture = TestBed.createComponent(FairnessComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.ts b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.ts new file mode 100644 index 00000000..787e839b --- /dev/null +++ b/dashboard-front/coreui/src/app/views/dashboard/fairness/fairness.component.ts @@ -0,0 +1,70 @@ +import { Component, Input, OnInit, OnChanges, SimpleChanges } from '@angular/core'; +import { Router } from '@angular/router'; + + +@Component({ + selector: 'app-fairness', + templateUrl: './fairness.component.html', + styleUrls: ['./fairness.component.scss'] +}) +export class FairnessComponent implements OnInit, OnChanges { + + @Input() labels: string[] = [""]; + @Input() data_accuracy: number[] = [0]; + @Input() data_fn: number[] = [0] + + public chartsLoaded: Promise = Promise.resolve(false); + + public data_fn_rate = { + labels: this.labels, + datasets: [ + { + label: 'False Negative Rate (%)', + backgroundColor: '#627596', + data: this.data_fn + } + ] + }; + + public data_balanced_accuracy = { + labels: this.labels, + datasets: [ + { + label: 'Balanced Accuracy', + backgroundColor: '#627596', + data: this.data_accuracy + } + ] + }; + + public constructor(public router:Router){} + + ngOnInit(): void { + this.data_balanced_accuracy.labels = this.labels; + this.data_fn_rate.labels = this.labels; + this.data_balanced_accuracy.datasets[0].data = this.data_accuracy; + this.data_fn_rate.datasets[0].data = this.data_fn; + } + + ngOnChanges(changes: SimpleChanges): void { + this.ngOnInit(); + this.chartsLoaded = Promise.resolve(true); + //this.reloadComponent(true); + } + + reloadComponent(self:boolean,urlToNavigateTo ?:string){ + //skipLocationChange:true means dont update the url to / when navigating + console.log("Current route I am on:",this.router.url); + const url=self ? this.router.url :urlToNavigateTo; + this.router.navigateByUrl('/',{skipLocationChange:true}).then(()=>{ + this.router.navigate([`/${url}`]).then(()=>{ + console.log(`After navigation I am on:${this.router.url}`) + }) + }) + } + + + + + +} diff --git a/dashboard-front/coreui/src/app/views/dashboard/historical-transactions/historical-transactions.component.html b/dashboard-front/coreui/src/app/views/dashboard/historical-transactions/historical-transactions.component.html index eabbae4f..71f9c4a5 100644 --- a/dashboard-front/coreui/src/app/views/dashboard/historical-transactions/historical-transactions.component.html +++ b/dashboard-front/coreui/src/app/views/dashboard/historical-transactions/historical-transactions.component.html @@ -121,7 +121,7 @@
Historical Transactions