Merge pull request #5 from ymurong/dev

Final Product Merge

README.md

@@ -28,7 +28,7 @@
 
 [Prototype Phase Slides Link](https://docs.google.com/presentation/d/1pRIDMuRsB5ZIwAhkmasCoDJcMUB6IIQHSfBdcfv4s7E/edit#slide=id.g192ae5ba20d_2_0)
 
-
+[Pre-Final Phase Slides Link](https://docs.google.com/presentation/d/1wihsksSqfJSjYV3IzQX-mVXwEKKzURQ54BUnT3h4LDI/edit#slide=id.p)
 
 ## Explorative Analysis
 Any missing values, outliers, bias?

backend/requirements.txt

@@ -12,6 +12,7 @@ dill==0.3.6
 dnspython==2.2.1
 ecdsa==0.18.0
 email-validator==1.3.0
+fairlearn==0.8.0
 fastapi==0.88.0
 fastapi-pagination==0.11.1
 h11==0.14.0

backend/src/common/BasePipeline.py

@@ -29,6 +29,10 @@ def __init__(self, model_file_name, model_training=False, **kwargs):
     def explain(self, transaction_sample: np.ndarray, ) -> dict:
         pass
 
+    @abstractmethod
+    def get_influential_features(self, transaction_sample: np.ndarray, ) -> list:
+        pass
+
     def load_explainer(self, explainer_file_name):
         dir = os.path.dirname(os.path.abspath(__file__))
         fname = os.path.join(dir, explainer_file_name)

backend/src/common/RFClassifierPipeline.py

@@ -49,6 +49,28 @@ def get_explainable_group(feature_name):
                 explanability_scores[explainable_group] += score
             return explanability_scores
         raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method")
+
+    def get_influential_features(self, transaction_sample: np.ndarray, ) -> list:
+        def get_feature_name(feature_array_exp):
+            for element in feature_array_exp:
+                if element in INPUT_FEATURES:
+                    return element
+            return feature_array_exp[0]
+
+        if self.explainer is not None:
+            predict_fn_rf = lambda x: self.pipeline.predict_proba(x).astype(float)
+            exp = self.explainer.explain_instance(transaction_sample, predict_fn_rf, num_features=100)
+            influential_features = []
+            for feature in exp.as_list():
+                feature_name = get_feature_name(feature[0].split(" "))
+                score = feature[1]
+                if score >= 0:
+                    influential_features.append(feature_name)
+                    if len(influential_features) >= 5:
+                        break
+            return influential_features
+        raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method")
+
 
 
 if __name__ == '__main__':

backend/src/metadata/service.py

@@ -64,6 +64,14 @@ def get_explainability_scores(psp_reference: int, explainer_name: str) -> dict:
     explanability_scores = pipeline.explain(transaction_sample)
     return explanability_scores
 
+def get_explainability_features(psp_reference: int, explainer_name: str) -> list:
+    X_test, _ = load_test_data_by_psp_ref(psp_reference)
+    transaction_sample = X_test.values[0]
+    pipeline = explainer_factory(explainer_name=explainer_name)
+    explanability_features = pipeline.get_influential_features(transaction_sample)
+    return explanability_features
+
+
 
 def get_classifier_metrics(classifier_name: str, threshold: float = 0.5) -> dict:
     X_test, y_test, A_test = load_test_data(sensitive_features_included=True)

backend/src/transactions/controller.py

@@ -37,6 +37,14 @@ def explain_transaction(
     explainability_scores = metadata_service.get_explainability_scores(psp_reference=psp_reference, explainer_name=explainer_name)
     return explainability_scores
 
+@transaction_app.get("/{psp_reference}/explainability_features", response_model=list,
+                     description="Get most influential features")
+def get_influential_features(
+        psp_reference: int,
+        explainer_name: ExplainerEnum = Query(ExplainerEnum.random_forest_lime)
+):
+    explainability_features = metadata_service.get_explainability_features(psp_reference=psp_reference, explainer_name=explainer_name)
+    return explainability_features
 
 @transaction_app.post("", response_model=schemas.ReadTransaction)
 def create_transaction(transaction: schemas.CreateTransaction, db: Session = Depends(get_db)):

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html

@@ -1,6 +1,6 @@
 <div *ngIf = "dataLoaded | async">
   <div class="alert alert-warning" style="text-align: center;" role="alert" *ngIf="over3percent">
-    <strong>With this configuration, CHARGEBACK COSTS exceed 3% of the TOTAL REVENUE. Lower the decision boundary.</strong>
+    <strong>With this configuration, CHARGEBACK RATE exceed 3%. Lower the decision boundary.</strong>
   </div>
   <app-widgets-dropdown [originalMetrics]="original_metrics_widget" [currentMetrics]="current_metrics_widget"></app-widgets-dropdown>
   <c-row>
@@ -59,6 +59,15 @@
       </c-card>
     </c-col>
   </c-row>
+  <c-card>
+    <c-card-header>
+      <strong>Fairness Metrics </strong>
+      <span cTooltip="Change the decision boundary to see how the different metrics that depict the historical transactions would react">
+        <svg class="text-info" cIcon name="cilInfo"></svg>
+      </span>
+    </c-card-header>
+    <app-fairness [labels]="data_fn_rate.labels" [data_accuracy]="data_balanced_accuracy.datasets[0].data" [data_fn]="data_fn_rate.datasets[0].data"></app-fairness>
+  </c-card>
 </div>
 
 <div *ngIf = "loading | async">

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss

@@ -6,6 +6,21 @@
   }
 }
 
+.column {
+  float: left;
+  width: 47%;
+  margin-top: 1%;
+  margin-left: 1.5%;
+  margin-right: 1.5%;
+  margin-bottom: 1%;
+}
+/* Clear floats after the columns */
+.row:after {
+  content: "";
+  display: table;
+  clear: both;
+}
+
 
 span {
   cursor: pointer;

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts

@@ -1,7 +1,8 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, ChangeDetectorRef } from '@angular/core';
 import { UntypedFormControl, UntypedFormGroup } from '@angular/forms';
 import { MetricsService } from './metrics.service'
 import { DashboardChartsData, IChartProps } from './dashboard-charts-data';
+import { Chart } from 'chart.js';
 
 
 interface ModelRates {
@@ -11,6 +12,13 @@ interface ModelRates {
   auc: number;
   block_rate: number;
   fraud_rate:number;
+  fairness: FairnessMatrics;
+}
+
+interface FairnessMatrics {
+  balanced_accuracy: JSON;
+  false_positive_rate: JSON;
+  false_negative_rate: JSON;
 }
 
 interface ModelCosts{
@@ -32,8 +40,6 @@ interface Metrics {
   styleUrls: ['dashboard.component.scss']
 })
 export class DashboardComponent implements OnInit {
-  constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService) {
-  }
 
   public mainChart: IChartProps = {};
   public chart: Array<IChartProps> = [];
@@ -55,11 +61,44 @@ export class DashboardComponent implements OnInit {
   public limitChargebackRario: number = 0.03;
   public over3percent = false;
 
+  public charts: any;
+  public data_balanced_accuracy = {
+    labels: [""],
+    datasets: [
+      {
+        label: 'Balanced Accuracy',
+        backgroundColor: '#f87979',
+        data: [0]
+      }
+    ]
+  };
+
+  public values_fn_rate: number[] = [];
+  public data_fn_rate = {
+    labels: [""],
+    datasets: [
+      {
+        label: 'False Negative Rate',
+        backgroundColor: '#66ccff',
+        data: [0]
+      }
+    ]
+  };
+
+  constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService, private cdr: ChangeDetectorRef) {
+  }
+
+
   ngOnInit(): void {
     this.initCharts();
     this.updateMetrics();
   }
 
+  ngAfterViewInit() {
+    this.charts = (<any>document.getElementById('fn_rate'))._chart;
+    this.charts
+  }
+
   updateMetrics(): void {
     this.updateAccuracy();
     this.updateRevenue();
@@ -69,10 +108,18 @@ export class DashboardComponent implements OnInit {
     this.metricsService.getAccuracyMetrics(this.current_threshold).subscribe(
       (rates: any) => {
         this.model_rates = rates as ModelRates;
+        this.updateFairnessMetrics(this.model_rates.fairness);
       }
     )
   }
 
+  updateFairnessMetrics(fairness: FairnessMatrics): void {
+    this.data_balanced_accuracy.labels = Object.keys(fairness.balanced_accuracy);
+    this.data_balanced_accuracy.datasets[0].data = Object.values(fairness.balanced_accuracy);
+    this.data_fn_rate.labels = Object.keys(fairness.false_negative_rate);
+    this.data_fn_rate.datasets[0].data = Object.values(fairness.false_negative_rate);
+  }
+
   updateRevenue(): void {
     this.metricsService.getStoreCosts(this.current_threshold).subscribe(
       (rates: any) => {
@@ -81,8 +128,8 @@ export class DashboardComponent implements OnInit {
         this.checkInconsistencies();
         if (this.firstTime){
           this.createOriginalMetricWidget();
-          this.dataLoaded = Promise.resolve(true);
           this.loading = Promise.resolve(false);
+          this.dataLoaded = Promise.resolve(true);
           this.firstTime = false;
         }
       }
@@ -122,12 +169,11 @@ export class DashboardComponent implements OnInit {
   }
 
   checkInconsistencies() {
-    if (this.current_metrics_widget.chargeback_costs/this.current_metrics_widget.total_revenue > this.limitChargebackRario){
+    if (this.current_metrics_widget.fraud_rate > this.limitChargebackRario){
       this.over3percent = true
     } else {
       this.over3percent = false
     }
-
   }
 
   initCharts(): void {

dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts

@@ -25,6 +25,7 @@ import { ExplanationComponent } from './explainability/explanation/explanation.c
 import { DetailedChartsComponent } from './detailed-charts/detailed-charts.component';
 import { HttpClientModule } from '@angular/common/http';
 import { HistoricalTransactionsComponent } from './historical-transactions/historical-transactions.component';
+import { FairnessComponent } from './fairness/fairness.component';
 
 @NgModule({
   imports: [
@@ -51,7 +52,7 @@ import { HistoricalTransactionsComponent } from './historical-transactions/histo
     NgxChartsModule,
     HttpClientModule
   ],
-  declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent]
+  declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent, FairnessComponent]
 })
 export class DashboardModule {
 }

dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html

@@ -1,16 +1,26 @@
 <div class="row">
     <div class="column">
-        <div class="col-lg-6">
+        <div class="col-lg-5">
             <c-chart type="radar" [data]="chartRadarData" [options]="chartOptions"></c-chart>
             <div style="text-align: center;">
                 <b>Transaction Risk Score:</b> {{ risk_score | number:'1.4-4' }}
             </div>
         </div>
     </div>
-    <div class="col-md-5 ms-auto inline" style="margin-top: -43%; margin-right: 3%;">
-        <p align="justify">
+    <div class="col-md-5 ms-auto inline" style="margin-top: -43%; margin-right: 4%;">
+        <div *ngIf="accepted">
+            <h6 align="justify" style="margin-top: 150px; line-height: 1.7;">There is no signifiant set of attributes whose value is out of the usual behaviour. Therefore this transaction has been labeled as accepted.
+            </h6>
+        </div>
+        <div *ngIf="!accepted">
+            <h6 style="margin-top: 43px;">Most influential predictors:</h6>
             <br>
-            Both the IP and the amount spent in the purchase are unlikely according to the historical patterns of this user. The payment has been blocked because it's likely to have suffered an <b>Account Takeover Attack</b>.  <br>
-        </p>
+            <!-- Both the IP and the amount spent in the purchase are unlikely according to the historical patterns of this user. The payment has been blocked because it's likely to have suffered an <b>Account Takeover Attack</b>.-->  
+            <ul>
+                <li *ngFor="let explanation of verbose_explanation" style="margin-bottom: 10px;">{{ explanation }} <br></li>
+            </ul>
+        </div>
+
+        <br>
     </div>
 </div>