Merge pull request #4 from ymurong/UI-Changes

Final UI

backend/requirements.txt

@@ -12,6 +12,7 @@ dill==0.3.6
 dnspython==2.2.1
 ecdsa==0.18.0
 email-validator==1.3.0
+fairlearn==0.8.0
 fastapi==0.88.0
 fastapi-pagination==0.11.1
 h11==0.14.0

backend/src/common/BasePipeline.py

@@ -29,6 +29,10 @@ def __init__(self, model_file_name, model_training=False, **kwargs):
     def explain(self, transaction_sample: np.ndarray, ) -> dict:
         pass
 
+    @abstractmethod
+    def get_influential_features(self, transaction_sample: np.ndarray, ) -> list:
+        pass
+
     def load_explainer(self, explainer_file_name):
         dir = os.path.dirname(os.path.abspath(__file__))
         fname = os.path.join(dir, explainer_file_name)

backend/src/common/RFClassifierPipeline.py

@@ -49,6 +49,28 @@ def get_explainable_group(feature_name):
                 explanability_scores[explainable_group] += score
             return explanability_scores
         raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method")
+
+    def get_influential_features(self, transaction_sample: np.ndarray, ) -> list:
+        def get_feature_name(feature_array_exp):
+            for element in feature_array_exp:
+                if element in INPUT_FEATURES:
+                    return element
+            return feature_array_exp[0]
+
+        if self.explainer is not None:
+            predict_fn_rf = lambda x: self.pipeline.predict_proba(x).astype(float)
+            exp = self.explainer.explain_instance(transaction_sample, predict_fn_rf, num_features=100)
+            influential_features = []
+            for feature in exp.as_list():
+                feature_name = get_feature_name(feature[0].split(" "))
+                score = feature[1]
+                if score >= 0:
+                    influential_features.append(feature_name)
+                    if len(influential_features) >= 5:
+                        break
+            return influential_features
+        raise RuntimeError("explainer needs to be loaded first by invoking load_explainer method")
+
 
 
 if __name__ == '__main__':

backend/src/metadata/service.py

@@ -64,6 +64,14 @@ def get_explainability_scores(psp_reference: int, explainer_name: str) -> dict:
     explanability_scores = pipeline.explain(transaction_sample)
     return explanability_scores
 
+def get_explainability_features(psp_reference: int, explainer_name: str) -> list:
+    X_test, _ = load_test_data_by_psp_ref(psp_reference)
+    transaction_sample = X_test.values[0]
+    pipeline = explainer_factory(explainer_name=explainer_name)
+    explanability_features = pipeline.get_influential_features(transaction_sample)
+    return explanability_features
+
+
 
 def get_classifier_metrics(classifier_name: str, threshold: float = 0.5) -> dict:
     X_test, y_test, A_test = load_test_data(sensitive_features_included=True)

backend/src/transactions/controller.py

@@ -37,6 +37,14 @@ def explain_transaction(
     explainability_scores = metadata_service.get_explainability_scores(psp_reference=psp_reference, explainer_name=explainer_name)
     return explainability_scores
 
+@transaction_app.get("/{psp_reference}/explainability_features", response_model=list,
+                     description="Get most influential features")
+def get_influential_features(
+        psp_reference: int,
+        explainer_name: ExplainerEnum = Query(ExplainerEnum.random_forest_lime)
+):
+    explainability_features = metadata_service.get_explainability_features(psp_reference=psp_reference, explainer_name=explainer_name)
+    return explainability_features
 
 @transaction_app.post("", response_model=schemas.ReadTransaction)
 def create_transaction(transaction: schemas.CreateTransaction, db: Session = Depends(get_db)):

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.html

@@ -1,6 +1,6 @@
 <div *ngIf = "dataLoaded | async">
   <div class="alert alert-warning" style="text-align: center;" role="alert" *ngIf="over3percent">
-    <strong>With this configuration, CHARGEBACK COSTS exceed 3% of the TOTAL REVENUE. Lower the decision boundary.</strong>
+    <strong>With this configuration, CHARGEBACK RATE exceed 3%. Lower the decision boundary.</strong>
   </div>
   <app-widgets-dropdown [originalMetrics]="original_metrics_widget" [currentMetrics]="current_metrics_widget"></app-widgets-dropdown>
   <c-row>
@@ -59,6 +59,15 @@
       </c-card>
     </c-col>
   </c-row>
+  <c-card>
+    <c-card-header>
+      <strong>Fairness Metrics </strong>
+      <span cTooltip="Change the decision boundary to see how the different metrics that depict the historical transactions would react">
+        <svg class="text-info" cIcon name="cilInfo"></svg>
+      </span>
+    </c-card-header>
+    <app-fairness [labels]="data_fn_rate.labels" [data_accuracy]="data_balanced_accuracy.datasets[0].data" [data_fn]="data_fn_rate.datasets[0].data"></app-fairness>
+  </c-card>
 </div>
 
 <div *ngIf = "loading | async">

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.scss

@@ -6,6 +6,21 @@
   }
 }
 
+.column {
+  float: left;
+  width: 47%;
+  margin-top: 1%;
+  margin-left: 1.5%;
+  margin-right: 1.5%;
+  margin-bottom: 1%;
+}
+/* Clear floats after the columns */
+.row:after {
+  content: "";
+  display: table;
+  clear: both;
+}
+
 
 span {
   cursor: pointer;

dashboard-front/coreui/src/app/views/dashboard/dashboard.component.ts

@@ -1,7 +1,8 @@
-import { Component, OnInit } from '@angular/core';
+import { Component, OnInit, ChangeDetectorRef } from '@angular/core';
 import { UntypedFormControl, UntypedFormGroup } from '@angular/forms';
 import { MetricsService } from './metrics.service'
 import { DashboardChartsData, IChartProps } from './dashboard-charts-data';
+import { Chart } from 'chart.js';
 
 
 interface ModelRates {
@@ -11,6 +12,13 @@ interface ModelRates {
   auc: number;
   block_rate: number;
   fraud_rate:number;
+  fairness: FairnessMatrics;
+}
+
+interface FairnessMatrics {
+  balanced_accuracy: JSON;
+  false_positive_rate: JSON;
+  false_negative_rate: JSON;
 }
 
 interface ModelCosts{
@@ -32,8 +40,6 @@ interface Metrics {
   styleUrls: ['dashboard.component.scss']
 })
 export class DashboardComponent implements OnInit {
-  constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService) {
-  }
 
   public mainChart: IChartProps = {};
   public chart: Array<IChartProps> = [];
@@ -55,11 +61,44 @@ export class DashboardComponent implements OnInit {
   public limitChargebackRario: number = 0.03;
   public over3percent = false;
 
+  public charts: any;
+  public data_balanced_accuracy = {
+    labels: [""],
+    datasets: [
+      {
+        label: 'Balanced Accuracy',
+        backgroundColor: '#f87979',
+        data: [0]
+      }
+    ]
+  };
+
+  public values_fn_rate: number[] = [];
+  public data_fn_rate = {
+    labels: [""],
+    datasets: [
+      {
+        label: 'False Negative Rate',
+        backgroundColor: '#66ccff',
+        data: [0]
+      }
+    ]
+  };
+
+  constructor(private chartsData: DashboardChartsData, private metricsService: MetricsService, private cdr: ChangeDetectorRef) {
+  }
+
+
   ngOnInit(): void {
     this.initCharts();
     this.updateMetrics();
   }
 
+  ngAfterViewInit() {
+    this.charts = (<any>document.getElementById('fn_rate'))._chart;
+    this.charts
+  }
+
   updateMetrics(): void {
     this.updateAccuracy();
     this.updateRevenue();
@@ -69,10 +108,18 @@ export class DashboardComponent implements OnInit {
     this.metricsService.getAccuracyMetrics(this.current_threshold).subscribe(
       (rates: any) => {
         this.model_rates = rates as ModelRates;
+        this.updateFairnessMetrics(this.model_rates.fairness);
       }
     )
   }
 
+  updateFairnessMetrics(fairness: FairnessMatrics): void {
+    this.data_balanced_accuracy.labels = Object.keys(fairness.balanced_accuracy);
+    this.data_balanced_accuracy.datasets[0].data = Object.values(fairness.balanced_accuracy);
+    this.data_fn_rate.labels = Object.keys(fairness.false_negative_rate);
+    this.data_fn_rate.datasets[0].data = Object.values(fairness.false_negative_rate);
+  }
+
   updateRevenue(): void {
     this.metricsService.getStoreCosts(this.current_threshold).subscribe(
       (rates: any) => {
@@ -81,8 +128,8 @@ export class DashboardComponent implements OnInit {
         this.checkInconsistencies();
         if (this.firstTime){
           this.createOriginalMetricWidget();
-          this.dataLoaded = Promise.resolve(true);
           this.loading = Promise.resolve(false);
+          this.dataLoaded = Promise.resolve(true);
           this.firstTime = false;
         }
       }
@@ -122,12 +169,11 @@ export class DashboardComponent implements OnInit {
   }
 
   checkInconsistencies() {
-    if (this.current_metrics_widget.chargeback_costs/this.current_metrics_widget.total_revenue > this.limitChargebackRario){
+    if (this.current_metrics_widget.fraud_rate > this.limitChargebackRario){
       this.over3percent = true
     } else {
       this.over3percent = false
     }
-
   }
 
   initCharts(): void {

dashboard-front/coreui/src/app/views/dashboard/dashboard.module.ts

@@ -25,6 +25,7 @@ import { ExplanationComponent } from './explainability/explanation/explanation.c
 import { DetailedChartsComponent } from './detailed-charts/detailed-charts.component';
 import { HttpClientModule } from '@angular/common/http';
 import { HistoricalTransactionsComponent } from './historical-transactions/historical-transactions.component';
+import { FairnessComponent } from './fairness/fairness.component';
 
 @NgModule({
   imports: [
@@ -51,7 +52,7 @@ import { HistoricalTransactionsComponent } from './historical-transactions/histo
     NgxChartsModule,
     HttpClientModule
   ],
-  declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent]
+  declarations: [DashboardComponent, ExplanationComponent, DetailedChartsComponent, HistoricalTransactionsComponent, FairnessComponent]
 })
 export class DashboardModule {
 }

dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.html

@@ -1,16 +1,26 @@
 <div class="row">
     <div class="column">
-        <div class="col-lg-6">
+        <div class="col-lg-5">
             <c-chart type="radar" [data]="chartRadarData" [options]="chartOptions"></c-chart>
             <div style="text-align: center;">
                 <b>Transaction Risk Score:</b> {{ risk_score | number:'1.4-4' }}
             </div>
         </div>
     </div>
-    <div class="col-md-5 ms-auto inline" style="margin-top: -43%; margin-right: 3%;">
-        <p align="justify">
+    <div class="col-md-5 ms-auto inline" style="margin-top: -43%; margin-right: 4%;">
+        <div *ngIf="accepted">
+            <h6 align="justify" style="margin-top: 150px; line-height: 1.7;">There is no signifiant set of attributes whose value is out of the usual behaviour. Therefore this transaction has been labeled as accepted.
+            </h6>
+        </div>
+        <div *ngIf="!accepted">
+            <h6 style="margin-top: 43px;">Most influential predictors:</h6>
             <br>
-            Both the IP and the amount spent in the purchase are unlikely according to the historical patterns of this user. The payment has been blocked because it's likely to have suffered an <b>Account Takeover Attack</b>.  <br>
-        </p>
+            <!-- Both the IP and the amount spent in the purchase are unlikely according to the historical patterns of this user. The payment has been blocked because it's likely to have suffered an <b>Account Takeover Attack</b>.-->  
+            <ul>
+                <li *ngFor="let explanation of verbose_explanation" style="margin-bottom: 10px;">{{ explanation }} <br></li>
+            </ul>
+        </div>
+
+        <br>
     </div>
 </div>

dashboard-front/coreui/src/app/views/dashboard/explainability/explanation/explanation.component.ts

@@ -13,6 +13,10 @@ export class ExplanationComponent implements OnInit{
 
   @Input() psp_reference: bigint = BigInt(0);
   @Input() risk_score: number = 0;
+  public accepted :boolean = this.risk_score < 0.5;
+
+  public verbose_explanation: string[] = [];
+  public general_explanations: string[] = [];
 
   chartRadarData = {
     labels: ['General Evidences Score', 'IP Score', 'Card Behaviour Score', 'Amount Spent Score', 'E-Mail Score'],
@@ -49,7 +53,9 @@ export class ExplanationComponent implements OnInit{
   };
 
   ngOnInit(): void {
-    this.createRadarChart()
+    this.createRadarChart();
+    this.getMostInfluentialFeatures();
+    this.accepted = this.risk_score < 0.5;
   }
 
   createRadarChart(): void {
@@ -60,10 +66,78 @@ export class ExplanationComponent implements OnInit{
     )
   }
 
+  getMostInfluentialFeatures(): void {
+    this.explanationService.getInfluentialFeatures(Number(this.psp_reference)).subscribe(
+      (influential_features: any[]) => {
+        this.createReadableExplanation(influential_features)
+      }
+    )
+  }
+
   fillExplanationScores(explanationScores: any) {
     const data = [explanationScores["general_evidences"], explanationScores["ip_risk"], explanationScores["risk_card_behaviour"], explanationScores["risk_card_amount"], explanationScores["email_risk"]]
     this.chartRadarData.datasets[0].data = data;
   }
 
+  createReadableExplanation(influential_features: string[]){
+    this.verbose_explanation = [];
+    for (let sentence of influential_features){
+      this.createOneSentence(sentence);
+    }
+    if (this.general_explanations.length > 0){
+      let features = ""
+      for (let explanation of this.general_explanations){
+        features += explanation;
+        features +=  " and "
+      }
+      let last_sentence = "General evidences such as (" + features.slice(0, -5) + ") have been decisive to block this transaction";
+      this.verbose_explanation.push(last_sentence);
+    }
+  }
+
+  createOneSentence(sentence: string): void {
+    let explanation: string = "";
+    if (this.risk_score > 0.5) {
+      const token0 = sentence.split("_")[0]
+      const N = sentence.split("_").slice(-2)[0].slice(0, -3).toString();    
+      const window = sentence.split("_").slice(-1).toString()
+      if (window == "window") {
+        if (token0 == "card"){
+          const type = sentence.split("_")[1]
+          if (type == "avg"){
+            explanation = "The amount spent with this CARD during the last " + N + " days.";
+          }else if (type == "nb"){
+            explanation = "The number of transactions made with this CARD during the last "+ N +" days it's been higher than usual.";
+          }
+        } else if (token0 == "email"){
+          const type = sentence.split("_")[2]
+          if (type == "risk"){
+            explanation = "There have been transactions made with this EMAIL during the last "+ N +" days that have been fraudulent.";
+          }else if (type == "nb"){
+            explanation = "The number of transactions made with this EMAIL during the last "+ N +" days it's been higher than usual.";
+          }
+        } else if (token0 == "ip"){
+          const type = sentence.split("_")[2]
+          if (type == "risk"){
+            explanation = "There have been transactions made with this IP during the last "+ N +" days that have been fraudulent.";
+          }else if (type == "nb"){
+            explanation = "The number of transactions made with this IP during the last "+ N +" days it's been higher than usual.";
+          }
+        }
+      } else if (sentence == "diff_tx_time_in_hours" || sentence == "is_night" || sentence == "is_weekend"){
+        explanation = "The time in the day this transaction has been made differs from what is usual.";
+      } else if (sentence == "same_country"  || sentence == "is_diff_previous_ip_country"){
+        explanation = "This transaction has been made in another country, which is unusual for this user.";
+      } else if (sentence == "is_credit"){
+        this.general_explanations.push("having used a credit card")
+      } else if (sentence == "shopper_interaction_POS"){
+        this.general_explanations.push("being an online transaction")
+      }
+    }
+    if (explanation != ""){
+      this.verbose_explanation.push(explanation);
+    } 
+  }
+
 
 }