Vision Data Explorer Flyout tests

libs/e2e/src/lib/describer/modelAssessment/Constants.ts

@@ -180,6 +180,9 @@ export enum Locators {
   AggregateBalanceMeasuresTable = "#aggregateBalanceMeasures .ms-DetailsList",
   AggregateBalanceMeasuresTableColumns = "#aggregateBalanceMeasures .ms-DetailsList-headerWrapper div[aria-label]",
   AggregateBalanceMeasuresTableRows = "#aggregateBalanceMeasures .ms-DetailsRow",
+  VisionDataAnalysisPivot = "#visionDataAnalysisPivot",
+  VisionDataAnalysisTab = "#visionDataAnalysisPivot .ms-Pivot-link",
+  VisionDataBalancePivotItem = "#visionDataAnalysisPivot button[name='Data balance']",
   ForecastingDashboard = "#ModelAssessmentDashboard #ForecastingDashboard",
   ForecastingTimeSeriesDropdown = "#ForecastingDashboard #ForecastingTimeSeriesDropdown",
   ForecastingTimeSeriesDropdownOptions = "#ForecastingDashboard button[role='option']",
@@ -197,6 +200,16 @@ export enum Locators {
   ForecastingScenarioChartCurves = "#ForecastingDashboard .highcharts-series-group .highcharts-series",
   ForecastingScenarioChartLegendItems = "#ForecastingDashboard .highcharts-a11y-proxy-button",
   VisionDataExplorer = "#VisionDataExplorer",
+  VisionDataExplorerImageExplorerViewSuccessImage = "#VisionDataExplorer #visionImage_0",
+  VisionDataExplorerFlyoutPredictionTitle = "#flyoutPredictionTitle",
+  VisionDataExplorerFlyoutPredictionLabel = "#flyoutPredLabel",
+  VisionDataExplorerFlyoutGroundTruthLabel = "#flyoutGtLabel",
+  VisionDataExplorerFlyoutImage = "#flyoutImage_0",
+  VisionDataExplorerFlyoutExplanationImage = "#flyoutExplanationImage_0",
+  VisionDataExplorerFlyoutMetadata = "#flyoutMetadata",
+  VisionDataExplorerFlyoutCloseButton = "button[aria-label='Close']",
+  VisionDataExplorerFlyoutCanvasToolsImageContainer = "#canvasToolsDiv #selectionDiv #editorDiv",
+  VisionDataExplorerFlyoutObjectSelection = "#Object Selection",
   VisionDataExplorerCohortDropDown = "#VisionDataExplorer #dataExplorerCohortDropdown",
   VisionDataExplorerSearchBox = "#VisionDataExplorer #dataExplorerSearchBox",
   VisionDataExplorerThumbnailSize = "#VisionDataExplorer #dataExplorerThumbnailSize",

libs/e2e/src/lib/describer/modelAssessment/visionDataExplorer/describeVisionDataExplorer.ts

@@ -8,6 +8,7 @@ import { IModelAssessmentData } from "../IModelAssessmentData";
 
 import { ensureAllVisionDataExplorerBasicElementsArePresent } from "./ensureAllVisionDataExplorerBasicElementsArePresent";
 import { ensureAllVisionDataExplorerClassViewElementsAfterSelectionArePresent } from "./ensureAllVisionDataExplorerClassViewElementsAfterSelectionArePresent";
+import { ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent } from "./ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent";
 import { ensureAllVisionDataExplorerImageExplorerViewElementsAfterSelectionArePresent } from "./ensureAllVisionDataExplorerImageExplorerViewElementsAfterSelectionArePresent";
 import { ensureAllVisionDataExplorerTableViewElementsAfterSelectionArePresent } from "./ensureAllVisionDataExplorerTableViewElementsAfterSelectionArePresent";
 import {
@@ -35,6 +36,11 @@ export function describeVisionDataExplorer(
     }
 
     if (datasetShape.visionDataExplorerData?.hasVisionDataExplorerComponent) {
+      it("should Flyout view components when selected", () => {
+        ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent(
+          datasetShape
+        );
+      });
       it("should have basic components in the initial state", () => {
         ensureAllVisionDataExplorerBasicElementsArePresent();
       });

libs/e2e/src/lib/describer/modelAssessment/visionDataExplorer/ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent.ts

@@ -0,0 +1,74 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { expect } from "chai";
+
+import { Locators } from "../Constants";
+import { IModelAssessmentData } from "../IModelAssessmentData";
+
+import "cypress-wait-until";
+
+
+export function ensureAllVisionDataExplorerFlyoutElementsAfterSelectionArePresent(
+  datasetShape: IModelAssessmentData
+): void {
+  cy.get(Locators.VisionDataExplorerImageExplorerViewButton).click();
+
+  cy.waitUntil(() => {
+    return Cypress.$(Locators.VisionDataExplorerImageExplorerViewSuccessImage).length > 0
+    && (Cypress.$(Locators.VisionDataExplorerImageExplorerViewSuccessImage)[0] as HTMLImageElement).naturalWidth > 0
+    && (Cypress.$(Locators.VisionDataExplorerImageExplorerViewSuccessImage)[0] as HTMLImageElement).naturalHeight > 0;
+  }).get(Locators.VisionDataExplorerImageExplorerViewSuccessImage).click();
+
+  // cy.get(Locators.VisionDataExplorerImageExplorerViewSuccessImage, { timeout: 30000 })
+  // .should("be.visible")
+  // .and(($image) => {
+  //   cy.wait(10000);
+  //   // verifies the image is loaded
+  //   expect(($image[0] as HTMLImageElement).naturalWidth).to.be.greaterThan(0);
+  //   expect(($image[0] as HTMLImageElement).naturalHeight).to.be.greaterThan(
+  //     0
+  //   );
+  // });
+
+  // cy.get(Locators.VisionDataExplorerImageExplorerViewSuccessImage).click();
+
+  cy.get(Locators.VisionDataExplorerFlyoutPredictionTitle).should("exist");
+
+  cy.get(Locators.VisionDataExplorerFlyoutPredictionLabel)
+    .invoke("text")
+    .then((predLabel) => {
+      cy.get(Locators.VisionDataExplorerFlyoutGroundTruthLabel)
+        .invoke("text")
+        .should((gtLabel) => {
+          expect(predLabel.split(": ")[1]).to.equal(gtLabel.split(": ")[1]);
+        });
+    });
+
+    cy.waitUntil(() => {
+      cy.reload()
+      return Cypress.$(Locators.VisionDataExplorerFlyoutImage).length > 0
+      && (Cypress.$(Locators.VisionDataExplorerFlyoutImage)[0] as HTMLImageElement).naturalWidth > 0
+      && (Cypress.$(Locators.VisionDataExplorerFlyoutImage)[0] as HTMLImageElement).naturalHeight > 0;
+    });
+
+  // cy.get(Locators.VisionDataExplorerFlyoutImage, { timeout: 10000 })
+  //   .should("be.visible")
+  //   .and(($image) => {
+  //     cy.wait(10000);
+  //     // verifies the image is loaded
+  //     expect(($image[0] as HTMLImageElement).naturalWidth).to.be.greaterThan(0);
+  //     expect(($image[0] as HTMLImageElement).naturalHeight).to.be.greaterThan(
+  //       0
+  //     );
+  //   });
+
+  if (datasetShape.isObjectDetection) {
+    cy.get(Locators.VisionDataExplorerFlyoutObjectSelection).should("exist");
+  } else {
+    cy.get(Locators.VisionDataExplorerFlyoutExplanationImage).should("exist");
+  }
+  cy.get(Locators.VisionDataExplorerFlyoutMetadata).should("exist");
+
+  cy.get(Locators.VisionDataExplorerFlyoutCloseButton).click();
+}

libs/e2e/tsconfig.e2e.json

@@ -4,7 +4,7 @@
     "sourceMap": false,
     "outDir": "../../dist/out-tsc",
     "allowJs": true,
-    "types": ["cypress", "node"],
+    "types": ["cypress", "cypress-wait-until", "node"],
     "isolatedModules": false
   },
   "include": ["src/**/*.ts", "src/**/*.js"]

libs/e2e/tsconfig.lib.json

@@ -2,7 +2,7 @@
   "extends": "./tsconfig.json",
   "compilerOptions": {
     "outDir": "../../dist/out-tsc",
-    "types": ["cypress", "node"]
+    "types": ["cypress", "cypress-wait-until", "node"]
   },
   "files": [
     "../../node_modules/@nrwl/react/typings/cssmodule.d.ts",

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/Flyout.tsx

@@ -149,7 +149,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                           }
                         />
                       </Stack.Item>
-                      <Stack.Item id="predictionTitle">
+                      <Stack.Item id="flyoutPredictionTitle">
                         {predictedY !== trueY ? (
                           <Text
                             variant="large"
@@ -179,13 +179,13 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                         {item?.index}
                       </Text>
                     </Stack.Item>
-                    <Stack.Item id="predLabel">
+                    <Stack.Item id="flyoutPredLabel">
                       <Text variant="large">
                         {localization.InterpretVision.Dashboard.predictedY}
                         {predictedY}
                       </Text>
                     </Stack.Item>
-                    <Stack.Item id="gtLabel">
+                    <Stack.Item id="flyoutGtLabel">
                       <Text variant="large">
                         {localization.InterpretVision.Dashboard.trueY}
                         {trueY}
@@ -218,6 +218,7 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
               {!this.props.loadingExplanation[0][index] ? (
                 <Stack.Item>
                   <Image
+                    id={`flyoutExplanationImage_${item?.index}`}
                     src={`data:image/jpg;base64,${this.props.explanations
                       .get(0)
                       ?.get(index)}`}
@@ -245,7 +246,10 @@ export class Flyout extends React.Component<IFlyoutProps, IFlyoutState> {
                   {localization.InterpretVision.Dashboard.panelInformation}
                 </Text>
               </Stack.Item>
-              <Stack.Item className={classNames.featureListContainer}>
+              <Stack.Item
+                id="flyoutMetadata"
+                className={classNames.featureListContainer}
+              >
                 <List
                   items={this.state.metadata}
                   onRenderCell={this.onRenderCell}

libs/interpret-vision/src/lib/VisionExplanationDashboard/Controls/FlyoutObjectDetection.tsx

@@ -128,7 +128,7 @@ export class FlyoutObjectDetection extends React.Component<
                             }
                           />
                         </FluentUI.Stack.Item>
-                        <FluentUI.Stack.Item>
+                        <FluentUI.Stack.Item id="flyoutPredictionTitle">
                           {predictedY !== trueY ? (
                             <FluentUI.Text
                               variant="large"
@@ -158,13 +158,13 @@ export class FlyoutObjectDetection extends React.Component<
                           {item?.index}
                         </FluentUI.Text>
                       </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item id="flyoutPredLabel">
                         <FluentUI.Text variant="large">
                           {localization.InterpretVision.Dashboard.predictedY}
                           {predictedY}
                         </FluentUI.Text>
                       </FluentUI.Stack.Item>
-                      <FluentUI.Stack.Item>
+                      <FluentUI.Stack.Item id="flyoutGtLabel">
                         <FluentUI.Text variant="large">
                           {localization.InterpretVision.Dashboard.trueY}
                           {trueY}
@@ -194,6 +194,7 @@ export class FlyoutObjectDetection extends React.Component<
                   </FluentUI.Text>
                 </FluentUI.Stack.Item>
                 <FluentUI.Stack.Item
+                  id="flyoutMetadata"
                   className={classNames.featureListContainer}
                 >
                   <FluentUI.List

notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-image-classification-model-debugging.ipynb

@@ -187,7 +187,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "rai_insights = RAIVisionInsights(model, test_data.sample(10, random_state=42),\n",
+    "rai_insights = RAIVisionInsights(model, test_data.sample(3, random_state=42),\n",
     "                                 \"label\",\n",
     "                                 task_type=ModelTask.IMAGE_CLASSIFICATION,\n",
     "                                 classes=class_names)\n",
@@ -263,7 +263,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.8.17"
   },
   "orig_nbformat": 4,
   "vscode": {

notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-multilabel-image-classification-model-debugging.ipynb

@@ -322,7 +322,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "rai_insights = RAIVisionInsights(model, test_data.sample(10, random_state=42),\n",
+    "rai_insights = RAIVisionInsights(model, test_data.sample(2, random_state=42),\n",
     "                                 target_columns,\n",
     "                                 ModelTask.MULTILABEL_IMAGE_CLASSIFICATION)\n",
     "\n",
@@ -397,7 +397,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.16"
+   "version": "3.8.17"
   },
   "orig_nbformat": 4,
   "vscode": {

notebooks/responsibleaidashboard/vision/responsibleaidashboard-fridge-object-detection-model-debugging.ipynb

@@ -256,10 +256,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "rai_insights = RAIVisionInsights(model, test_data.sample(5, random_state=42),\n",
+    "rai_insights = RAIVisionInsights(model, test_data.head(2),\n",
     "                                 \"label\",\n",
     "                                 task_type=ModelTask.OBJECT_DETECTION,\n",
-    "                                 classes=class_names)\n",
+    "                                 classes=class_names,\n",
+    "                                 max_evals=1,\n",
+    "                                 num_masks=1,\n",
+    "                                 mask_res=1)\n",
     "\n",
     "rai_insights"
    ]

package.json

@@ -44,6 +44,7 @@
   "dependencies": {
     "@fluentui/react": "8.58.0",
     "canvas": "^2.11.2",
+    "chai": "^4.3.5",
     "core-js": "^3.6.5",
     "d3-array": "^2.8.0",
     "d3-color": "^2.0.0",
@@ -118,6 +119,7 @@
     "commander": "^7.1.0",
     "cross-env": "7.0.2",
     "cypress": "^4.12.1",
+    "cypress-wait-until": "^2.0.1",
     "dotenv": "8.2.0",
     "enzyme": "3.11.0",
     "enzyme-to-json": "3.5.0",