diff --git a/extensions/tabs/NLUResults.html b/extensions/tabs/NLUResults.html
index bca640f..941723e 100644
--- a/extensions/tabs/NLUResults.html
+++ b/extensions/tabs/NLUResults.html
@@ -12,15 +12,15 @@
-
-
-
+
+
+
-
+
diff --git a/extensions/tabs/README.md b/extensions/tabs/README.md
new file mode 100644
index 0000000..e45af97
--- /dev/null
+++ b/extensions/tabs/README.md
@@ -0,0 +1,50 @@
+### Instructions
+
+1. In a console, run the command: `tsc -w`
+
+ This will run the TypeScript compiler and have it constantly monitor for any changes to the *.tsc* files.
+
+ Any changes made to those will automatically be propogated to the */dist* folder where the `.tsx` files are converted to `.js` and referenced by the `NLUResults.html`.
+
+2. In a console, run the command: `live-server .`
+
+ This will run a dev server from the current location, open a new tab in your browser, and allow you to browse the pages
+
+3. Navigate to `nluresults.html` to view the charts
+
+
+### How to debug in Chrome, real-time
+
+1. Follow the instructions above
+
+2. Create a `launch.json` file, per the instructions in the documentation.
+
+Mine looks like this:
+
+```json
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "type": "chrome",
+ "request": "launch",
+ "name": "Launch Chrome against localhost",
+ "url": "http://localhost:8080/",
+ "webRoot": "${workspaceFolder}"
+ }
+ ]
+}
+
+```
+
+3. Read the **Attach** section of the [Code Debugger for Chrome docs](https://marketplace.visualstudio.com/items?itemName=msjsdiag.debugger-for-chrome) and attach the debugger to Chrome.
+
+3. Press *Debug* or **F5** to begin debugging. A new Chrome window will open.
+
+4. Set a breakpoint on any of the `.ts` files you are working on
+----------------
+
+**Written by:** Dave Voyles, Sept-12-2019
diff --git a/extensions/tabs/dist/confusion.js.map b/extensions/tabs/dist/confusion.js.map
new file mode 100644
index 0000000..538e787
--- /dev/null
+++ b/extensions/tabs/dist/confusion.js.map
@@ -0,0 +1 @@
+{"version":3,"file":"confusion.js","sourceRoot":"","sources":["../src/confusion.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,qDAAqD;AACrD,yCAAyC;AAEzC,mDAAmD;AAEnD,+BAA+B,QAAQ,EAAE,OAAO;IAC9C,IAAI,gBAAgB,GAAG,CAAC,CAAC;IACzB,IAAI,QAAQ,EAAE,kBAAkB,EAAE,oBAAoB,CAAC;IACvD,IAAI,iBAAiB,EAAE,mBAAmB,CAAC;IAE3C,mDAAmD;IACnD,8DAA8D;IAC9D,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,SAAA,EAAE,CAAC,CAAC,IAAI,CAAC,UAAS,IAAI;QAC/C,QAAQ,GAAG,EAAE,CAAC,IAAI,EAAE;aACjB,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,UAAU,CAAA,CAAC,CAAC,CAAC;aACxC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,oBAAoB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC7B,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,MAAM,CAAA,CAAC,CAAC,CAAC;aACtD,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,kBAAkB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC3B,GAAG,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAA,CAAC,CAAC,CAAC;aACpD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjB,iBAAiB,GAAG,EAAE,CAAC,IAAI,EAAE;aACxB,GAAG,CAAC,UAAS,CAAC;YACX,IAAI,UAAU,GAAG,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,UAAU,CAAC;YACvD,IAAI,QAAQ,GAAK,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC;YAE5C,uDAAuD;YACvD,EAAE,CAAC,CAAC,UAAU,IAAI,SAAS,CAAC,CAAC,CAAC;gBAC1B,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;oBACvD,IAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEhC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC1B,CAAC;gBAAA,CAAC;YACN,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,MAAM,CAAC,CAAC,iEAAiE;YAC7E,CAAC;QACL,CAAC,CAAC;aACD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEnB,mBAAmB,GAAG,EAAE,CAAC,IAAI,EAAE;aAC1B,GAAG,CAAC,UAAS,CAAC;YACX,IAAI,QAAQ,GAAG,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC;YAE5C,EAAE,CAAC,CAAC,QAAQ,IAAI,SAAS,CAAC,CAAC,CAAC;gBACxB,uDAAuD;gBACvD,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;oBACnD,IAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;oBAEhC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC;gBAC9B,CAAC;gBAAA,CAAC;YACN,CAAC;YAAC,IAAI,CAAC,CAAC;gBACJ,MAAM,CAAC,CAAC,iEAAiE;YAC7E,CAAC;QACL,CAAC,CAAC;aACD,OAAO,CAAC,IAAI,CAAC,CAAC;QAEnB,GAAG,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC;YAC/C,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3B,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC7E,CAAC;QAED,IAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAM,KAAK,GAAI,IAAI,KAAK,EAAE,CAAC;QAC3B,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAE1C,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QACvC,SAAS,CAAC,gBAAgB,EAAI,kBAAkB,CAAC,CAAC;QAClD,SAAS,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC;QAEpD,SAAS,CAAC,eAAe,EAAI,iBAAiB,CAAC,CAAC;QAChD,SAAS,CAAC,iBAAiB,EAAE,mBAAmB,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;AACL,CAAC;AAED,mBAAmB,OAAO,EAAE,IAAI;IAC9B,IAAI,SAAS,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC;SACpB,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;SACxB,IAAI,CAAC,OAAO,CAAC,CAAC;IAEjB,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC;SACrB,IAAI,CAAC,IAAI,CAAC;SACV,KAAK,EAAE;SACP,MAAM,CAAC,KAAK,CAAC;SACb,IAAI,CAAC,UAAS,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC;AAED,uBAAuB,IAAI,EAAE,MAAM,EAAE,KAAK;4BAC/B,KAAK;QACZ,IAAI,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC1C,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC;aACxB,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QACvB,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;aACb,SAAS,CAAC,UAAU,CAAC;aACrB,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;aACxB,KAAK,EAAE;aACP,MAAM,CAAC,QAAQ,CAAC;aAChB,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;aACZ,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;aACjF,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aACzE,IAAI,CAAC,QAAQ,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC5D,IAAI,CAAC,MAAM,EAAE,UAAS,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;aAC1D,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC;aACpB,EAAE,CAAC,WAAW,EAAE,UAAS,CAAC,EAAE,CAAC;YAC5B,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACjD,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC;iBACb,MAAM,CAAC,QAAQ,CAAC;iBAChB,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;iBACtD,IAAI,CAAC,IAAI,EAAE,qBAAqB,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;iBAC9C,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBACjC,IAAI,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;iBAC/B,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC;iBACpB,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;iBACvB,EAAE,CAAC,OAAO,EAAE;gBACX,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;gBACnD,IAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACnC,GAAG,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;gBACvC,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;YACnD,CAAC,CAAC;iBACD,EAAE,CAAC,WAAW,EAAE;gBACf,OAAO,CAAC,UAAU,EAAE;qBACjB,QAAQ,CAAC,GAAG,CAAC;qBACb,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC;qBAClC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC;qBACtC,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;YAChD,CAAC,CAAC;iBACD,EAAE,CAAC,UAAU,EAAE;gBACd,OAAO,CAAC,UAAU,EAAE;qBACjB,QAAQ,CAAC,GAAG,CAAC;qBACb,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;gBACvB,EAAE,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACnD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACP,CAAC;IA9CD,GAAG,CAAC,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;gBAAvC,KAAK;KA8Cb;AACH,CAAC;AAED,kBAAkB,KAAK;IACrB,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAE9B,IAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAM,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,kBAAkB,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SACpD,IAAI,CAAC,SAAS,EAAE,OAAO,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,GAAG,KAAK,CAAC,WAAW,CAAC;SACrE,MAAM,CAAC,GAAG,CAAC;SACX,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAEtF,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC;SACf,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC;SACtB,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,UAAU,CAAC;SAC/B,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,WAAW,CAAC,CAAC;IAErC,IAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC;SACtB,IAAI,CAAC,WAAW,EAAE,YAAY,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IAExF,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACV,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC;SACvB,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC;SACnC,IAAI,CAAC,KAAK,CAAC,CAAC;IAEf,CAAC,CAAC,SAAS,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;SACtD,KAAK,EAAE;SACP,MAAM,CAAC,MAAM,CAAC;SACd,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC;SAC/B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;SACvB,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC7C,IAAI,CAAC,IAAI,EAAE,UAAS,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC7C,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC;SACpB,IAAI,CAAC,iBAAiB,EAAE,YAAY,CAAC;SACrC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC;SAC3B,IAAI,CAAC,kBAAkB,EAAE,OAAO,CAAC;SACjC,IAAI,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC;IAE/B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,eAAe,CAAC,CAAC;IACzB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;SAC5B,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC1B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,KAAK,GAAG,GAAG,CAAC;SAC5B,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,eAAe,CAAC,CAAC;IACzB,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC5B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC;SACvB,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;SAC5B,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;IAC/B,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;SACb,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC;SAC3B,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC;SACxB,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;AACjC,CAAC;AAED,2BAA2B,IAAI,EAAE,MAAM;IACrC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;IAC9B,IAAM,CAAC,GAAG;QACR,IAAI,EAAE;YACJ,IAAI,EAAE,KAAK;YACX,OAAO,EAAE,OAAO;YAChB,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE;gBAC/D;oBACE,IAAI,EAAE,KAAK;oBACX,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE;wBACN,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,GAAG,MAAM,CAAC,YAAY,EAAE,EAAE;wBAChF,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,UAAU,GAAG,MAAM,CAAC,SAAS,EAAE,EAAE;wBAC1E,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,EAAE;wBAClE,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,GAAG,MAAM,CAAC,WAAW,EAAE,EAAE;qBAC/E;iBACF;gBACD,EAAE,IAAI,EAAE,IAAI,EAAE;gBACd,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,aAAa,EAAE;gBAC1D,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,EAAE;gBAC9C,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,2BAA2B,EAAE;gBAClD;oBACE,IAAI,EAAE,KAAK;oBACX,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE;wBACN;4BACE,IAAI,EAAE,OAAO;4BACb,OAAO,EAAE,OAAO;4BAChB,MAAM,EAAE;gCACN;oCACE,IAAI,EAAE,OAAO;oCACb,MAAM,EAAE;wCACN;4CACE,IAAI,EAAE,IAAI;4CACV,MAAM,EAAE;gDACN,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,oBAAoB,EAAE;gDAC5D,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,EAAE;6CAC3D;yCACF;qCACF;iCACF;gCACD;oCACE,IAAI,EAAE,OAAO;oCACb,MAAM,EAAE;wCACN;4CACE,IAAI,EAAE,IAAI;4CACV,MAAM,EAAE;gDACN;oDACE,IAAI,EAAE,IAAI;oDACV,MAAM;wDACJ,IAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;wDAC3E,IAAM,MAAM,GAAG,KAAK,CAAC;wDACrB,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,iBAAiB,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oDACnF,CAAC;iDACF;gDACD;oDACE,IAAI,EAAE,IAAI;oDACV,MAAM;wDACJ,IAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;wDACzE,IAAM,MAAM,GAAG,IAAI,CAAC;wDACpB,MAAM,CAAC,aAAa,CAAC,SAAS,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;oDACjF,CAAC;iDACF;6CACF;yCACF;qCACF;iCACF;6BACF;yBACF;qBACF;iBACF;aACF;SACF;QACD,SAAS,EAAE;YACT,IAAI,EAAE,GAAG;YACT,MAAM,EAAE;gBACN,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,eAAe,EAAE;gBACtC,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,mBAAmB,EAAE;aAC3C;SACF;KACF,CAAC;IACF,MAAM,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;AAC3C,CAAC;AAED,uBAAuB,IAAI,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe;IACjE,IAAI,UAAU,GAAG,kBAAkB,CAAC,CAAC,wBAAwB;IAC7D,EAAE,CAAC,CACD,CAAC,eAAe,IAAI,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,eAAe,IAAI,CAAC,UAAU,KAAK,eAAe,CAAC,CAChH,CAAC,CAAC,CAAC;QAAC,UAAU,GAAG,kBAAkB,CAAC;IAAC,CAAC,CAAC,sBAAsB;IAC7D,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC;QACrC,EAAE,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAClC,IAAM,UAAU,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACnD,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,iCAAiC,GAAG,UAAU,GAAG,YAAY,GAAG,UAAU,GAAG,SAAS,CAAC,CAAC;QAC1H,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC;AACd,CAAC;AAED,qBAAqB,CAAC,EAAE,CAAC;IACvB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,CAAC,KAAK,CAAC;AACf,CAAC;AAED,+BAA+B,KAAK,EAAE,CAAC,EAAE,CAAC,EAAE,MAAM;IAChD,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAC9B,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QACrB,KAAK,cAAc;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC3B,KAAK,eAAe;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC5B,KAAK,cAAc;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;QAC3B,KAAK,eAAe;YAAE,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC;YACf,KAAK,CAAC;IAC9B,CAAC;IACD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC/D,CAAC;IAAC,IAAI,CAAC,CAAC;QACN,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,CAAC,CAAC,CAAC;AACX,CAAC;AAED,+BAA+B,KAAK,EAAE,CAAC,EAAE,CAAC;IACxC,IAAM,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;IAC9B,IAAM,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,KAAK,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,KAAK,eAAe,CAAC,CAAC,CAAC,CAAC;QAC5E,IAAM,QAAQ,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5B,CAAC;IAAC,IAAI,CAAC,CAAC;QACN,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC"}
\ No newline at end of file
diff --git a/extensions/tabs/dist/graph.js.map b/extensions/tabs/dist/graph.js.map
new file mode 100644
index 0000000..3d4ae2f
--- /dev/null
+++ b/extensions/tabs/dist/graph.js.map
@@ -0,0 +1 @@
+{"version":3,"file":"graph.js","sourceRoot":"","sources":["../src/graph.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,eAAe;AAEf,0CAA0C;AAC1C;IACE;QACE,IAAI,CAAC,MAAM,GAAa,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;QACpE,IAAI,CAAC,OAAO,GAAY,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;QACpE,IAAI,CAAC,UAAU,GAAS,GAAG,CAAA;QAC3B,IAAI,CAAC,WAAW,GAAQ,GAAG,CAAA;QAC3B,IAAI,CAAC,UAAU,GAAS,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAA;QAC9E,IAAI,CAAC,WAAW,GAAQ,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;QAC/E,IAAI,CAAC,KAAK,GAAc,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,CAAA;QAChF,IAAI,CAAC,MAAM,GAAa,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAA;QACjF,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAA;IAC3B,CAAC;IAED,sBAAM,GAAN;QACE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;aACpB,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;aACnB,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC5B,CAAC;IAED,sBAAM,GAAN;QACE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC;aACtC,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC,CAAA;IAChC,CAAC;IACH,YAAC;AAAD,CAAC,AAxBD,IAwBC"}
\ No newline at end of file
diff --git a/extensions/tabs/dist/init.js.map b/extensions/tabs/dist/init.js.map
new file mode 100644
index 0000000..90f4679
--- /dev/null
+++ b/extensions/tabs/dist/init.js.map
@@ -0,0 +1 @@
+{"version":3,"file":"init.js","sourceRoot":"","sources":["../src/init.ts"],"names":[],"mappings":"AAAA,uCAAuC;AAEvC,aAAa;AACb,8BAA8B;AAC9B,kCAAkC;AAClC,eAAe;AACf,4BAA4B;AAC5B,QAAQ;AACR,MAAM;AACN,KAAK;AAEL,oBAAoB;AACpB,wCAAwC;AACxC,qDAAqD;AACrD,6HAA6H;AAC7H,uJAAuJ;AACvJ,uBAAuB;AACvB,yBAAyB;AACzB,8CAA8C;AAC9C,gLAAgL;AAChL,sGAAsG;AACtG,4GAA4G;AAC5G,cAAc;AACd,aAAa;AAEb,2BAA2B;AAC3B,wEAAwE;AACxE,mBAAmB;AACnB,iDAAiD;AACjD,0FAA0F;AAC1F,4EAA4E;AAC5E,eAAe;AACf,YAAY;AAEZ,6BAA6B;AAC7B,0EAA0E;AAC1E,YAAY;AACZ,WAAW;AACX,SAAS;AACT,OAAO;AACP,KAAK;AAEL,kEAAkE;AAClE,IAAM,QAAQ,GAAG,eAAe,CAAC;AACjC,qBAAqB,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC,CAAC"}
\ No newline at end of file
diff --git a/extensions/tabs/dist/matrix.js.map b/extensions/tabs/dist/matrix.js.map
new file mode 100644
index 0000000..9049f34
--- /dev/null
+++ b/extensions/tabs/dist/matrix.js.map
@@ -0,0 +1 @@
+{"version":3,"file":"matrix.js","sourceRoot":"","sources":["../src/matrix.ts"],"names":[],"mappings":"AAAA,4DAA4D;AAC5D,kCAAkC;AAElC,0CAA0C;AAC1C;IACE;QACE,IAAI,CAAC,CAAC,GAAI,CAAC,CAAC;QACZ,IAAI,CAAC,CAAC,GAAI,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;QACZ,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IACd,CAAC;IAEM,4BAAW,GAAlB;QACE,IAAM,QAAQ,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAClF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACzC,CAAC;IAEM,sBAAK,GAAZ;QACE,IAAM,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC1D,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACnC,CAAC;IAEM,6BAAY,GAAnB;QACE,IAAM,SAAS,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC/C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IAC1C,CAAC;IAEM,0BAAS,GAAhB;QACE,IAAM,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,CAAA;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,CAAA;IACvC,CAAC;IACH,aAAC;AAAD,CAAC,AA7BD,IA6BC"}
\ No newline at end of file
diff --git a/extensions/tabs/metadata.json b/extensions/tabs/metadata.json
new file mode 100644
index 0000000..c8e069e
--- /dev/null
+++ b/extensions/tabs/metadata.json
@@ -0,0 +1,2606 @@
+[
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'start playing music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.92854017,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'play music')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "score": 0.907853544,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('PlayMusic', 'listen to hip hop')",
+ "group": "PlayMusic",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.539262056,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'skip this one')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "score": 0.449134439,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TruePositiveIntent('Skip', 'next please')",
+ "group": "Skip",
+ "resultKind": "TruePositive",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "score": 0.469399869,
+ "because": "Utterances have matching intent.",
+ "categories": [
+ "Intent",
+ "TruePositive"
+ ]
+ },
+ {
+ "testName": "FalseNegativeIntent('Skip', 'go forward')",
+ "group": "Skip",
+ "resultKind": "FalseNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "score": 0.463593155,
+ "because": "Actual intent is 'None', expected 'Skip'",
+ "categories": [
+ "Intent",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "score": 0.873349249,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "score": 0.42890653,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeIntent('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Intent",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "score": 0.5407783,
+ "because": "Both intents are 'None'.",
+ "categories": [
+ "Intent",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('start playing music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.92854017,
+ "entities": [],
+ "text": "start playing music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('play music')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "actualUtterance": {
+ "score": 0.907853544,
+ "entities": [],
+ "text": "play music",
+ "intent": "PlayMusic"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "FalseNegativeEntity('Genre', '\"hip hop\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalseNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "because": "Actual utterance does not have entity matching 'hip hop'.",
+ "categories": [
+ "Entity",
+ "FalseNegative"
+ ]
+ },
+ {
+ "testName": "FalsePositiveEntity('Genre', '\"hip\"', 'listen to hip hop')",
+ "group": "Genre",
+ "resultKind": "FalsePositive",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "listen to hip hop",
+ "intent": "PlayMusic",
+ "entities": [
+ {
+ "entityType": "Genre",
+ "matchText": "hip hop"
+ }
+ ]
+ },
+ "actualUtterance": {
+ "score": 0.539262056,
+ "entities": [
+ {
+ "score": 0.662749946,
+ "entityType": "Genre",
+ "matchText": "hip"
+ }
+ ],
+ "text": "listen to hip hop",
+ "intent": "PlayMusic"
+ },
+ "score": 0.662749946,
+ "because": "Expected utterance does not have entity matching 'hip'.",
+ "categories": [
+ "Entity",
+ "FalsePositive"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('skip this one')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "skip this one",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.449134439,
+ "entities": [],
+ "text": "skip this one",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('next please')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.469399869,
+ "entities": [],
+ "text": "next please",
+ "intent": "Skip"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('go forward')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "go forward",
+ "intent": "Skip"
+ },
+ "actualUtterance": {
+ "score": 0.463593155,
+ "entities": [],
+ "text": "go forward",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('is it cold out')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.873349249,
+ "entities": [],
+ "text": "is it cold out",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('how many days until Christmas')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.42890653,
+ "entities": [],
+ "text": "how many days until Christmas",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ },
+ {
+ "testName": "TrueNegativeEntity('what's the weather like')",
+ "resultKind": "TrueNegative",
+ "targetKind": "Entity",
+ "expectedUtterance": {
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "actualUtterance": {
+ "score": 0.5407783,
+ "entities": [],
+ "text": "what's the weather like",
+ "intent": "None"
+ },
+ "because": "Neither utterances have entities.",
+ "categories": [
+ "Entity",
+ "TrueNegative"
+ ]
+ }
+]
diff --git a/extensions/tabs/package.json b/extensions/tabs/package.json
index bf86cee..1b0cf5e 100644
--- a/extensions/tabs/package.json
+++ b/extensions/tabs/package.json
@@ -1,5 +1,9 @@
{
"dependencies": {
+ "@types/d3": "^5.7.2",
+ "awesome-typescript-loader": "^5.2.1",
+ "live-server": "^1.2.1",
+ "source-map-loader": "^0.2.4",
"vss-web-extension-sdk": "^5.141.0"
}
}
diff --git a/extensions/tabs/scripts/confusion.js b/extensions/tabs/scripts/confusion.js
deleted file mode 100644
index 84d5c10..0000000
--- a/extensions/tabs/scripts/confusion.js
+++ /dev/null
@@ -1,310 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-// d3 and json2html included in HTML script reference
-/* global d3, json2html, Matrix, Graph */
-
-// json2html uses curly brace syntax for templating
-/* eslint no-template-curly-in-string: "warn" */
-
-// eslint-disable-next-line no-unused-vars
-function renderConfusionMatrix (filename, headers) {
- let quadrantCountMax = 0
- let nestData, facetActualIntents, facetExpectedIntents
-
- d3.json(filename, { headers }).then(function (data) {
- nestData = d3.nest()
- .key(function (d) { return d.resultKind })
- .entries(data)
- facetExpectedIntents = d3.nest()
- .key(function (d) { return d.expectedUtterance.intent })
- .entries(data)
- facetActualIntents = d3.nest()
- .key(function (d) { return d.actualUtterance.intent })
- .entries(data)
- for (let key = 0; key < nestData.length; key++) {
- console.log(nestData[key])
- quadrantCountMax = Math.max(quadrantCountMax, nestData[key].values.length)
- }
-
- const matrix = new Matrix()
- const graph = new Graph()
- graph.quadrantCountMax = quadrantCountMax
-
- addGraph(graph)
- addDataPoints(nestData, matrix, graph)
- addFacets('Actual Intents', facetActualIntents)
- addFacets('Expected Intents', facetExpectedIntents)
- })
-}
-
-function addFacets (section, data) {
- var slicerDiv = d3.select('#facets').append('div')
-
- slicerDiv.append('div')
- .attr('class', 'heading')
- .text(section)
-
- slicerDiv.selectAll('p')
- .data(data)
- .enter()
- .append('div')
- .text(function (d) { return d.key })
-}
-
-function addDataPoints (data, matrix, graph) {
- for (let index = 0; index < data.length; index++) {
- var tooltip = d3.select('body').append('div')
- .attr('class', 'tooltip')
- .style('opacity', 0)
- d3.select('svg')
- .selectAll('elements')
- .data(data[index].values)
- .enter()
- .append('circle')
- .attr('r', 5)
- .attr('cx', function (d, i) { return getXConfusionQuadrant(graph, d, i, matrix) })
- .attr('cy', function (d, i) { return getYConfusionQuadrant(graph, d, i) })
- .attr('stroke', function (d, i) { return getDotColor(d, i) })
- .attr('fill', function (d, i) { return getDotColor(d, i) })
- .attr('opacity', 0.5)
- .on('mouseover', function (d, i) {
- d3.selectAll('.active').classed('active', false)
- d3.select('svg')
- .append('circle')
- .attr('cx', getXConfusionQuadrant(graph, d, i, matrix))
- .attr('cy', getYConfusionQuadrant(graph, d, i))
- .attr('stroke', getDotColor(d, i))
- .attr('fill', getDotColor(d, i))
- .attr('opacity', 0.5)
- .classed('active', true)
- .on('click', function () {
- d3.selectAll('.clicked').classed('clicked', false)
- var div = d3.select('#testData')
- div.html(getTestResultHtml(d, matrix))
- d3.selectAll('.active').classed('clicked', true)
- })
- .on('mouseover', function () {
- tooltip.transition()
- .duration(500)
- .style('opacity', 0.9)
- tooltip.html(d.score || 'not scored')
- .style('left', (d3.event.pageX) + 'px')
- .style('top', (d3.event.pageY - 28) + 'px')
- })
- .on('mouseout', function () {
- tooltip.transition()
- .duration(500)
- .style('opacity', 0)
- d3.selectAll('.active').classed('active', false)
- })
- })
- }
-}
-
-function addGraph (graph) {
- const yScale = graph.yScale()
-
- var yAxis = d3.axisLeft().scale(yScale)
-
- var svg = d3.select('#confusionMatrix').append('svg')
- .attr('viewBox', '0, 0,' + graph.outerWidth + ',' + graph.outerHeight)
- .append('g')
- .attr('transform', 'translate(' + graph.margin.left + ',' + graph.margin.top + ')')
-
- svg.append('rect')
- .attr('class', 'outer')
- .attr('width', graph.innerWidth)
- .attr('height', graph.innerHeight)
-
- var g = svg.append('g')
- .attr('transform', 'translate(' + graph.padding.left + ',' + graph.padding.top + ')')
-
- g.append('g')
- .attr('class', 'y axis')
- .attr('transform', 'translate(0,0)')
- .call(yAxis)
-
- g.selectAll('line.horizontalGrid').data(yScale.ticks(13))
- .enter()
- .append('line')
- .attr('class', 'horizontalGrid')
- .attr('x1', 0)
- .attr('x2', graph.width)
- .attr('y1', function (d) { return yScale(d) })
- .attr('y2', function (d) { return yScale(d) })
- .attr('fill', 'none')
- .attr('shape-rendering', 'crispEdges')
- .attr('stroke', 'steelblue')
- .attr('stroke-dasharray', '10,10')
- .attr('stroke-width', '1px')
-
- g.append('text')
- .attr('x', 10)
- .attr('y', 15)
- .text('True Positive')
- g.append('text')
- .attr('x', graph.width - 100)
- .attr('y', 15)
- .text('False Positive')
- g.append('text')
- .attr('x', 10)
- .attr('y', graph.height - 5)
- .text('False Negative')
- g.append('text')
- .attr('x', graph.width - 100)
- .attr('y', graph.height - 5)
- .text('True Negative')
- g.append('line')
- .attr('x1', 0)
- .attr('y1', graph.height / 2)
- .attr('x2', graph.width)
- .attr('y2', graph.height / 2)
- .attr('stroke', 'steelblue')
- g.append('line')
- .attr('x1', graph.width / 2)
- .attr('y1', 0)
- .attr('x2', graph.width / 2)
- .attr('y2', graph.height)
- .attr('stroke', 'steelblue')
-}
-
-function getTestResultHtml (test, matrix) {
- console.log('getTestResults')
- var t = {
- test: {
- '<>': 'div',
- class: 'col-8',
- html: [
- { '<>': 'div', class: 'heading', html: 'Model Statistics' },
- {
- '<>': 'div',
- class: 'row',
- html: [
- { '<>': 'div', class: 'col-3', html: 'Precision: ' + matrix.getPrecision() },
- { '<>': 'div', class: 'col-3', html: 'Recall: ' + matrix.getRecall() },
- { '<>': 'div', class: 'col-3', html: 'F1: ' + matrix.getF1() },
- { '<>': 'div', class: 'col-3', html: 'Accuracy: ' + matrix.getAccuracy() }
- ]
- },
- { '<>': 'br' },
- { '<>': 'div', class: 'heading', html: '${testName}' },
- { '<>': 'p', html: 'Target: ${targetKind}' },
- { '<>': 'p', html: '${resultKind}: ${because}' },
- {
- '<>': 'div',
- class: 'row',
- html: [
- {
- '<>': 'table',
- class: 'table',
- html: [
- {
- '<>': 'thead',
- html: [
- {
- '<>': 'tr',
- html: [
- { '<>': 'th', scope: 'col', html: 'Expected Utterance' },
- { '<>': 'th', scope: 'col', html: 'Actual Utterance' }
- ]
- }
- ]
- },
- {
- '<>': 'tbody',
- html: [
- {
- '<>': 'tr',
- html: [
- {
- '<>': 'td',
- html: function () {
- var innerHtml = json2html.transform(this.expectedUtterance, t.utterance)
- var actual = false
- return styleEntities(innerHtml, this.expectedUtterance, this.resultKind, actual)
- }
- },
- {
- '<>': 'td',
- html: function () {
- var innerHtml = json2html.transform(this.actualUtterance, t.utterance)
- var actual = true
- return styleEntities(innerHtml, this.actualUtterance, this.resultKind, actual)
- }
- }
- ]
- }
- ]
- }
- ]
- }
- ]
- }
- ]
- },
- utterance: {
- '<>': 'p',
- html: [
- { '<>': 'p', html: 'Text: ${text}' },
- { '<>': 'p', html: 'Intent: ${intent}' }
- ]
- }
- }
- return json2html.transform(test, t.test)
-}
-
-function styleEntities (html, utterance, resultKind, actualUtterance) {
- var styleColor = 'rgba(0,255,0,.1)' // 50% transparent green
- if (
- (actualUtterance && (resultKind === 'FalsePositive')) || (!actualUtterance && (resultKind === 'FalseNegative'))
- ) { styleColor = 'rgba(255,0,0,.1)' } // 50% transparent red
- if (utterance.entities !== undefined) {
- if (utterance.entities.length > 0) {
- var searchText = utterance.entities[0].matchText
- html = html.replace(searchText, '' + searchText + '')
- }
- }
- return html
-}
-
-function getDotColor (d, i) {
- if ((d.resultKind === 'TruePositive') || (d.resultKind === 'TrueNegative')) {
- return 'green'
- }
- return 'red'
-}
-
-function getXConfusionQuadrant (graph, d, i, matrix) {
- const xScale = graph.xScale()
- switch (d.resultKind) {
- case 'TruePositive': matrix.tp += 1
- break
- case 'FalsePositive': matrix.fp += 1
- break
- case 'TrueNegative': matrix.tn += 1
- break
- case 'FalseNegative': matrix.fn += 1
- break
- }
- var x = 0
- if ((d.resultKind === 'TruePositive') || (d.resultKind === 'FalseNegative')) {
- x = xScale(i) + (graph.margin.left + graph.padding.left + 5)
- } else {
- x = xScale(i + graph.quadrantCountMax) + (graph.margin.left + graph.padding.left + 10)
- }
- return x
-}
-
-function getYConfusionQuadrant (graph, d, i) {
- const yScale = graph.yScale()
- var value = d.score || 1
- var retVal = 0
- if ((d.resultKind === 'TrueNegative') || (d.resultKind === 'FalseNegative')) {
- var negValue = 0 - value
- retVal = yScale(negValue)
- } else {
- retVal = yScale(value)
- }
- return retVal + (graph.margin.top + graph.padding.top)
-}
diff --git a/extensions/tabs/scripts/graph.js b/extensions/tabs/scripts/graph.js
deleted file mode 100644
index 7113f27..0000000
--- a/extensions/tabs/scripts/graph.js
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
-// Licensed under the MIT License.
-
-/* global d3 */
-
-// eslint-disable-next-line no-unused-vars
-class Graph {
- constructor () {
- this.margin = { top: 20, right: 20, bottom: 20, left: 20 }
- this.padding = { top: 60, right: 60, bottom: 60, left: 60 }
- this.outerWidth = 960
- this.outerHeight = 500
- this.innerWidth = this.outerWidth - this.margin.left - this.margin.right
- this.innerHeight = this.outerHeight - this.margin.top - this.margin.bottom
- this.width = this.innerWidth - this.padding.left - this.padding.right
- this.height = this.innerHeight - this.padding.top - this.padding.bottom
- this.quadrantCountMax = 0
- }
-
- yScale () {
- return d3.scaleLinear()
- .domain([1.2, -1.2])
- .range([0, this.height])
- }
-
- xScale () {
- return d3.scaleLinear()
- .domain([0, this.quadrantCountMax * 2])
- .range([0, this.width - 10])
- }
-}
diff --git a/extensions/tabs/scripts/init.js b/extensions/tabs/scripts/init.js
deleted file mode 100644
index 9a4f1a7..0000000
--- a/extensions/tabs/scripts/init.js
+++ /dev/null
@@ -1,41 +0,0 @@
-/* global VSS, renderConfusionMatrix */
-
-VSS.init({
- usePlatformScripts: true,
- taskRestClientLoaderConfig: {
- paths: {
- enhancer: 'scripts'
- }
- }
-})
-
-VSS.ready(() => {
- const context = VSS.getWebContext()
- VSS.getConfiguration().onBuildChanged(build => {
- VSS.require(['TFS/DistributedTask/TaskRestClient', 'VSS/Authentication/Services'], (taskRestClient, authServices) => {
- taskRestClient.getClient().getPlanAttachments(context.project.id, 'build', build.orchestrationPlan.planId, 'nlu.devops').then(attachments => {
- let metadata
- let statistics
- attachments.forEach(attachment => {
- if ((attachment.name === 'metadata.json' || attachment.name === 'statistics.json') && attachment._links && attachment._links.self && attachment._links.self.href) {
- metadata = attachment.name === 'metadata.json' ? attachment._links.self.href : metadata
- statistics = attachment.name === 'statistics.json' ? attachment._links.self.href : statistics
- }
- })
-
- if (!metadata) {
- console.warn('Could not find attachment for NLU metadata.')
- } else {
- VSS.getAccessToken().then(token => {
- var authToken = authServices.authTokenManager.getAuthorizationHeader(token)
- renderConfusionMatrix(metadata, { Authorization: authToken })
- })
- }
-
- if (!statistics) {
- console.warn('Could not find attachment for NLU statistics.')
- }
- })
- })
- })
-})
diff --git a/extensions/tabs/src/confusion.ts b/extensions/tabs/src/confusion.ts
new file mode 100644
index 0000000..e17a921
--- /dev/null
+++ b/extensions/tabs/src/confusion.ts
@@ -0,0 +1,350 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+// d3 and json2html included in HTML script reference
+/* global d3, json2html, Matrix, Graph */
+
+// json2html uses curly brace syntax for templating
+
+function renderConfusionMatrix(filename, headers) {
+ let quadrantCountMax = 0;
+ let nestData, facetActualIntents, facetExpectedIntents;
+ let facetActualEntity, facetExpectedEntity;
+
+ // Group elements into a hierarchial tree structure
+ // Explanation here: https://github.com/d3/d3-collection#nests
+ d3.json(filename, { headers }).then(function(data) {
+ nestData = d3.nest()
+ .key(function(d) { return d.resultKind })
+ .entries(data);
+ facetExpectedIntents = d3.nest()
+ .key(function(d) { return d.expectedUtterance.intent })
+ .entries(data);
+ facetActualIntents = d3.nest()
+ .key(function(d) { return d.actualUtterance.intent })
+ .entries(data);
+
+ facetActualEntity = d3.nest()
+ .key(function(d) {
+ let entityType = d.actualUtterance.entities.entityType;
+ let entities = d.actualUtterance.entities;
+
+ // Loop through entities where an entityType is present
+ if (entityType == undefined) {
+ for (let index = 0; index < entities.length; index++) {
+ const element = entities[index];
+
+ return element.entityType;
+ };
+ } else {
+ return; // TODO: Need to return something stating that there is no entity
+ }
+ })
+ .entries(data);
+
+ facetExpectedEntity = d3.nest()
+ .key(function(d) {
+ let entities = d.expectedUtterance.entities;
+
+ if (entities != undefined) {
+ // Loop through entities where an entityType is present
+ for (let index = 0; index < entities.length; index++) {
+ const element = entities[index];
+
+ return element.entityType;
+ };
+ } else {
+ return; // TODO: Need to return something stating that there is no entity
+ }
+ })
+ .entries(data);
+
+ for (let key = 0; key < nestData.length; key++) {
+ console.log(nestData[key]);
+ quadrantCountMax = Math.max(quadrantCountMax, nestData[key].values.length);
+ }
+
+ const matrix = new Matrix();
+ const graph = new Graph();
+ graph.quadrantCountMax = quadrantCountMax;
+
+ addGraph(graph);
+ addDataPoints(nestData, matrix, graph);
+ addFacets("Actual Intents" , facetActualIntents);
+ addFacets("Expected Intents", facetExpectedIntents);
+
+ addFacets("Actual Entity" , facetActualEntity);
+ addFacets("Expected Entity", facetExpectedEntity);
+ });
+}
+
+function addFacets(section, data) {
+ let slicerDiv = d3.select("#facets").append("div");
+
+ slicerDiv.append("div")
+ .attr("class", "heading")
+ .text(section);
+
+ slicerDiv.selectAll("p")
+ .data(data)
+ .enter()
+ .append("div")
+ .text(function(d) { return d.key; });
+}
+
+function addDataPoints(data, matrix, graph) {
+ for (let index = 0; index < data.length; index++) {
+ let tooltip = d3.select("body").append("div")
+ .attr("class", "tooltip")
+ .style("opacity", 0);
+ d3.select("svg")
+ .selectAll("elements")
+ .data(data[index].values)
+ .enter()
+ .append("circle")
+ .attr("r", 5)
+ .attr("cx", function(d, i) { return getXConfusionQuadrant(graph, d, i, matrix); })
+ .attr("cy", function(d, i) { return getYConfusionQuadrant(graph, d, i); })
+ .attr("stroke", function(d, i) { return getDotColor(d, i); })
+ .attr("fill", function(d, i) { return getDotColor(d, i); })
+ .attr("opacity", 0.5)
+ .on("mouseover", function(d, i) {
+ d3.selectAll(".active").classed("active", false);
+ d3.select("svg")
+ .append("circle")
+ .attr("cx", getXConfusionQuadrant(graph, d, i, matrix))
+ .attr("cy", getYConfusionQuadrant(graph, d, i))
+ .attr("stroke", getDotColor(d, i))
+ .attr("fill", getDotColor(d, i))
+ .attr("opacity", 0.5)
+ .classed("active", true)
+ .on("click", function() {
+ d3.selectAll(".clicked").classed("clicked", false);
+ const div = d3.select("#testData");
+ div.html(getTestResultHtml(d, matrix));
+ d3.selectAll(".active").classed("clicked", true);
+ })
+ .on("mouseover", function() {
+ tooltip.transition()
+ .duration(500)
+ .style("opacity", 0.9);
+ tooltip.html(d.score || "not scored")
+ .style("left", (d3.event.pageX) + "px")
+ .style("top", (d3.event.pageY - 28) + "px");
+ })
+ .on("mouseout", function() {
+ tooltip.transition()
+ .duration(500)
+ .style("opacity", 0);
+ d3.selectAll(".active").classed("active", false);
+ });
+ });
+ }
+}
+
+function addGraph(graph) {
+ const yScale = graph.yScale();
+
+ const yAxis = d3.axisLeft().scale(yScale);
+
+ const svg = d3.select("#confusionMatrix").append("svg")
+ .attr("viewBox", "0, 0," + graph.outerWidth + "," + graph.outerHeight)
+ .append("g")
+ .attr("transform", "translate(" + graph.margin.left + "," + graph.margin.top + ")");
+
+ svg.append("rect")
+ .attr("class", "outer")
+ .attr("width", graph.innerWidth)
+ .attr("height", graph.innerHeight);
+
+ const g = svg.append("g")
+ .attr("transform", "translate(" + graph.padding.left + "," + graph.padding.top + ")");
+
+ g.append("g")
+ .attr("class", "y axis")
+ .attr("transform", "translate(0,0)")
+ .call(yAxis);
+
+ g.selectAll("line.horizontalGrid").data(yScale.ticks(13))
+ .enter()
+ .append("line")
+ .attr("class", "horizontalGrid")
+ .attr("x1", 0)
+ .attr("x2", graph.width)
+ .attr("y1", function(d) { return yScale(d); })
+ .attr("y2", function(d) { return yScale(d); })
+ .attr("fill", "none")
+ .attr("shape-rendering", "crispEdges")
+ .attr("stroke", "steelblue")
+ .attr("stroke-dasharray", "10,10")
+ .attr("stroke-width", "1px");
+
+ g.append("text")
+ .attr("x", 10)
+ .attr("y", 15)
+ .text("True Positive");
+ g.append("text")
+ .attr("x", graph.width - 100)
+ .attr("y", 15)
+ .text("False Positive");
+ g.append("text")
+ .attr("x", 10)
+ .attr("y", graph.height - 5)
+ .text("False Negative");
+ g.append("text")
+ .attr("x", graph.width - 100)
+ .attr("y", graph.height - 5)
+ .text("True Negative");
+ g.append("line")
+ .attr("x1", 0)
+ .attr("y1", graph.height / 2)
+ .attr("x2", graph.width)
+ .attr("y2", graph.height / 2)
+ .attr("stroke", "steelblue");
+ g.append("line")
+ .attr("x1", graph.width / 2)
+ .attr("y1", 0)
+ .attr("x2", graph.width / 2)
+ .attr("y2", graph.height)
+ .attr("stroke", "steelblue");
+}
+
+function getTestResultHtml(test, matrix) {
+ console.log("getTestResults");
+ const t = {
+ test: {
+ "<>": "div",
+ "class": "col-8",
+ "html": [
+ { "<>": "div", "class": "heading", "html": "Model Statistics" },
+ {
+ "<>": "div",
+ "class": "row",
+ "html": [
+ { "<>": "div", "class": "col-3", "html": "Precision: " + matrix.getPrecision() },
+ { "<>": "div", "class": "col-3", "html": "Recall: " + matrix.getRecall() },
+ { "<>": "div", "class": "col-3", "html": "F1: " + matrix.getF1() },
+ { "<>": "div", "class": "col-3", "html": "Accuracy: " + matrix.getAccuracy() },
+ ],
+ },
+ { "<>": "br" },
+ { "<>": "div", "class": "heading", "html": "${testName}" },
+ { "<>": "p", "html": "Target: ${targetKind}" },
+ { "<>": "p", "html": "${resultKind}: ${because}" },
+ {
+ "<>": "div",
+ "class": "row",
+ "html": [
+ {
+ "<>": "table",
+ "class": "table",
+ "html": [
+ {
+ "<>": "thead",
+ "html": [
+ {
+ "<>": "tr",
+ "html": [
+ { "<>": "th", "scope": "col", "html": "Expected Utterance" },
+ { "<>": "th", "scope": "col", "html": "Actual Utterance" },
+ ],
+ },
+ ],
+ },
+ {
+ "<>": "tbody",
+ "html": [
+ {
+ "<>": "tr",
+ "html": [
+ {
+ "<>": "td",
+ "html"() {
+ const innerHtml = json2html.transform(this.expectedUtterance, t.utterance);
+ const actual = false;
+ return styleEntities(innerHtml, this.expectedUtterance, this.resultKind, actual);
+ },
+ },
+ {
+ "<>": "td",
+ "html"() {
+ const innerHtml = json2html.transform(this.actualUtterance, t.utterance);
+ const actual = true;
+ return styleEntities(innerHtml, this.actualUtterance, this.resultKind, actual);
+ },
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ ],
+ },
+ utterance: {
+ "<>": "p",
+ "html": [
+ { "<>": "p", "html": "Text: ${text}" },
+ { "<>": "p", "html": "Intent: ${intent}" },
+ ],
+ },
+ };
+ return json2html.transform(test, t.test);
+}
+
+function styleEntities(html, utterance, resultKind, actualUtterance) {
+ let styleColor = "rgba(0,255,0,.1)"; // 50% transparent green
+ if (
+ (actualUtterance && (resultKind === "FalsePositive")) || (!actualUtterance && (resultKind === "FalseNegative"))
+ ) { styleColor = "rgba(255,0,0,.1)"; } // 50% transparent red
+ if (utterance.entities !== undefined) {
+ if (utterance.entities.length > 0) {
+ const searchText = utterance.entities[0].matchText;
+ html = html.replace(searchText, '' + searchText + "");
+ }
+ }
+ return html;
+}
+
+function getDotColor(d, i) {
+ if ((d.resultKind === "TruePositive") || (d.resultKind === "TrueNegative")) {
+ return "green";
+ }
+ return "red";
+}
+
+function getXConfusionQuadrant(graph, d, i, matrix) {
+ const xScale = graph.xScale();
+ switch (d.resultKind) {
+ case "TruePositive": matrix.tp += 1;
+ break;
+ case "FalsePositive": matrix.fp += 1;
+ break;
+ case "TrueNegative": matrix.tn += 1;
+ break;
+ case "FalseNegative": matrix.fn += 1;
+ break;
+ }
+ let x = 0;
+ if ((d.resultKind === "TruePositive") || (d.resultKind === "FalseNegative")) {
+ x = xScale(i) + (graph.margin.left + graph.padding.left + 5);
+ } else {
+ x = xScale(i + graph.quadrantCountMax) + (graph.margin.left + graph.padding.left + 10);
+ }
+ return x;
+}
+
+function getYConfusionQuadrant(graph, d, i) {
+ const yScale = graph.yScale();
+ const value = d.score || 1;
+ let retVal = 0;
+ if ((d.resultKind === "TrueNegative") || (d.resultKind === "FalseNegative")) {
+ const negValue = 0 - value;
+ retVal = yScale(negValue);
+ } else {
+ retVal = yScale(value);
+ }
+ return retVal + (graph.margin.top + graph.padding.top);
+}
diff --git a/extensions/tabs/src/graph.ts b/extensions/tabs/src/graph.ts
new file mode 100644
index 0000000..d5f92c3
--- /dev/null
+++ b/extensions/tabs/src/graph.ts
@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+/* global d3 */
+
+// eslint-disable-next-line no-unused-vars
+class Graph {
+ constructor () {
+ this.margin = { top: 20, right: 20, bottom: 20, left: 20 }
+ this.padding = { top: 60, right: 60, bottom: 60, left: 60 }
+ this.outerWidth = 960
+ this.outerHeight = 500
+ this.innerWidth = this.outerWidth - this.margin.left - this.margin.right
+ this.innerHeight = this.outerHeight - this.margin.top - this.margin.bottom
+ this.width = this.innerWidth - this.padding.left - this.padding.right
+ this.height = this.innerHeight - this.padding.top - this.padding.bottom
+ this.quadrantCountMax = 0
+ }
+
+ yScale () {
+ return d3.scaleLinear()
+ .domain([1.2, -1.2])
+ .range([0, this.height])
+ }
+
+ xScale () {
+ return d3.scaleLinear()
+ .domain([0, this.quadrantCountMax * 2])
+ .range([0, this.width - 10])
+ }
+}
diff --git a/extensions/tabs/src/init.ts b/extensions/tabs/src/init.ts
new file mode 100644
index 0000000..3611eca
--- /dev/null
+++ b/extensions/tabs/src/init.ts
@@ -0,0 +1,45 @@
+/* global VSS, renderConfusionMatrix */
+
+// VSS.init({
+// usePlatformScripts: true,
+// taskRestClientLoaderConfig: {
+// paths: {
+// enhancer: 'scripts'
+// }
+// }
+// })
+
+// VSS.ready(() => {
+// const context = VSS.getWebContext()
+// VSS.getConfiguration().onBuildChanged(build => {
+// VSS.require(['TFS/DistributedTask/TaskRestClient', 'VSS/Authentication/Services'], (taskRestClient, authServices) => {
+// taskRestClient.getClient().getPlanAttachments(context.project.id, 'build', build.orchestrationPlan.planId, 'nlu.devops').then(attachments => {
+// let metadata
+// let statistics
+// attachments.forEach(attachment => {
+// if ((attachment.name === 'metadata.json' || attachment.name === 'statistics.json') && attachment._links && attachment._links.self && attachment._links.self.href) {
+// metadata = attachment.name === 'metadata.json' ? attachment._links.self.href : metadata
+// statistics = attachment.name === 'statistics.json' ? attachment._links.self.href : statistics
+// }
+// })
+
+// if (!metadata) {
+// console.warn('Could not find attachment for NLU metadata.')
+// } else {
+// VSS.getAccessToken().then(token => {
+// var authToken = authServices.authTokenManager.getAuthorizationHeader(token)
+// renderConfusionMatrix(metadata, { Authorization: authToken })
+// })
+// }
+
+// if (!statistics) {
+// console.warn('Could not find attachment for NLU statistics.')
+// }
+// })
+// })
+// })
+// })
+
+// Metadata for developing charts locally. Not used for production
+const metadata = "metadata.json";
+renderConfusionMatrix(metadata, { Authorization: "" });
diff --git a/extensions/tabs/scripts/matrix.js b/extensions/tabs/src/matrix.ts
similarity index 73%
rename from extensions/tabs/scripts/matrix.js
rename to extensions/tabs/src/matrix.ts
index 4848cee..497b55e 100644
--- a/extensions/tabs/scripts/matrix.js
+++ b/extensions/tabs/src/matrix.ts
@@ -3,31 +3,31 @@
// eslint-disable-next-line no-unused-vars
class Matrix {
- constructor () {
- this.p = 0
- this.n = 0
- this.tp = 0
- this.tn = 0
- this.fp = 0
- this.fn = 0
+ constructor() {
+ this.p = 0;
+ this.n = 0;
+ this.tp = 0;
+ this.tn = 0;
+ this.fp = 0;
+ this.fn = 0;
}
- getAccuracy () {
+ public getAccuracy() {
const accuracy = (this.tp + this.tn) / ((this.tp + this.fn) + (this.fp + this.tn))
return Math.round(accuracy * 100) / 100
}
- getF1 () {
+ public getF1() {
const f1 = 2 * this.tp / (2 * this.tp + this.fp + this.fn)
return Math.round(f1 * 100) / 100
}
- getPrecision () {
+ public getPrecision() {
const precision = this.tp / (this.tp + this.fp)
return Math.round(precision * 100) / 100
}
- getRecall () {
+ public getRecall() {
const recall = this.tp / (this.tp + this.fn)
return Math.round(recall * 100) / 100
}
diff --git a/extensions/tabs/tsconfig.json b/extensions/tabs/tsconfig.json
new file mode 100644
index 0000000..6644f4b
--- /dev/null
+++ b/extensions/tabs/tsconfig.json
@@ -0,0 +1,11 @@
+{
+ "compilerOptions": {
+ "outDir": "dist",
+ "allowJs": true,
+ "target": "es5",
+ "sourceMap": true
+ },
+ "include": [
+ "./src/**/*"
+ ]
+}