Merge pull request #9759 from keymanapp/feat/core/9468-test-improveme…

…nt-epic-ldml

common/web/types/src/ldml-keyboard/ldml-keyboard-testdata-xml.ts

@@ -41,34 +41,41 @@ export interface LKTTests {
 export interface LKTTest {
   name?: string;
   startContext?: LKTStartContext;
-  actions?: LKTAction[];  // differs from XML, to represent order of actions
+  actions?: LKTAnyAction[];  // differs from XML, to represent order of actions
 };
 
 export interface LKTStartContext {
   to?: string;
 };
 
-export interface LKTCheck {
+/**
+ * Test Actions.
+ * The expectation is that each LKTAction object will have exactly one non-falsy field.
+ */
+export interface LKTAction {
+  type?: "check" | "emit" | "keystroke" | "backspace";
+};
+
+export interface LKTCheck extends LKTAction {
+  type: "check";
   result?: string;
 };
 
-export interface LKTEmit {
+export interface LKTEmit extends LKTAction {
+  type: "emit";
   to?: string;
 };
 
-export interface LKTKeystroke {
+export interface LKTKeystroke extends LKTAction {
+  type: "keystroke";
   key?: string;
   flick?: string;
   longPress?: string;
   tapCount?: string;
 };
 
-/**
- * Test Actions.
- * The expectation is that each LKTAction object will have exactly one non-falsy field.
- */
-export interface LKTAction {
-  check?: LKTCheck;
-  emit?: LKTEmit;
-  keystroke?: LKTKeystroke;
-};
+export interface LKTBackspace extends LKTAction {
+  type: "backspace";
+}
+
+export type LKTAnyAction = LKTCheck | LKTEmit | LKTKeystroke | LKTBackspace;

common/web/types/src/ldml-keyboard/ldml-keyboard-xml-reader.ts

@@ -393,15 +393,12 @@ export class LDMLKeyboardXMLSourceFileReader {
         r.stuffBoxes(test, $$, 'startContext'); // singleton
         // now the actions
         test.actions = $$.map(v => {
-          const subtag = v['#name'];
-          const subv = LDMLKeyboardXMLSourceFileReader.defaultMapper(v, r);
-          switch(subtag) {
-            case 'keystroke': return { keystroke: subv };
-            case 'check':     return { check:     subv };
-            case 'emit':      return { emit:      subv };
-            case 'startContext': return null; // handled above
-            default: this.callbacks.reportMessage(CommonTypesMessages.Error_TestDataUnexpectedAction({ subtag })); return null;
+          const type = v['#name']; // element name
+          if (type === 'startContext') {
+            return null; // handled above
           }
+          const subv = LDMLKeyboardXMLSourceFileReader.defaultMapper(v, r);
+          return Object.assign({ type }, subv);
         }).filter(v => v !== null);
         return test;
       });

common/web/types/src/util/common-events.ts

@@ -43,8 +43,4 @@ export class CommonTypesMessages {
     m(this.ERROR_TestDataUnexpectedArray,
       `Problem reading test data: expected single ${o.subtag} element, found multiple`);
   static ERROR_TestDataUnexpectedArray = SevError | 0x0007;
-  static Error_TestDataUnexpectedAction = (o: {subtag: string}) =>
-    m(this.ERROR_TestDataUnexpectedAction,
-      `Problem reading test data: unexpected action element ${o.subtag}`);
-  static ERROR_TestDataUnexpectedAction = SevError | 0x0008;
 };

common/web/types/test/fixtures/test-fr.xml

@@ -1,6 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE keyboardTest3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest3.dtd">
 <keyboardTest3 conformsTo="techpreview">
+    <!--
+      Read by:
+      - common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts
+
+      Also see the other test-fr.xml in developer
+    -->
     <info keyboard="fr-t-k0-azerty.xml" author="Team Keyboard" name="fr-test" />
     <repertoire name="simple-repertoire" chars="[a b c d e \u{22}]" type="simple" /> <!-- verify that these outputs are all available from simple keys on any layer, for all form factors -->
     <repertoire name="chars-repertoire" chars="[á é ó]" type="gesture" /> <!-- verify that these outputs are all available from simple or gesture keys on any layer, for touch -->
@@ -17,6 +23,8 @@
             <!-- tests by specifying 'to' output char -->
             <emit to="v"/>
             <check result="abc\u0022...stuv" />
+            <backspace />
+            <check result="abc\u0022...stu" />
         </test>
     </tests>
 </keyboardTest3>

common/web/types/test/ldml-keyboard/test-ldml-keyboard-testdata-reader.ts

@@ -2,12 +2,14 @@ import { constants } from '@keymanapp/ldml-keyboard-constants';
 import { assert } from 'chai';
 import 'mocha';
 import { testTestdataReaderCases } from '../helpers/reader-callback-test.js';
+import { LKTAnyAction } from './ldml-keyboard-testdata-xml.js';
 
 describe('ldml keyboard xml reader tests', function () {
   this.slow(500); // 0.5 sec -- json schema validation takes a while
 
   testTestdataReaderCases([
     {
+      // Note! There's another test case against similar data, in developer/src/kmc-ldml/test/test-testdata-e2e.ts using test-fr.json
       subpath: 'test-fr.xml',
       callback: (data, source) => {
         assert.ok(source);
@@ -35,16 +37,19 @@ describe('ldml keyboard xml reader tests', function () {
         const test0 = source.keyboardTest3.tests[0].test[0];
         assert.equal('key-test', test0.name);
         assert.equal('abc\\u0022...', test0.startContext?.to);
-        assert.sameDeepOrderedMembers([
-          { keystroke: { key: 's' } },
-          { check: { result: 'abc\\u0022...s' } },
-          { keystroke: { key: 't' } },
-          { check: { result: 'abc\\u0022...st' } },
-          { keystroke: { key: 'u' } },
-          { check: { result: 'abc\\u0022...stu' } },
-          { emit: { to: 'v' } },
-          { check: { result: 'abc\\u0022...stuv' } },
-        ], test0.actions);
+        const expectedActions : LKTAnyAction[] = [
+          { type: "keystroke", key: 's' },
+          { type: "check", result: 'abc\\u0022...s' },
+          { type: "keystroke", key: 't' },
+          { type: "check", result: 'abc\\u0022...st' },
+          { type: "keystroke", key: 'u' },
+          { type: "check", result: 'abc\\u0022...stu' },
+          { type: "emit", to: 'v' },
+          { type: "check", result: 'abc\\u0022...stuv' },
+          { type: "backspace" },
+          { type: "check", result: 'abc\\u0022...stu' },
+        ];
+        assert.sameDeepOrderedMembers(expectedActions, test0.actions, 'Static data in .ts file should match parsed test-fr.xml');
       },
     }
   ]);

core/tests/unit/ldml/keyboards/k_006_backspace-test.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE keyboardTest3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboardTest3.dtd">
+<keyboardTest3 conformsTo="techpreview">
+  <info keyboard="k_006_backspace.xml" author="Team Keyboard" name="backspace" />
+  <tests name="backspace">
+    <test name="regex-test-0-a">
+      <startContext to="" />
+      <keystroke key="t" />
+      <keystroke key="a" />
+      <keystroke key="b" />
+      <backspace />
+      <check result="ta" /> <!-- TODO-LDML: #9760 Should be 't' due to backspace transform -->
+    </test>
+  </tests>
+</keyboardTest3>

core/tests/unit/ldml/keyboards/k_006_backspace.xml

@@ -1,12 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<!--
-from https://github.com/unicode-org/cldr/blob/keyboard-preview/docs/ldml/tr35-keyboards.md#element-transform
-
-@@keys: [K_T][K_A][K_B][K_BKSP]
-@@expected: t
-
--->
 <!DOCTYPE keyboard3 SYSTEM "../../../../../resources/standards-data/ldml-keyboards/techpreview/dtd/ldmlKeyboard3.dtd">
 <keyboard3 locale="en" conformsTo="techpreview">
   <info author="srl295" indicator="🙀" layout="qwerty" normalization="NFC" />

core/tests/unit/ldml/keyboards/meson.build

@@ -21,7 +21,6 @@ tests_without_testdata = [
   'k_003_transform',
   'k_004_tinyshift',
   'k_005_modbittest',
-  # 'k_006_backspace', ## not quite there yet. TODO-LDML
   'k_010_mt',
   'k_011_mt_iso',
   'k_100_keytest',
@@ -33,6 +32,7 @@ tests_without_testdata = [
 # These tests have a k_001_tiny-test.xml file as well.
 tests_with_testdata = [
   'k_001_tiny',
+  'k_006_backspace',
   'k_007_transform_rgx',
   'k_008_transform_norm',
   'k_020_fr', # TODO-LDML: move to cldr above (fix vkey)

core/tests/unit/ldml/ldml.cpp

@@ -213,6 +213,7 @@ run_test(const km::kbp::path &source, const km::kbp::path &compiled, km::tests::
 
   // Run through actions, applying output for each event
   for (test_source.next_action(action); action.type != km::tests::LDML_ACTION_DONE; test_source.next_action(action)) {
+    // handle backspace here
     if (action.type == km::tests::LDML_ACTION_KEY_EVENT) {
       auto &p = action.k;
       std::cout << "- key action: 0x" << std::hex << p.vk << "/modifier 0x" << p.modifier_state << std::dec << std::endl;
@@ -294,17 +295,25 @@ run_test(const km::kbp::path &source, const km::kbp::path &compiled, km::tests::
 /**
  * Run all tests for this keyboard
  */
-int run_all_tests(const km::kbp::path &source, const km::kbp::path &compiled) {
+int run_all_tests(const km::kbp::path &source, const km::kbp::path &compiled, const std::string &filter) {
   std::wcout << console_color::fg(console_color::BLUE) << "source file   = " << source << std::endl
             << "compiled file = " << compiled << console_color::reset() << std::endl;
+  if(!filter.empty()) {
+    std::wcout << "Running only tests matching (substring search): " << filter.c_str() << std::endl;
+  }
 
   km::tests::LdmlEmbeddedTestSource embedded_test_source;
 
   std::vector<std::string> failures; // track failures for summary
 
   int embedded_result = embedded_test_source.load_source(source);
 
-  if (embedded_result == 0) {
+  if (!filter.empty()) {
+    // Always skip the embedded test if there's a filter.
+    std::wcout << console_color::fg(console_color::YELLOW) << "SKIP: " << source.name() << " (embedded)" << console_color::reset()
+               << std::endl;
+    embedded_result = 0;  // no error
+  } else if (embedded_result == 0) {
     // embedded loaded OK, try it
     std::wcout << console_color::fg(console_color::BLUE) << console_color::bold() << "TEST: " << source.name() << " (embedded)"
                << console_color::reset() << std::endl;
@@ -325,31 +334,43 @@ int run_all_tests(const km::kbp::path &source, const km::kbp::path &compiled) {
   if (json_result != -1) {
     const km::tests::JsonTestMap& json_tests = json_factory.get_tests();
 
+    size_t skip_count = 0;
     assert(json_tests.size() > 0);
     // Loop over all tests
     for (const auto& n : json_tests) {
-      std::wcout << console_color::fg(console_color::BLUE) << console_color::bold() << "TEST: " << json_path.stem().c_str() << "/" << n.first.c_str() << console_color::reset() << std::endl;
+      const auto test_name = n.first;
+      auto qq = test_name.find(filter);
+      if (filter == "--list" || (qq == std::string::npos)) {
+        skip_count ++;
+        std::wcout << console_color::fg(console_color::YELLOW) << "SKIP: " << json_path.stem().c_str() << "/" << console_color::bold() << n.first.c_str() << console_color::reset() << std::endl;
+        continue;
+      }
+      std::wcout << console_color::fg(console_color::BLUE) << "TEST: " << json_path.stem().c_str() << "/" << console_color::bold() << n.first.c_str() << console_color::reset() << std::endl;
       int sub_test = run_test(source, compiled, *n.second);
       if (sub_test != 0) {
-        std::wcout << console_color::fg(console_color::BRIGHT_RED) << "FAIL: " << json_path.stem() << "/" << n.first.c_str()
+        std::wcout << console_color::fg(console_color::BRIGHT_RED) << "FAIL: " << json_path.stem() << "/" << console_color::bold() << n.first.c_str()
                    << console_color::reset() << std::endl;
         failures.push_back(json_path.stem() + "/" + n.first);
         json_result = sub_test; // set to last failure
       } else {
-        std::wcout << console_color::fg(console_color::GREEN) << " PASS: " << console_color::reset() << json_path.stem()
-                  << "/" << n.first.c_str() << std::endl;
+        std::wcout << console_color::fg(console_color::GREEN) << "PASS: " << console_color::reset() << json_path.stem()
+                  << "/" << console_color::bold() << n.first.c_str() << std::endl;
       }
     }
-    auto all_count = json_tests.size();
+    auto all_count  = json_tests.size();
     auto fail_count = failures.size();
-    auto pass_count = all_count - fail_count;
+    auto pass_count = all_count - fail_count - skip_count;
     if (pass_count > 0) {
       std::wcout << console_color::fg(console_color::GREEN) << " +" << pass_count;
     }
     if (fail_count > 0) {
      std::wcout << console_color::fg(console_color::BRIGHT_RED) <<
                 " -" << fail_count;
     }
+    if (skip_count > 0) {
+     std::wcout << console_color::fg(console_color::YELLOW) <<
+                " (skipped " << skip_count << ")";
+    }
     std::wcout << console_color::reset() << " of " << all_count << " JSON tests in "
               << json_path.stem() << std::endl;
   }
@@ -386,10 +407,11 @@ int run_all_tests(const km::kbp::path &source, const km::kbp::path &compiled) {
 
 constexpr const auto help_str =
     "\
-ldml [--color] <LDML_TEST_FILE> <KMX_FILE>\n\
+ldml [--color] <LDML_FILE> <KMX_FILE> [ <TEST_FILTER> | --list ]\n\
 help:\n\
-\tKMN_FILE:\tThe ldml test file for the keyboard under test.\n\
-\tKMX_FILE:\tThe corresponding compiled kmx file.\n";
+\tLDML_FILE:\tThe .xml file for the keyboard under test.\n\
+\tKMX_FILE:\tThe corresponding compiled kmx file.\n\
+\tTEST_FILTER:\tIf present, only run json tests containing the filter substring.  --list will list all tests\n";
 
 }  // namespace
 
@@ -402,20 +424,28 @@ int error_args() {
 int main(int argc, char *argv[]) {
   int first_arg = 1;
 
-  if (argc < 3) {
+  if ((argc - first_arg) < 2) { // if < 2 remaining args
     return error_args();
   }
 
-  auto arg_color = std::string(argv[1]) == "--color";
+  auto arg_color = std::string(argv[first_arg]) == "--color";
   if(arg_color) {
     first_arg++;
-    if(argc < 4) {
-      return error_args();
-    }
   }
   console_color::enabled = console_color::isaterminal() || arg_color;
 
-  int rc = run_all_tests(argv[first_arg], argv[first_arg + 1]);
+  if ((argc - first_arg) < 2) {
+    return error_args();
+  }
+  const km::kbp::path ldml_file = argv[first_arg++];
+  const km::kbp::path kmx_file  = argv[first_arg++];
+
+  std::string filter; // default to 'all tests'
+  if ((argc - first_arg) >= 1) {
+    filter = argv[first_arg++];
+  }
+
+  int rc = run_all_tests(ldml_file, kmx_file, filter);
   if (rc != EXIT_SUCCESS) {
     std::wcerr << console_color::fg(console_color::BRIGHT_RED) << "FAILED" << console_color::reset() << std::endl;
     rc = EXIT_FAILURE;