Merge branch 'feat/developer/7394-verify-normalization-epic-ldml' int…

…o feat/developer/10649-local-imports

HISTORY.md

@@ -1,5 +1,16 @@
 # Keyman Version History
 
+## 18.0.149 alpha 2024-12-01
+
+* refactor(developer): unify test action (#12736)
+
+## 18.0.148 alpha 2024-11-29
+
+* test(common/web/types): unit tests for unicodeset-parser-api (#12714)
+* chore(developer): rename test files (#12707)
+* feat(core,linux,developer,windows): implement loading KMX from blob (#12721)
+* chore(common): add offline support for emscripten (#12740)
+
 ## 18.0.147 alpha 2024-11-28
 
 * docs(android): Add android/docs/internal/README (#12717)

VERSION.md

@@ -1 +1 @@
-18.0.148
+18.0.150

common/include/test_assert.h

@@ -12,10 +12,10 @@
 #include <string>
 #include "test_color.h"
 
-#ifdef _assert_failed
-#undef _assert_failed
+#ifdef _test_assert_failed
+#undef _test_assert_failed
 #endif
-#define _assert_failed(result, exprText) { \
+#define _test_assert_failed(result, exprText) { \
   std::wcerr << console_color::fg(console_color::BRIGHT_RED) \
              << "Test failed with " << (result) \
              << " at " << __FILE__ << ":" << __LINE__ << ":" \
@@ -31,23 +31,23 @@
 #define try_status(expr) { \
   auto __s = (expr); \
   if (__s != KM_CORE_STATUS_OK) { \
-    _assert_failed(__s, u ## #expr); \
+    _test_assert_failed(__s, u ## #expr); \
   } \
 }
 
-#ifdef assert
-#undef assert
+#ifdef test_assert
+#undef test_assert
 #endif
-#define assert(expr) { \
+#define test_assert(expr) { \
   if (!(expr)) { \
-    _assert_failed(0, u ## #expr); \
+    _test_assert_failed(0, u ## #expr); \
   } \
 }
 
-#ifdef assert_equal
-#undef assert_equal
+#ifdef test_assert_equal
+#undef test_assert_equal
 #endif
-#define assert_equal(actual, expected) { \
+#define test_assert_equal(actual, expected) { \
   if ((actual) != (expected)) { \
     std::wcerr << console_color::fg(console_color::BRIGHT_RED) \
              << "Test failed at " << __FILE__ << ":" << __LINE__ << ":" \
@@ -59,10 +59,10 @@
   } \
 }
 
-#ifdef assert_string_equal
-#undef assert_string_equal
+#ifdef test_assert_string_equal
+#undef test_assert_string_equal
 #endif
-#define assert_string_equal(actual, expected) { \
+#define test_assert_string_equal(actual, expected) { \
   if (u16cmp((actual), (expected)) != 0) { \
     std::wcerr << console_color::fg(console_color::BRIGHT_RED) \
              << "Test failed at " << __FILE__ << ":" << __LINE__ << ":" \

common/include/test_color.h

@@ -8,6 +8,12 @@
 
 #include <ostream>
 
+#ifdef _MSC_VER
+#include <io.h>
+#else
+#include <unistd.h>
+#endif
+
 namespace console_color {
 
 enum ansi_code {
@@ -65,12 +71,10 @@ __define_ansi_code__(reversed, "7");
 #undef __define_ansi_code__
 
 #ifdef _MSC_VER
-#include <io.h>
 inline bool isaterminal() {
   return _isatty(_fileno(stdout));
 }
 #else
-#include <unistd.h>
 inline bool isaterminal() {
   return isatty(STDOUT_FILENO);
 }

common/web/types/src/kmx/kmx-plus/kmx-plus.ts

@@ -191,8 +191,8 @@ export class Strs extends Section {
     return result;
   }
 
-  /** process everything according to opts */
-  processString(s: string, opts: StrsOptions, sections: DependencySections) {
+  /** process everything according to opts, and add the string to this.allProcessedStrings */
+  private processString(s: string, opts: StrsOptions, sections: DependencySections) {
     s = s ?? '';
     // type check everything else
     if (typeof s !== 'string') {

common/web/types/src/util/util.ts

@@ -305,13 +305,13 @@ export function isPUA(ch: number) {
     (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END));
 }
 
-/** @returns true if s is NEITHER NFC nor NFD */
-export function isDenormalized(s: string) : boolean {
-  if(!s) return false; // empty or null
+/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */
+export function isNormalized(s: string) : boolean {
+  if(!s) return true; // empty or null
   const nfc = s.normalize("NFC");
   const nfd = s.normalize("NFD");
-  if (s !== nfc && s !== nfd) return true;
-  return false;
+  if (s !== nfc && s !== nfd) return false;
+  return true;
 }
 
 class BadStringMap extends Map<BadStringType, Set<number>> {

common/web/types/test/util/test-unescape.ts

@@ -1,6 +1,6 @@
 import 'mocha';
 import {assert} from 'chai';
-import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isDenormalized} from '../../src/util/util.js';
+import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js';
 
 describe('test UTF32 functions()', function() {
   it('should properly categorize strings', () => {
@@ -197,11 +197,11 @@ describe('test bad char functions', () => {
         'fas\u0323\u0307cinating', // NFD
         'd\u0323\u0307', // NFD
         '\u1e0d\u0307', // NFC
-      ].map(s => assert.isFalse(isDenormalized(s), `for string ${s}`));
+      ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`));
       [
         'd\u0307\u0323', // NFD but reversed marks
         'fas\u0307\u0323cinating', // not-NFD
-      ].map(s => assert.isTrue(isDenormalized(s), `for string ${s}`));
+      ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`));
     });
   });
 });

common/windows/delphi/general/KeymanPaths.pas

@@ -436,12 +436,18 @@ class function TKeymanPaths.RunningFromSource(var keyman_root: string): Boolean;
 class function TKeymanPaths.KeymanCoreLibraryPath(const Filename: string): string;
 var
   keyman_root: string;
+  configuration: string;
 begin
   // Look up KEYMAN_ROOT development variable -- if found and executable
   // within that path then use that as source path
   if TKeymanPaths.RunningFromSource(keyman_root) then
   begin
-    Exit(keyman_root + 'core\build\x86\debug\src\' + Filename);
+{$IFDEF DEBUG}
+    configuration := 'debug';
+{$ELSE}
+    configuration := 'release';
+{$ENDIF}
+    Exit(keyman_root + 'core\build\x86\'+configuration+'\src\' + Filename);
   end;
 
   Result := GetDebugPath('KeymanCoreLibraryPath', '');

core/src/action.cpp

@@ -61,7 +61,9 @@ bool km::core::action_item_list_to_actions_object(
             if(output.empty()) {
               actions->code_points_to_delete++;
             } else {
+#ifndef NDEBUG
               auto last_context_item = output.back();
+#endif
               output.pop_back();
               assert(last_context_item.type == KM_CORE_CT_CHAR);
               assert(last_context_item.character == action_items->backspace.expected_value);
@@ -71,7 +73,9 @@ bool km::core::action_item_list_to_actions_object(
             if(output.empty()) {
               // deleting a marker has no effect on the application
             } else {
+#ifndef NDEBUG
               auto last_context_item = output.back();
+#endif
               output.pop_back();
               assert(last_context_item.type == KM_CORE_CT_MARKER);
               assert(last_context_item.marker == action_items->backspace.expected_value);

core/src/ldml/ldml_markers.cpp

@@ -56,15 +56,19 @@ void add_back_markers(std::u32string &str, const std::u32string &src, marker_map
   str.clear();
   // iterator over the marker map
   auto marki = map2.rbegin();
+#ifndef NDEBUG
   // number of markers left to processnfd
   size_t max_markers = count_markers(map);
   size_t processed_markers = 0;
+#endif
 
   // add any end-of-text markers
   while(marki != map2.rend() && marki->ch == MARKER_BEFORE_EOT) {
     if (!marki->end) {
       prepend_marker(str, marki->marker, encoding);
+#ifndef NDEBUG
       processed_markers++;
+#endif
     }
     marki->processed = true;  // mark as done
     marki++;
@@ -88,7 +92,9 @@ void add_back_markers(std::u32string &str, const std::u32string &src, marker_map
           break;
         } else {
           prepend_marker(str, i->marker, encoding);
+#ifndef NDEBUG
           processed_markers++;
+#endif
         }
       }
     }

core/src/ldml/ldml_processor.cpp

@@ -410,8 +410,10 @@ ldml_event_state::remove_text(std::u32string &str, size_t length) {
   /** track how many context items have been removed, via push_backspace() */
   size_t contextRemoved = 0;
   for (auto c = state->context().rbegin(); length > 0 && c != state->context().rend(); c++, contextRemoved++) {
+#ifndef NDEBUG
     /** last char of context */
     km_core_usv lastCtx = str.back();
+#endif
     uint8_t type        = c->type;
     assert(type == KM_CORE_BT_CHAR || type == KM_CORE_BT_MARKER);
     if (type == KM_CORE_BT_CHAR) {

core/tests/kmx_test_source/kmx_test_source.cpp

@@ -18,6 +18,9 @@
 
 #include "kmx_test_source.hpp"
 
+#include <test_assert.h>
+#include <test_color.h>
+
 namespace km {
 namespace tests {
 
@@ -51,15 +54,15 @@ KmxTestSource::parse_source_string(std::string const &s) {
     if (*p == '\\') {
       p++;
       km_core_usv v;
-      assert(p != s.end());
+      test_assert(p != s.end());
       if (*p == 'u' || *p == 'U') {
         // Unicode value
         p++;
         size_t n;
         std::string s1 = s.substr(p - s.begin(), 8);
         v              = std::stoul(s1, &n, 16);
         // Allow deadkey_number (U+0001) characters and onward
-        assert(v >= 0x0001 && v <= 0x10FFFF);
+        test_assert(v >= 0x0001 && v <= 0x10FFFF);
         p += n - 1;
         if (v < 0x10000) {
           t += km_core_cu(v);
@@ -70,7 +73,7 @@ KmxTestSource::parse_source_string(std::string const &s) {
       } else if (*p == 'd') {
         // Deadkey
         // TODO, not yet supported
-        assert(false);
+        test_assert(false);
       }
     } else {
       t += *p;
@@ -212,7 +215,7 @@ KmxTestSource::get_keyboard_options(kmx_options options) {
 
 key_event
 KmxTestSource::char_to_event(char ch) {
-  assert(ch >= 32);
+  test_assert(ch >= 32);
   return {
       km::core::kmx::s_char_to_vkey[(int)ch - 32].vk,
       (uint16_t)(km::core::kmx::s_char_to_vkey[(int)ch - 32].shifted ? KM_CORE_MODIFIER_SHIFT : 0)};
@@ -258,8 +261,8 @@ KmxTestSource::vkey_to_event(std::string const &vk_event) {
   }
 
   // The string should be empty at this point
-  assert(!std::getline(f, s, ' '));
-  assert(vk != 0);
+  test_assert(!std::getline(f, s, ' '));
+  test_assert(vk != 0);
 
   return {vk, modifier_state};
 }
@@ -276,7 +279,7 @@ KmxTestSource::next_key(std::string &keys) {
       return char_to_event(ch);
     }
     auto n = keys.find(']');
-    assert(n != std::string::npos);
+    test_assert(n != std::string::npos);
     auto vkey = keys.substr(1, n - 1);
     keys.erase(0, n + 1);
     return vkey_to_event(vkey);

core/tests/meson.build

@@ -7,7 +7,7 @@
 #
 
 # Note: this version of cmpfiles ignores line endings, which is better for platform independence
-cmpfiles = ['-c', 'import sys; a = open(sys.argv[1], \'r\').read(); b = open(sys.argv[2], \'r\').read(); exit(not (a==b))']
+cmpfiles = ['-c', 'import sys; a = open(sys.argv[1], \'r\').read(); b = open(sys.argv[2], \'r\').read(); sys.exit(not (a==b))']
 stnds = join_paths(meson.current_source_dir(), 'standards')
 
 libsrc = include_directories(

core/tests/unit/emscripten_filesystem.cpp

@@ -4,7 +4,7 @@
 
 #include <emscripten.h>
 #include <iostream>
-#include <cassert>
+#include <test_assert.h>
 
 const std::string get_wasm_file_path(const std::string& filename) {
   // Verify that we are passing a fully-qualified path
@@ -13,13 +13,13 @@ const std::string get_wasm_file_path(const std::string& filename) {
   std::cout << "get_wasm_file_path ENTER (" << filename << ")" << std::endl;
 #endif
 
-  assert(
+  test_assert(
     (filename.length() > 0 && filename.at(0) == '/') ||
     (filename.length() > 1 && filename.at(1) == ':')
   );
 
 #if _DEBUG_FOPEN
-  std::cout << "get_wasm_file_path assert passed " << std::endl;
+  std::cout << "get_wasm_file_path test_assert passed " << std::endl;
 #endif
 
   EM_ASM_({