diff --git a/.github/workflows/deb-packaging.yml b/.github/workflows/deb-packaging.yml index 9c374f5d050..52c485aa658 100644 --- a/.github/workflows/deb-packaging.yml +++ b/.github/workflows/deb-packaging.yml @@ -116,7 +116,7 @@ jobs: strategy: fail-fast: true matrix: - dist: [focal, jammy, noble] + dist: [focal, jammy, noble, oracular] steps: - name: Checkout @@ -142,7 +142,7 @@ jobs: strategy: fail-fast: true matrix: - dist: [oracular] + dist: [plucky] steps: - name: Checkout diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4ebf41b87d1..6216a806143 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,7 @@ # Contributing to Keyman +(Keyman team members, see also the [onboarding](https://docs.google.com/document/d/1i6fBi9K38-LitcJZiRfAvRu1-7H0iQ_op5kxDMdhSec/edit?usp=sharing) doc) + ⭐ Thank you for your contribution! ⭐ The following is a set of guidelines for contributing to Keyman, Keyman diff --git a/HISTORY.md b/HISTORY.md index 8c03d3470cd..b831dadee8a 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,97 @@ # Keyman Version History +## 18.0.152 alpha 2024-12-04 + +* refactor(mac): pass kmx data blob to keyman core instead of file path (#12760) +* fix(developer): honour provided script when checking for matching scripts (#12768) +* chore(common): rename test files (#12709) +* fix(common): rename test file (#12770) + +## 18.0.151 alpha 2024-12-03 + +* feat(android): Enhance how ENTER key is handled for FV and KMSample2 (#12745) +* feat(developer): report on mismatching lang tag scripts when building keyboard-info (#12753) + +## 18.0.150 alpha 2024-12-02 + +* fix(core,developer): use `NDEBUG` flag to disable assertions in release build (#12715) + +## 18.0.149 alpha 2024-12-01 + +* refactor(developer): unify test action (#12736) + +## 18.0.148 alpha 2024-11-29 + +* test(common/web/types): unit tests for unicodeset-parser-api (#12714) +* chore(developer): rename test files (#12707) +* feat(core,linux,developer,windows): implement loading KMX from blob (#12721) +* chore(common): add offline support for emscripten (#12740) + +## 18.0.147 alpha 2024-11-28 + +* docs(android): Add android/docs/internal/README (#12717) +* test(common/web/types): unit tests for string-list (#12702) +* docs(common): linux and macOS emscripten setup (#12701) +* refactor(developer): output number of tests when running on TC (#12710) +* refactor(common): output number of tests when running on TC (#12719) +* chore(web): rename file missed in #12704 (#12720) +* fix(core): permanently disable logging (#12724) +* fix(linux): disable assertions in release builds of ibus-keyman (#12725) +* chore(common): improve offline builds (#12739) + +## 18.0.146 alpha 2024-11-27 + +* test(developer): kmcmplib compiler unit tests 5 (#12612) +* refactor(common): move all lexical model types into `LexicalModelTypes` container (#12712) +* refactor(common): move remaining LDML keyboard types into `LdmlKeyboardTypes` (#12713) +* chore(web): rename test files and folders (#12704) +* chore(core): rename test files (#12705) +* chore(linux): rename test files (#12706) + +## 18.0.145 alpha 2024-11-26 + +* docs(windows): update emscripten bash setup (#12700) +* chore(common): Add link to onboarding doc to `CONTRIBUTING.md` (#12697) + +## 18.0.144 alpha 2024-11-25 + +* chore(deps): bump cross-spawn from 7.0.3 to 7.0.6 in /developer/src/server/src/win32/trayicon/addon-src (#12687) +* chore(developer): make package subfile description fully optional (#12665) +* fix(developer): box package compiler info fields (#12666) +* fix(developer): correct whitespace handling in virtual keys and remove partially implemented virtual key series in kmcmplib compiler (#12604) + +## 18.0.143 alpha 2024-11-22 + +* chore(deps): bump cross-spawn from 7.0.3 to 7.0.6 (#12685) + +## 18.0.142 alpha 2024-11-20 + +* chore(common): Update CODEOWNERS (#12680) + +## 18.0.141 alpha 2024-11-15 + +* chore(linux): add support for Ubuntu 25.04 Plucky Puffin (#12675) + +## 18.0.140 alpha 2024-11-13 + +* chore(common): Add 17.0.330 - 17.0.332 to version history (#12663) +* fix(developer): reconnect `--full-test` in kmcmplib build and enable for CI (#12631) +* docs(developer): kmc-generate (#12647) + +## 18.0.139 alpha 2024-11-12 + +* fix(windows): help links updated (#12646) + +## 18.0.138 alpha 2024-11-08 + +* fix(common): check for invalid markers (#12613) +* chore: update minimum versions (#12632) +* fix(windows): correct path to output file in publish step for fv keyboards (#12637) +* chore(core): move API docs from help.keyman.com (#12642) +* feat(developer): kmc generate (#11014) +* feat(developer): kmc-copy (#12555) +* feat(developer): add GitHub and Cloud support to kmc-copy (#12586) + ## 18.0.137 alpha 2024-11-07 * fix(windows): correct engine help source path for upload (#12625) @@ -948,6 +1040,27 @@ * chore(common): move to 18.0 alpha (#10713) * chore: move to 18.0 alpha +## 17.0.332 stable 2024-11-06 + +* fix(developer): create Server config directory before options save (#12609) +* fix(developer): handle merge commits when checking git log date (#12628) +* fix(linux): set environment variable for rendering of downloads dialog (#12617) + +## 17.0.331 stable 2024-10-30 + +* fix(android): Hide suggestion banner on password fields (#12466) +* fix(common): declare dep on @keymanapp/ldml-keyboard-constants (#12475) +* fix(oem/fv): Update keyboard versions and names for fv_all.kmp (#12504) +* chore(ios): renew certificate (#12513) +* fix(developer): prevent invalid string ids (#12524) +* fix(developer): ignore excess whitespace in `` attribute (#12523) + +## 17.0.330 stable 2024-09-16 + +* refactor(android): Move Sentry and APK to publish task (#12392) +* fix(developer): rewrite ldml visual keyboard compiler (#12406) +* fix(developer): check vars string usage before definition (#12407) + ## 17.0.329 stable 2024-09-09 * chore(android,ios): Add ojibwa ifinal/rdot keyboards to FirstVoices (#12020) diff --git a/VERSION.md b/VERSION.md index 2ef34fe0072..6269a6496fb 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.138 \ No newline at end of file +18.0.153 \ No newline at end of file diff --git a/android/README.md b/android/README.md index 8b32b99f543..6abb94161d5 100644 --- a/android/README.md +++ b/android/README.md @@ -34,7 +34,7 @@ analytics for Debug are associated with an App Bundle ID ### Compiling From Command Line 1. Launch a command prompt and cd to the directory **keyman/android** -2. Run the top level build script `./build.sh configure build --debug` which will: +2. Run the top level build script `./build.sh configure build:engine build:app --debug` which will: * Compile KMEA (and its KMW dependency) * Download default keyboard and dictionary resources as needed * Compile KMAPro @@ -79,7 +79,7 @@ analytics for Debug are associated with an App Bundle ID Replace `SERIAL` with the device serial number listed in step 2. ### Compiling the app's offline help -Keyman for Android help is maintained in the Markdown files in android/docs/. +Keyman for Android help is maintained in the Markdown files in android/docs/help. The script `/resources/build/build-help.inc.sh` uses the `pandoc` tool to convert the Markdown files into html. ```bash @@ -121,7 +121,7 @@ Building these projects follow the same steps as KMAPro: ## How to Build Keyman Engine for Android 1. Open a terminal or Git Bash prompt and go to the Android project folder (e.g. `cd ~/keyman/android/`) -2. Run `./build.sh --debug` +2. Run `./build.sh build:engine --debug` Keyman Engine for Android library (**keyman-engine.aar**) is now ready to be imported in any project. @@ -167,3 +167,10 @@ dependencies { ```` 5. include `import com.keyman.engine.*;` to use Keyman Engine in a class. + +### Keyman Engine for Android help content +Keyman Engine for Android help is maintained in the Markdown files in android/docs/engine/. + +## Design Documentation + +Internal design documents about features pertaining to Keyman for Android and Keyman Engine for Android are maintained in the Markdown files in android/docs/internal/. diff --git a/android/Samples/KMSample2/app/src/main/java/com/keyman/kmsample2/SystemKeyboard.java b/android/Samples/KMSample2/app/src/main/java/com/keyman/kmsample2/SystemKeyboard.java index 4ad462a76c1..438ed237dbc 100644 --- a/android/Samples/KMSample2/app/src/main/java/com/keyman/kmsample2/SystemKeyboard.java +++ b/android/Samples/KMSample2/app/src/main/java/com/keyman/kmsample2/SystemKeyboard.java @@ -130,6 +130,11 @@ public void onStartInput(EditorInfo attribute, boolean restarting) { super.onStartInput(attribute, restarting); KMManager.onStartInput(attribute, restarting); KMManager.resetContext(KeyboardType.KEYBOARD_TYPE_SYSTEM); + + // Determine special handling for ENTER key + int inputType = attribute.inputType; + KMManager.setEnterMode(attribute.imeOptions, inputType); + // User switched to a new input field so we should extract the text from input field // and pass it to Keyman Engine together with selection range InputConnection ic = getCurrentInputConnection(); diff --git a/android/docs/internal/README.md b/android/docs/internal/README.md new file mode 100644 index 00000000000..ef51df7f60f --- /dev/null +++ b/android/docs/internal/README.md @@ -0,0 +1,5 @@ +# Keyman for Android and Keyman Engine for Android + +## Internal Documents + +This folder is for storing design documents of new features pertaining to Keyman for Android and Keyman Engine for Android diff --git a/common/include/test_assert.h b/common/include/test_assert.h index 75c586bcc67..9669c16a022 100644 --- a/common/include/test_assert.h +++ b/common/include/test_assert.h @@ -12,10 +12,10 @@ #include #include "test_color.h" -#ifdef _assert_failed -#undef _assert_failed +#ifdef _test_assert_failed +#undef _test_assert_failed #endif -#define _assert_failed(result, exprText) { \ +#define _test_assert_failed(result, exprText) { \ std::wcerr << console_color::fg(console_color::BRIGHT_RED) \ << "Test failed with " << (result) \ << " at " << __FILE__ << ":" << __LINE__ << ":" \ @@ -31,23 +31,23 @@ #define try_status(expr) { \ auto __s = (expr); \ if (__s != KM_CORE_STATUS_OK) { \ - _assert_failed(__s, u ## #expr); \ + _test_assert_failed(__s, u ## #expr); \ } \ } -#ifdef assert -#undef assert +#ifdef test_assert +#undef test_assert #endif -#define assert(expr) { \ +#define test_assert(expr) { \ if (!(expr)) { \ - _assert_failed(0, u ## #expr); \ + _test_assert_failed(0, u ## #expr); \ } \ } -#ifdef assert_equal -#undef assert_equal +#ifdef test_assert_equal +#undef test_assert_equal #endif -#define assert_equal(actual, expected) { \ +#define test_assert_equal(actual, expected) { \ if ((actual) != (expected)) { \ std::wcerr << console_color::fg(console_color::BRIGHT_RED) \ << "Test failed at " << __FILE__ << ":" << __LINE__ << ":" \ @@ -59,10 +59,10 @@ } \ } -#ifdef assert_string_equal -#undef assert_string_equal +#ifdef test_assert_string_equal +#undef test_assert_string_equal #endif -#define assert_string_equal(actual, expected) { \ +#define test_assert_string_equal(actual, expected) { \ if (u16cmp((actual), (expected)) != 0) { \ std::wcerr << console_color::fg(console_color::BRIGHT_RED) \ << "Test failed at " << __FILE__ << ":" << __LINE__ << ":" \ diff --git a/common/include/test_color.h b/common/include/test_color.h index f06f94c562f..71267dc3a94 100644 --- a/common/include/test_color.h +++ b/common/include/test_color.h @@ -8,6 +8,12 @@ #include +#ifdef _MSC_VER +#include +#else +#include +#endif + namespace console_color { enum ansi_code { @@ -65,12 +71,10 @@ __define_ansi_code__(reversed, "7"); #undef __define_ansi_code__ #ifdef _MSC_VER -#include inline bool isaterminal() { return _isatty(_fileno(stdout)); } #else -#include inline bool isaterminal() { return isatty(STDOUT_FILENO); } diff --git a/common/web/types/.eslintrc.cjs b/common/web/types/.eslintrc.cjs index b28ba0941b1..4fd5954e495 100644 --- a/common/web/types/.eslintrc.cjs +++ b/common/web/types/.eslintrc.cjs @@ -1,13 +1,13 @@ module.exports = { parserOptions: { - project: ["./tsconfig.json", "./test/tsconfig.json"], + project: ["./tsconfig.json", "./tests/tsconfig.json"], }, ignorePatterns: [ ".*/*", "build/*", "coverage/*", "node_modules/*", - "test/fixtures/*", + "tests/fixtures/*", "tools/*", "src/schemas/*" ], diff --git a/common/web/types/.gitignore b/common/web/types/.gitignore index 2f943a2f4e9..dcb567b832b 100644 --- a/common/web/types/.gitignore +++ b/common/web/types/.gitignore @@ -1,2 +1,3 @@ src/schemas/ -obj/ \ No newline at end of file +obj/ +coverage/ diff --git a/common/web/types/build.sh b/common/web/types/build.sh index 7dac43dba81..418840d1a85 100755 --- a/common/web/types/build.sh +++ b/common/web/types/build.sh @@ -80,20 +80,27 @@ function do_configure() { } function do_test() { + local MOCHA_FLAGS= + + if [[ "${TEAMCITY_GIT_PATH:-}" != "" ]]; then + # we're running in TeamCity + MOCHA_FLAGS="-reporter mocha-teamcity-reporter" + fi + eslint . - tsc --build test + tsc --build tests readonly C8_THRESHOLD=60 # Excludes are defined in .c8rc.json - c8 -skip-full --reporter=lcov --reporter=text --lines $C8_THRESHOLD --statements $C8_THRESHOLD --branches $C8_THRESHOLD --functions $C8_THRESHOLD mocha "${builder_extra_params[@]}" + c8 -skip-full --reporter=lcov --reporter=text --lines $C8_THRESHOLD --statements $C8_THRESHOLD --branches $C8_THRESHOLD --functions $C8_THRESHOLD mocha ${MOCHA_FLAGS} "${builder_extra_params[@]}" builder_echo warning "Coverage thresholds are currently $C8_THRESHOLD%, which is lower than ideal." builder_echo warning "Please increase threshold in build.sh as test coverage improves." } #------------------------------------------------------------------------------------------------------------------- -builder_run_action clean rm -rf ./build/ ./tsconfig.tsbuildinfo +builder_run_action clean rm -rf ./build/ ./tsconfig.tsbuildinfo ./src/schemas/ ./node_modules/ ./obj/ builder_run_action configure do_configure builder_run_action build tsc --build builder_run_action test do_test diff --git a/common/web/types/package.json b/common/web/types/package.json index e9f7c412947..d6efc635039 100644 --- a/common/web/types/package.json +++ b/common/web/types/package.json @@ -23,7 +23,7 @@ "build": "tsc -b", "build:schema": "ajv compile", "lint": "eslint .", - "test": "npm run lint && cd test && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" + "test": "npm run lint && cd tests && tsc -b && cd .. && c8 --skip-full --reporter=lcov --reporter=text mocha" }, "author": "Marc Durdin (https://github.com/mcdurdin)", "license": "MIT", @@ -48,7 +48,7 @@ "typescript": "^5.4.5" }, "mocha": { - "spec": "build/test/**/test-*.js", + "spec": "build/tests/**/*.tests.js", "require": [ "source-map-support/register" ] @@ -76,7 +76,7 @@ "src/keyman-touch-layout/keyman-touch-layout-file-writer.ts", "src/osk/osk.ts", "src/schemas/*", - "test/" + "tests/" ] }, "sideEffects": false diff --git a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts index 0cb49e527c7..a88688f7ab7 100644 --- a/common/web/types/src/kmx/kmx-plus/kmx-plus.ts +++ b/common/web/types/src/kmx/kmx-plus/kmx-plus.ts @@ -174,12 +174,7 @@ export class Strs extends Section { */ allocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem { // Run the string processing pipeline - s = Strs.processString(s, opts, sections); - - // add to the set, for testing - if (s) { - this.allProcessedStrings.add(s); - } + s = this.processString(s, opts, sections); // if it's a single char, don't push it into the strs table if (opts?.singleOk && isOneChar(s)) { @@ -196,8 +191,8 @@ export class Strs extends Section { return result; } - /** process everything according to opts */ - static processString(s: string, opts: StrsOptions, sections: DependencySections) { + /** process everything according to opts, and add the string to this.allProcessedStrings */ + private processString(s: string, opts: StrsOptions, sections: DependencySections) { s = s ?? ''; // type check everything else if (typeof s !== 'string') { @@ -215,6 +210,12 @@ export class Strs extends Section { if (opts?.unescape) { s = unescapeString(s); } + + if (s) { + // add all processed strings here, so that we catch denormalized strings in the input + this.allProcessedStrings.add(s); + } + // nfd if (opts?.nfd) { if (!sections?.meta?.normalizationDisabled) { diff --git a/common/web/types/src/main-ldml-keyboard.ts b/common/web/types/src/main-ldml-keyboard.ts new file mode 100644 index 00000000000..02bf990af26 --- /dev/null +++ b/common/web/types/src/main-ldml-keyboard.ts @@ -0,0 +1,8 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Just a wrapper to make `LdmlKeyboardTypes` export work. + */ +export { UnicodeSetParser, UnicodeSet } from './ldml-keyboard/unicodeset-parser-api.js'; +export { VariableParser, MarkerParser } from './ldml-keyboard/pattern-parser.js'; +export { ElementString } from './kmx/kmx-plus/element-string.js'; diff --git a/common/web/types/src/main.ts b/common/web/types/src/main.ts index c3b24c5123d..071c931e9e9 100644 --- a/common/web/types/src/main.ts +++ b/common/web/types/src/main.ts @@ -25,11 +25,8 @@ export * as Schemas from './schemas.js'; export * as SchemaValidators from './schema-validators.js'; export * as KMXPlus from './kmx/kmx-plus/kmx-plus.js'; -// TODO: these exports are really not well named -export { UnicodeSetParser, UnicodeSet } from './ldml-keyboard/unicodeset-parser-api.js'; -export { VariableParser, MarkerParser } from './ldml-keyboard/pattern-parser.js'; -export { ElementString } from './kmx/kmx-plus/element-string.js'; +export * as LdmlKeyboardTypes from './main-ldml-keyboard.js'; -export { USVString, CasingForm, CasingFunction, TextWithProbability, LexiconTraversal, LexicalModel, LexicalModelPunctuation, Transform, Suggestion, Reversion, Keep, SuggestionTag, Context, Distribution, Outcome, WithOutcome, ProbabilityMass, Configuration, Capabilities, WordBreakingFunction, Span } from './lexical-model-types.js'; +export * as LexicalModelTypes from './lexical-model-types.js'; export * as KeymanWebKeyboard from './keyboard-object.js'; diff --git a/common/web/types/src/package/kmp-json-file.ts b/common/web/types/src/package/kmp-json-file.ts index ae88b5ba3e9..4a1c1c159e4 100644 --- a/common/web/types/src/package/kmp-json-file.ts +++ b/common/web/types/src/package/kmp-json-file.ts @@ -44,7 +44,7 @@ export interface KmpJsonFileInfoItem { export interface KmpJsonFileContentFile { name: string; - description: string; + description?: string; copyLocation?: number; } diff --git a/common/web/types/src/util/util.ts b/common/web/types/src/util/util.ts index 0d13505cfc9..70e1077cc5a 100644 --- a/common/web/types/src/util/util.ts +++ b/common/web/types/src/util/util.ts @@ -305,6 +305,15 @@ export function isPUA(ch: number) { (ch >= Uni_PUA_16_START && ch <= Uni_PUA_16_END)); } +/** @returns false if s is NEITHER NFC nor NFD. (Returns true for falsy) */ +export function isNormalized(s: string) : boolean { + if(!s) return true; // empty or null + const nfc = s.normalize("NFC"); + const nfd = s.normalize("NFD"); + if (s !== nfc && s !== nfd) return false; + return true; +} + class BadStringMap extends Map> { public toString() : string { if (!this.size) { diff --git a/common/web/types/test/lexical-model-types.tests.ts b/common/web/types/test/lexical-model-types.tests.ts deleted file mode 100644 index 8944342ed6e..00000000000 --- a/common/web/types/test/lexical-model-types.tests.ts +++ /dev/null @@ -1,25 +0,0 @@ -/** - * This file "tests" the exports from the main module. - * - * Since the exports are all types, the "test" here is that the type can be - * imported and compiled without any compiler errors. - */ - -import { USVString, Transform, Suggestion, SuggestionTag, Context, Capabilities, Configuration, Distribution, WordBreakingFunction, Span, LexicalModelPunctuation, ElementString, KMXPlus } from '@keymanapp/common-types'; - -export let u: USVString; -export let l: Transform -export let s: Suggestion; -export let st: SuggestionTag; -export let c: Context; -export let cap: Capabilities; -export let conf: Configuration; -export let d: Distribution; -export let wbf: WordBreakingFunction; -export let sp: Span; -export let lmp: LexicalModelPunctuation; - - -// try some of the other types - that should still work -export let elemString: ElementString; -export let section: KMXPlus.Section; diff --git a/common/web/types/test/fixtures/kmx/khmer_angkor.kmx b/common/web/types/tests/fixtures/kmx/khmer_angkor.kmx similarity index 100% rename from common/web/types/test/fixtures/kmx/khmer_angkor.kmx rename to common/web/types/tests/fixtures/kmx/khmer_angkor.kmx diff --git a/common/web/types/test/fixtures/kvk/balochi_inpage.kvk b/common/web/types/tests/fixtures/kvk/balochi_inpage.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/balochi_inpage.kvk rename to common/web/types/tests/fixtures/kvk/balochi_inpage.kvk diff --git a/common/web/types/test/fixtures/kvk/khmer_angkor.kvk b/common/web/types/tests/fixtures/kvk/khmer_angkor.kvk similarity index 100% rename from common/web/types/test/fixtures/kvk/khmer_angkor.kvk rename to common/web/types/tests/fixtures/kvk/khmer_angkor.kvk diff --git a/common/web/types/test/helpers/index.ts b/common/web/types/tests/helpers/index.ts similarity index 87% rename from common/web/types/test/helpers/index.ts rename to common/web/types/tests/helpers/index.ts index 3e305eeca8a..717b2e75191 100644 --- a/common/web/types/test/helpers/index.ts +++ b/common/web/types/tests/helpers/index.ts @@ -9,5 +9,5 @@ import { fileURLToPath } from "url"; * @param components One or more path components. */ export function makePathToFixture(...components: string[]): string { - return fileURLToPath(new URL(path.join('..', '..', '..', 'test', 'fixtures', ...components), import.meta.url)); + return fileURLToPath(new URL(path.join('..', '..', '..', 'tests', 'fixtures', ...components), import.meta.url)); } diff --git a/common/web/types/test/kmx/test-keyman-targets.ts b/common/web/types/tests/kmx/keyman-targets.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-keyman-targets.ts rename to common/web/types/tests/kmx/keyman-targets.tests.ts diff --git a/common/web/types/test/kmx/test-kmx-file.ts b/common/web/types/tests/kmx/kmx-file.tests.ts similarity index 100% rename from common/web/types/test/kmx/test-kmx-file.ts rename to common/web/types/tests/kmx/kmx-file.tests.ts diff --git a/common/web/types/test/kvk/test-kvk-file.ts b/common/web/types/tests/kvk/kvk-file.tests.ts similarity index 90% rename from common/web/types/test/kvk/test-kvk-file.ts rename to common/web/types/tests/kvk/kvk-file.tests.ts index 8a1179b67b2..8e0387fefe3 100644 --- a/common/web/types/test/kvk/test-kvk-file.ts +++ b/common/web/types/tests/kvk/kvk-file.tests.ts @@ -2,7 +2,7 @@ import * as fs from 'fs'; import 'mocha'; import { makePathToFixture } from '../helpers/index.js'; import KvkFileReader from "../../src/kvk/kvk-file-reader.js"; -import { verify_balochi_inpage, verify_khmer_angkor } from './test-kvk-utils.js'; +import { verify_balochi_inpage, verify_khmer_angkor } from './kvk-utils.tests.js'; describe('kvk-file-reader', function () { it('kvk-file-reader should read a valid file', function() { diff --git a/common/web/types/test/kvk/test-kvk-utils.ts b/common/web/types/tests/kvk/kvk-utils.tests.ts similarity index 100% rename from common/web/types/test/kvk/test-kvk-utils.ts rename to common/web/types/tests/kvk/kvk-utils.tests.ts diff --git a/common/web/types/test/ldml-keyboard/test-pattern-parser.ts b/common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts similarity index 100% rename from common/web/types/test/ldml-keyboard/test-pattern-parser.ts rename to common/web/types/tests/ldml-keyboard/pattern-parser.tests.ts diff --git a/common/web/types/tests/ldml-keyboard/string-list.tests.ts b/common/web/types/tests/ldml-keyboard/string-list.tests.ts new file mode 100644 index 00000000000..58365bda66b --- /dev/null +++ b/common/web/types/tests/ldml-keyboard/string-list.tests.ts @@ -0,0 +1,216 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by Dr Mark C. Sinclair on 2024-11-28 + * + * Test code for string-lists.ts + */ + +import 'mocha'; +import { assert } from 'chai'; +import { StrsItem, StrsOptions, DependencySections, Strs } from '../../src/kmx/kmx-plus/kmx-plus.js'; +import { ListIndex, ListItem } from '../../src/ldml-keyboard/string-list.js'; + +describe('Test of String-List', () => { + describe('Test ListIndex', () => { + it('can construct a ListIndex', () => { + const strsItem = new StrsItem("abc"); + const actual = new ListIndex(strsItem); + assert.deepEqual(actual.value, strsItem); + }); + it('can check two ListIndex for equality', () => { + const listItemOne = new ListIndex(new StrsItem("abc")); + const listItemTwo = new ListIndex(new StrsItem("abc")); + assert.isTrue(listItemOne.isEqual(listItemTwo)); + }); + it('can check two different ListIndex are not equal', () => { + const listItemOne = new ListIndex(new StrsItem("abc")); + const listItemTwo = new ListIndex(new StrsItem("def")); + assert.isFalse(listItemOne.isEqual(listItemTwo)); + }); + it('can check a ListIndex and string for equality', () => { + const listItem = new ListIndex(new StrsItem("abc")); + const aString = "abc"; + assert.isTrue(listItem.isEqual(aString)); + }); + it('can check a ListIndex and string for inequality', () => { + const listItem = new ListIndex(new StrsItem("abc")); + const aString = "def"; + assert.isFalse(listItem.isEqual(aString)); + }); + it('can provide a correct string representation', () => { + const strsItem = new StrsItem("abc"); + const listItem = new ListIndex(strsItem); + const expected = "abc"; + assert.deepEqual(listItem.toString(), expected); + }); + }); + describe('Test ListItem', () => { + describe('Test fromStrings()', () => { + it('should return an empty ListItem if source is null', () => { + const actual = ListItem.fromStrings(null, null, null); + const expected = new ListItem(); + assert.deepEqual(actual, expected); + }); + it('should return a valid ListItem from a single source string', () => { + const source = ["abc"]; + const sections = { strs: new Strs }; + sections.strs.allocString = stubSectionsStrsAllocString; + const actual = ListItem.fromStrings(source, null, sections); + const expected = initListItem(source); + assert.deepEqual(actual, expected); + }); + it('should return a valid ListItem from a longer source', () => { + const source = ["abc", "def", "ghi"]; + const sections = { strs: new Strs }; + sections.strs.allocString = stubSectionsStrsAllocString; + const actual = ListItem.fromStrings(source, null, sections); + const expected = initListItem(source); + assert.deepEqual(actual, expected); + }); + }); + describe('Test getItemOrder()', () => { + it('should return a valid index for the first item', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + const index = listItem.getItemOrder("abc"); + assert.equal(index, 0); + }); + it('should return a valid index for a later item', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + const index = listItem.getItemOrder("ghi"); + assert.equal(index, 2); + }); + it('should return -1 for a missing item', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + const index = listItem.getItemOrder("jkl"); + assert.equal(index, -1); + }); + }); + describe('Test isEqual()', () => { + it('should return true for two empty ListItems', () => { + const listItemOne = new ListItem(); + const listItemTwo = new ListItem(); + assert.isTrue(listItemOne.isEqual(listItemTwo)); + }); + it('should return false for empty and non-empty ListItems', () => { + const listItemOne = new ListItem(); + const listItemTwo = initListItem(["abc"]); + assert.isFalse(listItemOne.isEqual(listItemTwo)); + }); + it('should return false for non-empty and empty ListItems', () => { + const listItemOne = initListItem(["abc"]); + const listItemTwo = new ListItem(); + assert.isFalse(listItemOne.isEqual(listItemTwo)); + }); + it('should return true for identical ListItems', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.isTrue(listItemOne.isEqual(listItemTwo)); + }); + it('should return false for different ListItems', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abd", "def", "ghi"]); + assert.isFalse(listItemOne.isEqual(listItemTwo)); + }); + it('should return false for different length ListItems', () => { + const listItemOne = initListItem(["abc", "def"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.isFalse(listItemOne.isEqual(listItemTwo)); + }); + it('should return true for empty ListItem and string[]', () => { + const listItem = new ListItem(); + assert.isTrue(listItem.isEqual([])); + }); + it('should return false for empty ListItem and non-empty string[]', () => { + const listItem = new ListItem(); + assert.isFalse(listItem.isEqual(["abc"])); + }); + it('should return false for non-empty ListItem and empty string[]', () => { + const listItem = initListItem(["abc"]);; + assert.isFalse(listItem.isEqual([])); + }); + it('should return true for identical ListItem and string[]', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + assert.isTrue(listItem.isEqual(["abc", "def", "ghi"])); + }); + it('should return false for different ListItem and string[]', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + assert.isFalse(listItem.isEqual(["abd", "def", "ghi"])); + }); + it('should return false for different length ListItem and string[]', () => { + const listItem = initListItem(["abc", "def"]); + assert.isFalse(listItem.isEqual(["abc", "def", "ghi"])); + }); + }); + describe('Test compareTo()', () => { + it('should return 0 for identical ListItems', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.equal(listItemOne.compareTo(listItemTwo), 0); + }); + it('should return -1 for ListItems with different first items (smallest first)', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abd", "def", "ghi"]); + assert.equal(listItemOne.compareTo(listItemTwo), -1); + }); + it('should return 1 for ListItems with different first items (smallest second)', () => { + const listItemOne = initListItem(["abd", "def", "ghi"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.equal(listItemOne.compareTo(listItemTwo), 1); + }); + it('should return -1 for ListItems with different later items (smallest first)', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abc", "def", "ghj"]); + assert.equal(listItemOne.compareTo(listItemTwo), -1); + }); + it('should return 1 for ListItems with different later items (smallest second)', () => { + const listItemOne = initListItem(["abc", "def", "ghj"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.equal(listItemOne.compareTo(listItemTwo), 1); + }); + it('should return -1 for identical ListItems, except shorter first', () => { + const listItemOne = initListItem(["abc", "def", "ghi"]); + const listItemTwo = initListItem(["abc", "def", "ghi", "jkl"]); + assert.equal(listItemOne.compareTo(listItemTwo), -1); + }); + it('should return 1 for identical ListItems, except longer first', () => { + const listItemOne = initListItem(["abc", "def", "ghi", "jkl"]); + const listItemTwo = initListItem(["abc", "def", "ghi"]); + assert.equal(listItemOne.compareTo(listItemTwo), 1); + }); + }); + describe('Test toString()', () => { + it('should return correct string', () => { + const listItem = initListItem(["abc", "def", "ghi"]); + assert.deepEqual(listItem.toString(), "abc def ghi"); + }); + it('should return correct string for empty ListItem', () => { + const listItem = new ListItem; + assert.deepEqual(listItem.toString(), ""); + }); + }); + describe('Test toStringArray()', () => { + it('should return correct string[]', () => { + const source = ["abc", "def", "ghi"]; + const listItem = initListItem(source); + assert.deepEqual(listItem.toStringArray(), source); + }); + it('should return correct string[] for empty ListItem', () => { + const listItem = new ListItem; + assert.deepEqual(listItem.toStringArray(), []); + }); + }); + }); +}); + +function stubSectionsStrsAllocString(s?: string, opts?: StrsOptions, sections?: DependencySections): StrsItem { + return new StrsItem(s); +} + +function initListItem(source: Array): ListItem { + const listItem = new ListItem(); + for (const s of source) { + listItem.push(new ListIndex(new StrsItem(s))); + } + return listItem; +} diff --git a/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts new file mode 100644 index 00000000000..763bc6d5bb9 --- /dev/null +++ b/common/web/types/tests/ldml-keyboard/unicodeset-parser-api.tests.ts @@ -0,0 +1,24 @@ +/* + * Keyman is copyright (C) SIL Global. MIT License. + * + * Created by Dr Mark C. Sinclair on 2024-11-29 + * + * Test code for unicodeset-parser-api.ts + */ + +import 'mocha'; +import { assert } from 'chai'; +import { UnicodeSet } from '../../src/ldml-keyboard/unicodeset-parser-api.js'; + +describe('Test of Unicode-Parser-API', () => { + describe('Test UnicodeSet', () => { + it('can provide a correct ranges length', () => { + const unicodeSet = new UnicodeSet("[ħa-z]", [[0x41, 0x7A], [0x0127, 0x0127]]); + assert.equal(unicodeSet.length, 2); + }); + it('can provide a correct string representation', () => { + const unicodeSet = new UnicodeSet("[ħa-z]", [[0x41, 0x7A], [0x0127, 0x0127]]); + assert.deepEqual(unicodeSet.toString(), "[ħa-z]"); + }); + }); +}); diff --git a/common/web/types/tests/lexical-model-types.tests.ts b/common/web/types/tests/lexical-model-types.tests.ts new file mode 100644 index 00000000000..ef8bed28a50 --- /dev/null +++ b/common/web/types/tests/lexical-model-types.tests.ts @@ -0,0 +1,25 @@ +/** + * This file "tests" the exports from the main module. + * + * Since the exports are all types, the "test" here is that the type can be + * imported and compiled without any compiler errors. + */ + +import { KMXPlus, LdmlKeyboardTypes, LexicalModelTypes } from "@keymanapp/common-types"; + +export let u: LexicalModelTypes.USVString; +export let l: LexicalModelTypes.Transform; +export let s: LexicalModelTypes.Suggestion; +export let st: LexicalModelTypes.SuggestionTag; +export let c: LexicalModelTypes.Context; +export let cap: LexicalModelTypes.Capabilities; +export let conf: LexicalModelTypes.Configuration; +export let d: LexicalModelTypes.Distribution; +export let wbf: LexicalModelTypes.WordBreakingFunction; +export let sp: LexicalModelTypes.Span; +export let lmp: LexicalModelTypes.LexicalModelPunctuation; + + +// try some of the other types - that should still work +export let elemString: LdmlKeyboardTypes.ElementString; +export let section: KMXPlus.Section; diff --git a/common/web/types/test/tsconfig.json b/common/web/types/tests/tsconfig.json similarity index 90% rename from common/web/types/test/tsconfig.json rename to common/web/types/tests/tsconfig.json index 9678c49945f..45e5311fdc0 100644 --- a/common/web/types/test/tsconfig.json +++ b/common/web/types/tests/tsconfig.json @@ -4,13 +4,12 @@ "compilerOptions": { "rootDir": ".", "rootDirs": ["./", "../src/"], - "outDir": "../build/test", + "outDir": "../build/tests", "baseUrl": ".", "strictNullChecks": false, // TODO: get rid of this as some point "allowSyntheticDefaultImports": true }, "include": [ - "**/test-*.ts", "**/*.tests.ts", "./helpers/*.ts", ], diff --git a/common/web/types/test/util/test-unescape.ts b/common/web/types/tests/util/unescape.tests.ts similarity index 94% rename from common/web/types/test/util/test-unescape.ts rename to common/web/types/tests/util/unescape.tests.ts index 8c75b491824..3eaafd7150f 100644 --- a/common/web/types/test/util/test-unescape.ts +++ b/common/web/types/tests/util/unescape.tests.ts @@ -1,6 +1,6 @@ import 'mocha'; import {assert} from 'chai'; -import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer} from '../../src/util/util.js'; +import {unescapeString, UnescapeError, isOneChar, toOneChar, unescapeOneQuadString, BadStringAnalyzer, isValidUnicode, describeCodepoint, isPUA, BadStringType, unescapeStringToRegex, unescapeQuadString, NFDAnalyzer, isNormalized} from '../../src/util/util.js'; describe('test UTF32 functions()', function() { it('should properly categorize strings', () => { @@ -186,6 +186,24 @@ describe('test bad char functions', () => { assert.isTrue(isPUA(ch), describeCodepoint(ch)); } }); + describe('test isDenormalized()', () => { + it('should correctly categorize strings', () => { + [ + undefined, + null, + '', + 'ABC', + 'fa\u1E69cinating', // NFC + 'fas\u0323\u0307cinating', // NFD + 'd\u0323\u0307', // NFD + '\u1e0d\u0307', // NFC + ].map(s => assert.isTrue(isNormalized(s), `for string ${s}`)); + [ + 'd\u0307\u0323', // NFD but reversed marks + 'fas\u0307\u0323cinating', // not-NFD + ].map(s => assert.isFalse(isNormalized(s), `for string ${s}`)); + }); + }); }); describe('test BadStringAnalyzer', () => { diff --git a/common/windows/delphi/general/KeymanPaths.pas b/common/windows/delphi/general/KeymanPaths.pas index bc534b9f526..fbfa9961d70 100644 --- a/common/windows/delphi/general/KeymanPaths.pas +++ b/common/windows/delphi/general/KeymanPaths.pas @@ -436,12 +436,18 @@ class function TKeymanPaths.RunningFromSource(var keyman_root: string): Boolean; class function TKeymanPaths.KeymanCoreLibraryPath(const Filename: string): string; var keyman_root: string; + configuration: string; begin // Look up KEYMAN_ROOT development variable -- if found and executable // within that path then use that as source path if TKeymanPaths.RunningFromSource(keyman_root) then begin - Exit(keyman_root + 'core\build\x86\debug\src\' + Filename); +{$IFDEF DEBUG} + configuration := 'debug'; +{$ELSE} + configuration := 'release'; +{$ENDIF} + Exit(keyman_root + 'core\build\x86\'+configuration+'\src\' + Filename); end; Result := GetDebugPath('KeymanCoreLibraryPath', ''); diff --git a/core/docs/api/index.md b/core/docs/api/index.md index 8112d9c8793..c33246f7f44 100644 --- a/core/docs/api/index.md +++ b/core/docs/api/index.md @@ -75,6 +75,7 @@ modifiers such as Windows key are excluded from this set. Some modifiers are transient, such as Control, and others have long-lasting state, such as Caps Lock. +- __See more in__ [Keyman Glossary](https://github.com/keymanapp/keyman/wiki/Keyman-glossary) [km_core_cp]: background#km_core_cp "km_core_cp type" [km_core_usv]: background#km_core_usv "km_core_usv type" diff --git a/core/include/keyman/keyman_core_api.h b/core/include/keyman/keyman_core_api.h index 328965e8761..3c41182bab2 100644 --- a/core/include/keyman/keyman_core_api.h +++ b/core/include/keyman/keyman_core_api.h @@ -1007,7 +1007,11 @@ Provides read-only information about a keyboard. typedef struct { km_core_cu const * version_string; km_core_cu const * id; + + // TODO-web-core: Deprecate this field (#12497) + // KMN_DEPRECATED km_core_path_name folder_path; + km_core_option_item const * default_options; } km_core_keyboard_attrs; @@ -1022,7 +1026,7 @@ typedef struct { : Keyman keyboard ID string. `folder_path` -: Path to the unpacked folder containing the keyboard and associated resources. +: Path to the unpacked folder containing the keyboard and associated resources (deprecated). `default_options` : Set of default values for any options included in the keyboard. @@ -1096,12 +1100,16 @@ typedef struct { ## Description +DEPRECATED: use [km_core_keyboard_load_from_blob] instead. + Parse and load keyboard from the supplied path and a pointer to the loaded keyboard -into the out paramter. +into the out parameter. ## Specification ```c */ +// TODO-web-core: Deprecate this function (#12497) +// KMN_DEPRECATED_API KMN_API km_core_status km_core_keyboard_load(km_core_path_name kb_path, @@ -1140,6 +1148,60 @@ km_core_keyboard_load(km_core_path_name kb_path, ------------------------------------------------------------------------------- +# km_core_keyboard_load_from_blob() + +## Description + +Parse and load keyboard from the supplied blob and a pointer to the loaded keyboard +into the out paramter. + +## Specification + +```c */ +KMN_API +km_core_status km_core_keyboard_load_from_blob(const km_core_path_name kb_name, + const void* blob, + const size_t blob_size, + km_core_keyboard** keyboard); + +/* +``` + +## Parameters + +`kb_name` +: a string with the name of the keyboard. + +`blob` +: a byte array containing the content of a KMX/KMX+ file. + +`blob_size` +: a size_t variable with the size of the blob in bytes. + +`keyboard` +: A pointer to result variable: A pointer to the opaque keyboard + object returned by the Processor. This memory must be freed with a + call to [km_core_keyboard_dispose]. + +## Returns + +`KM_CORE_STATUS_OK` +: On success. + +`KM_CORE_STATUS_NO_MEM` +: In the event an internal memory allocation fails. + +`KM_CORE_STATUS_IO_ERROR` +: In the event the keyboard file is unparseable for any reason + +`KM_CORE_STATUS_INVALID_ARGUMENT` +: In the event `keyboard` is null. + +`KM_CORE_STATUS_OS_ERROR` +: Bit 31 (high bit) set, bits 0-30 are an OS-specific error code. + +------------------------------------------------------------------------------- + # km_core_keyboard_dispose() ## Description diff --git a/core/include/keyman/keyman_core_api_bits.h b/core/include/keyman/keyman_core_api_bits.h index e00f4698a8c..bd1f519bffe 100644 --- a/core/include/keyman/keyman_core_api_bits.h +++ b/core/include/keyman/keyman_core_api_bits.h @@ -23,7 +23,6 @@ #define _kmn_unused(x) UNUSED_ ## x __attribute__((__unused__)) #else #define _kmn_unused(x) UNUSED_ ## x - #endif #if defined _WIN32 || defined __CYGWIN__ @@ -36,7 +35,7 @@ #undef _kmn_static_flag #else // How MSVC sepcifies function level attributes adn deprecation #define _kmn_and - #define _kmn_tag_fn(a) __declspec(a) + #define _kmn_tag_fn(a) __declspec(a) #define _kmn_deprecated_flag deprecated #endif #define _kmn_export_flag dllexport @@ -48,6 +47,8 @@ #define _KM_CORE_EXT_SEPARATOR ('.') #endif +#define KMN_DEPRECATED _kmn_tag_fn(_kmn_deprecated_flag) + #if defined KM_CORE_LIBRARY_STATIC #define KMN_API _kmn_tag_fn(_kmn_static_flag) #define KMN_DEPRECATED_API _kmn_tag_fn(_kmn_deprecated_flag _kmn_and _kmn_static_flag) diff --git a/core/src/action.cpp b/core/src/action.cpp index 5f591444091..1ff7917575c 100644 --- a/core/src/action.cpp +++ b/core/src/action.cpp @@ -61,7 +61,9 @@ bool km::core::action_item_list_to_actions_object( if(output.empty()) { actions->code_points_to_delete++; } else { +#ifndef NDEBUG auto last_context_item = output.back(); +#endif output.pop_back(); assert(last_context_item.type == KM_CORE_CT_CHAR); assert(last_context_item.character == action_items->backspace.expected_value); @@ -71,7 +73,9 @@ bool km::core::action_item_list_to_actions_object( if(output.empty()) { // deleting a marker has no effect on the application } else { +#ifndef NDEBUG auto last_context_item = output.back(); +#endif output.pop_back(); assert(last_context_item.type == KM_CORE_CT_MARKER); assert(last_context_item.marker == action_items->backspace.expected_value); diff --git a/core/src/keyboard.cpp b/core/src/keyboard.cpp index 2c3a0c11b2c..0f104f9422a 100644 --- a/core/src/keyboard.cpp +++ b/core/src/keyboard.cpp @@ -17,18 +17,16 @@ void keyboard_attributes::render() // Make attributes point to the stored values above. id = _keyboard_id.c_str(); version_string = _version_string.c_str(); - folder_path = _folder_path.c_str(); default_options = _default_opts.data(); } keyboard_attributes::keyboard_attributes(std::u16string const & kbid, std::u16string const & version, - path_type const & path, options_store const &opts) : _keyboard_id(kbid), _version_string(version), - _folder_path(path), + _folder_path(""), _default_opts(opts) { // Ensure that the default_options array will be properly terminated. @@ -40,7 +38,7 @@ keyboard_attributes::keyboard_attributes(std::u16string const & kbid, keyboard_attributes::keyboard_attributes(keyboard_attributes &&rhs) : _keyboard_id(std::move(rhs._keyboard_id)), _version_string(std::move(rhs._version_string)), - _folder_path(std::move(rhs._folder_path)), + _folder_path(""), _default_opts(std::move(rhs._default_opts)) { rhs.id = rhs.version_string = nullptr; @@ -58,7 +56,6 @@ json & km::core::operator << (json & j, km::core::keyboard_attributes const & kb { j << json::object << "id" << kb.id - << "folder" << kb._folder_path << "version" << kb.version_string << "rules" << json::array << json::close; diff --git a/core/src/keyboard.hpp b/core/src/keyboard.hpp index 142bf40e860..2ca7118d870 100644 --- a/core/src/keyboard.hpp +++ b/core/src/keyboard.hpp @@ -26,6 +26,7 @@ namespace core { std::u16string _keyboard_id; std::u16string _version_string; + // unused and deprecated core::path _folder_path; std::vector