From 3e168aafb232615f29ca7b1b88d04eb13dc35130 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Oct 2024 08:34:24 -0500 Subject: [PATCH 1/3] chore(developer,common): deps: xml2js / fast-xml-parser - remove vendored xml2js - add fast-xml-parser@4.5.0 - Rewire the KeymanXMLReader / KeymanXMLWriter to use fast-xml-parser Four small changes to the test data on the generated .kvks: - allow " for XML generation (" is legal) - use the same XML prologue as the actual .kvks files. - treat as since we can't distinguish (removed a "_": "" entry) - remove whitespace because we now can ("_": "\n \n \n") on non-leaf nodes. This is ignored by the kvks code, but was difficult to maintain roundtrip. Fixes: #12208 --- developer/src/common/web/utils/package.json | 6 +- .../common/web/utils/src/deps/xml2js/LICENSE | 19 - .../web/utils/src/deps/xml2js/README.md | 507 ------------------ .../common/web/utils/src/deps/xml2js/bom.js | 8 - .../web/utils/src/deps/xml2js/builder.js | 118 ---- .../web/utils/src/deps/xml2js/defaults.js | 69 --- .../web/utils/src/deps/xml2js/parser.js | 381 ------------- .../web/utils/src/deps/xml2js/processors.js | 31 -- .../web/utils/src/deps/xml2js/xml2js.js | 27 - .../ldml-keyboard/ldml-keyboard-xml-reader.ts | 8 + .../src/common/web/utils/src/xml-utils.ts | 264 +++++++-- .../test/fixtures/xml/khmer_angkor.kvks.json | 19 +- .../test/fixtures/xml/khmer_angkor2.kvks | 6 +- .../utils/test/kpj/test-kpj-file-reader.ts | 4 +- developer/src/common/web/utils/tsconfig.json | 3 +- package-lock.json | 38 +- 16 files changed, 270 insertions(+), 1238 deletions(-) delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/LICENSE delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/README.md delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/bom.js delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/builder.js delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/defaults.js delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/parser.js delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/processors.js delete mode 100644 developer/src/common/web/utils/src/deps/xml2js/xml2js.js diff --git a/developer/src/common/web/utils/package.json b/developer/src/common/web/utils/package.json index 1ab9daddbf8..7f2d9cef279 100644 --- a/developer/src/common/web/utils/package.json +++ b/developer/src/common/web/utils/package.json @@ -9,19 +9,19 @@ "/build/" ], "dependencies": { - "@sentry/node": "^7.57.0", "@keymanapp/common-types": "*", + "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", + "fast-xml-parser": "^4.5.0", "restructure": "^3.0.1", - "semver": "^7.5.4", "sax": ">=0.6.0", + "semver": "^7.5.4", "xmlbuilder": "~11.0.0" }, "devDependencies": { "@types/git-diff": "^2.0.3", "@types/node": "^20.4.1", "@types/semver": "^7.3.12", - "@types/xml2js": "^0.4.5", "c8": "^7.12.0", "git-diff": "^2.0.6", "mocha": "^8.4.0", diff --git a/developer/src/common/web/utils/src/deps/xml2js/LICENSE b/developer/src/common/web/utils/src/deps/xml2js/LICENSE deleted file mode 100644 index e3b4222a66a..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/LICENSE +++ /dev/null @@ -1,19 +0,0 @@ -Copyright 2010, 2011, 2012, 2013. All rights reserved. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. diff --git a/developer/src/common/web/utils/src/deps/xml2js/README.md b/developer/src/common/web/utils/src/deps/xml2js/README.md deleted file mode 100644 index 67f2104a513..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/README.md +++ /dev/null @@ -1,507 +0,0 @@ -node-xml2js -=========== - -Ever had the urge to parse XML? And wanted to access the data in some sane, -easy way? Don't want to compile a C parser, for whatever reason? Then xml2js is -what you're looking for! - -Description -=========== - -Simple XML to JavaScript object converter. It supports bi-directional conversion. -Uses [sax-js](https://github.com/isaacs/sax-js/) and -[xmlbuilder-js](https://github.com/oozcitak/xmlbuilder-js/). - -Note: If you're looking for a full DOM parser, you probably want -[JSDom](https://github.com/tmpvar/jsdom). - -Installation -============ - -Simplest way to install `xml2js` is to use [npm](http://npmjs.org), just `npm -install xml2js` which will download xml2js and all dependencies. - -xml2js is also available via [Bower](http://bower.io/), just `bower install -xml2js` which will download xml2js and all dependencies. - -Usage -===== - -No extensive tutorials required because you are a smart developer! The task of -parsing XML should be an easy one, so let's make it so! Here's some examples. - -Shoot-and-forget usage ----------------------- - -You want to parse XML as simple and easy as possible? It's dangerous to go -alone, take this: - -```javascript -var parseString = require('xml2js').parseString; -var xml = "Hello xml2js!" -parseString(xml, function (err, result) { - console.dir(result); -}); -``` - -Can't get easier than this, right? This works starting with `xml2js` 0.2.3. -With CoffeeScript it looks like this: - -```coffeescript -{parseString} = require 'xml2js' -xml = "Hello xml2js!" -parseString xml, (err, result) -> - console.dir result -``` - -If you need some special options, fear not, `xml2js` supports a number of -options (see below), you can specify these as second argument: - -```javascript -parseString(xml, {trim: true}, function (err, result) { -}); -``` - -Simple as pie usage -------------------- - -That's right, if you have been using xml-simple or a home-grown -wrapper, this was added in 0.1.11 just for you: - -```javascript -var fs = require('fs'), - xml2js = require('xml2js'); - -var parser = new xml2js.Parser(); -fs.readFile(__dirname + '/foo.xml', function(err, data) { - parser.parseString(data, function (err, result) { - console.dir(result); - console.log('Done'); - }); -}); -``` - -Look ma, no event listeners! - -You can also use `xml2js` from -[CoffeeScript](https://github.com/jashkenas/coffeescript), further reducing -the clutter: - -```coffeescript -fs = require 'fs', -xml2js = require 'xml2js' - -parser = new xml2js.Parser() -fs.readFile __dirname + '/foo.xml', (err, data) -> - parser.parseString data, (err, result) -> - console.dir result - console.log 'Done.' -``` - -But what happens if you forget the `new` keyword to create a new `Parser`? In -the middle of a nightly coding session, it might get lost, after all. Worry -not, we got you covered! Starting with 0.2.8 you can also leave it out, in -which case `xml2js` will helpfully add it for you, no bad surprises and -inexplicable bugs! - -Promise usage -------------- - -```javascript -var xml2js = require('xml2js'); -var xml = ''; - -// With parser -var parser = new xml2js.Parser(/* options */); -parser.parseStringPromise(xml).then(function (result) { - console.dir(result); - console.log('Done'); -}) -.catch(function (err) { - // Failed -}); - -// Without parser -xml2js.parseStringPromise(xml /*, options */).then(function (result) { - console.dir(result); - console.log('Done'); -}) -.catch(function (err) { - // Failed -}); -``` - -Parsing multiple files ----------------------- - -If you want to parse multiple files, you have multiple possibilities: - - * You can create one `xml2js.Parser` per file. That's the recommended one - and is promised to always *just work*. - * You can call `reset()` on your parser object. - * You can hope everything goes well anyway. This behaviour is not - guaranteed work always, if ever. Use option #1 if possible. Thanks! - -So you wanna some JSON? ------------------------ - -Just wrap the `result` object in a call to `JSON.stringify` like this -`JSON.stringify(result)`. You get a string containing the JSON representation -of the parsed object that you can feed to JSON-hungry consumers. - -Displaying results ------------------- - -You might wonder why, using `console.dir` or `console.log` the output at some -level is only `[Object]`. Don't worry, this is not because `xml2js` got lazy. -That's because Node uses `util.inspect` to convert the object into strings and -that function stops after `depth=2` which is a bit low for most XML. - -To display the whole deal, you can use `console.log(util.inspect(result, false, -null))`, which displays the whole result. - -So much for that, but what if you use -[eyes](https://github.com/cloudhead/eyes.js) for nice colored output and it -truncates the output with `…`? Don't fear, there's also a solution for that, -you just need to increase the `maxLength` limit by creating a custom inspector -`var inspect = require('eyes').inspector({maxLength: false})` and then you can -easily `inspect(result)`. - -XML builder usage ------------------ - -Since 0.4.0, objects can be also be used to build XML: - -```javascript -var xml2js = require('xml2js'); - -var obj = {name: "Super", Surname: "Man", age: 23}; - -var builder = new xml2js.Builder(); -var xml = builder.buildObject(obj); -``` -will result in: - -```xml - - - Super - Man - 23 - -``` - -At the moment, a one to one bi-directional conversion is guaranteed only for -default configuration, except for `attrkey`, `charkey` and `explicitArray` options -you can redefine to your taste. Writing CDATA is supported via setting the `cdata` -option to `true`. - -To specify attributes: -```javascript -var xml2js = require('xml2js'); - -var obj = {root: {$: {id: "my id"}, _: "my inner text"}}; - -var builder = new xml2js.Builder(); -var xml = builder.buildObject(obj); -``` -will result in: -```xml - -my inner text -``` - -### Adding xmlns attributes - -You can generate XML that declares XML namespace prefix / URI pairs with xmlns attributes. - -Example declaring a default namespace on the root element: - -```javascript -let obj = { - Foo: { - $: { - "xmlns": "http://foo.com" - } - } -}; -``` -Result of `buildObject(obj)`: -```xml - -``` -Example declaring non-default namespaces on non-root elements: -```javascript -let obj = { - 'foo:Foo': { - $: { - 'xmlns:foo': 'http://foo.com' - }, - 'bar:Bar': { - $: { - 'xmlns:bar': 'http://bar.com' - } - } - } -} -``` -Result of `buildObject(obj)`: -```xml - - - -``` - - -Processing attribute, tag names and values ------------------------------------------- - -Since 0.4.1 you can optionally provide the parser with attribute name and tag name processors as well as element value processors (Since 0.4.14, you can also optionally provide the parser with attribute value processors): - -```javascript - -function nameToUpperCase(name){ - return name.toUpperCase(); -} - -//transform all attribute and tag names and values to uppercase -parseString(xml, { - tagNameProcessors: [nameToUpperCase], - attrNameProcessors: [nameToUpperCase], - valueProcessors: [nameToUpperCase], - attrValueProcessors: [nameToUpperCase]}, - function (err, result) { - // processed data -}); -``` - -The `tagNameProcessors` and `attrNameProcessors` options -accept an `Array` of functions with the following signature: - -```javascript -function (name){ - //do something with `name` - return name -} -``` - -The `attrValueProcessors` and `valueProcessors` options -accept an `Array` of functions with the following signature: - -```javascript -function (value, name) { - //`name` will be the node name or attribute name - //do something with `value`, (optionally) dependent on the node/attr name - return value -} -``` - -Some processors are provided out-of-the-box and can be found in `lib/processors.js`: - -- `normalize`: transforms the name to lowercase. -(Automatically used when `options.normalize` is set to `true`) - -- `firstCharLowerCase`: transforms the first character to lower case. -E.g. 'MyTagName' becomes 'myTagName' - -- `stripPrefix`: strips the xml namespace prefix. E.g `` will become 'Bar'. -(N.B.: the `xmlns` prefix is NOT stripped.) - -- `parseNumbers`: parses integer-like strings as integers and float-like strings as floats -E.g. "0" becomes 0 and "15.56" becomes 15.56 - -- `parseBooleans`: parses boolean-like strings to booleans -E.g. "true" becomes true and "False" becomes false - -Options -======= - -Apart from the default settings, there are a number of options that can be -specified for the parser. Options are specified by ``new Parser({optionName: -value})``. Possible options are: - - * `attrkey` (default: `$`): Prefix that is used to access the attributes. - Version 0.1 default was `@`. - * `charkey` (default: `_`): Prefix that is used to access the character - content. Version 0.1 default was `#`. - * `explicitCharkey` (default: `false`) Determines whether or not to use - a `charkey` prefix for elements with no attributes. - * `trim` (default: `false`): Trim the whitespace at the beginning and end of - text nodes. - * `normalizeTags` (default: `false`): Normalize all tag names to lowercase. - * `normalize` (default: `false`): Trim whitespaces inside text nodes. - * `explicitRoot` (default: `true`): Set this if you want to get the root - node in the resulting object. - * `emptyTag` (default: `''`): what will the value of empty nodes be. In case - you want to use an empty object as a default value, it is better to provide a factory - function `() => ({})` instead. Without this function a plain object would - become a shared reference across all occurrences with unwanted behavior. - * `explicitArray` (default: `true`): Always put child nodes in an array if - true; otherwise an array is created only if there is more than one. - * `ignoreAttrs` (default: `false`): Ignore all XML attributes and only create - text nodes. - * `mergeAttrs` (default: `false`): Merge attributes and child elements as - properties of the parent, instead of keying attributes off a child - attribute object. This option is ignored if `ignoreAttrs` is `true`. - * `validator` (default `null`): You can specify a callable that validates - the resulting structure somehow, however you want. See unit tests - for an example. - * `xmlns` (default `false`): Give each element a field usually called '$ns' - (the first character is the same as attrkey) that contains its local name - and namespace URI. - * `explicitChildren` (default `false`): Put child elements to separate - property. Doesn't work with `mergeAttrs = true`. If element has no children - then "children" won't be created. Added in 0.2.5. - * `childkey` (default `$$`): Prefix that is used to access child elements if - `explicitChildren` is set to `true`. Added in 0.2.5. - * `preserveChildrenOrder` (default `false`): Modifies the behavior of - `explicitChildren` so that the value of the "children" property becomes an - ordered array. When this is `true`, every node will also get a `#name` field - whose value will correspond to the XML nodeName, so that you may iterate - the "children" array and still be able to determine node names. The named - (and potentially unordered) properties are also retained in this - configuration at the same level as the ordered "children" array. Added in - 0.4.9. - * `charsAsChildren` (default `false`): Determines whether chars should be - considered children if `explicitChildren` is on. Added in 0.2.5. - * `includeWhiteChars` (default `false`): Determines whether whitespace-only - text nodes should be included. Added in 0.4.17. - * `async` (default `false`): Should the callbacks be async? This *might* be - an incompatible change if your code depends on sync execution of callbacks. - Future versions of `xml2js` might change this default, so the recommendation - is to not depend on sync execution anyway. Added in 0.2.6. - * `strict` (default `true`): Set sax-js to strict or non-strict parsing mode. - Defaults to `true` which is *highly* recommended, since parsing HTML which - is not well-formed XML might yield just about anything. Added in 0.2.7. - * `attrNameProcessors` (default: `null`): Allows the addition of attribute - name processing functions. Accepts an `Array` of functions with following - signature: - ```javascript - function (name){ - //do something with `name` - return name - } - ``` - Added in 0.4.14 - * `attrValueProcessors` (default: `null`): Allows the addition of attribute - value processing functions. Accepts an `Array` of functions with following - signature: - ```javascript - function (value, name){ - //do something with `name` - return name - } - ``` - Added in 0.4.1 - * `tagNameProcessors` (default: `null`): Allows the addition of tag name - processing functions. Accepts an `Array` of functions with following - signature: - ```javascript - function (name){ - //do something with `name` - return name - } - ``` - Added in 0.4.1 - * `valueProcessors` (default: `null`): Allows the addition of element value - processing functions. Accepts an `Array` of functions with following - signature: - ```javascript - function (value, name){ - //do something with `name` - return name - } - ``` - Added in 0.4.6 - -Options for the `Builder` class -------------------------------- -These options are specified by ``new Builder({optionName: value})``. -Possible options are: - - * `attrkey` (default: `$`): Prefix that is used to access the attributes. - Version 0.1 default was `@`. - * `charkey` (default: `_`): Prefix that is used to access the character - content. Version 0.1 default was `#`. - * `rootName` (default `root` or the root key name): root element name to be used in case - `explicitRoot` is `false` or to override the root element name. - * `renderOpts` (default `{ 'pretty': true, 'indent': ' ', 'newline': '\n' }`): - Rendering options for xmlbuilder-js. - * pretty: prettify generated XML - * indent: whitespace for indentation (only when pretty) - * newline: newline char (only when pretty) - * `xmldec` (default `{ 'version': '1.0', 'encoding': 'UTF-8', 'standalone': true }`: - XML declaration attributes. - * `xmldec.version` A version number string, e.g. 1.0 - * `xmldec.encoding` Encoding declaration, e.g. UTF-8 - * `xmldec.standalone` standalone document declaration: true or false - * `doctype` (default `null`): optional DTD. Eg. `{'ext': 'hello.dtd'}` - * `headless` (default: `false`): omit the XML header. Added in 0.4.3. - * `allowSurrogateChars` (default: `false`): allows using characters from the Unicode - surrogate blocks. - * `cdata` (default: `false`): wrap text nodes in `` instead of - escaping when necessary. Does not add `` if it is not required. - Added in 0.4.5. - -`renderOpts`, `xmldec`,`doctype` and `headless` pass through to -[xmlbuilder-js](https://github.com/oozcitak/xmlbuilder-js). - -Updating to new version -======================= - -Version 0.2 changed the default parsing settings, but version 0.1.14 introduced -the default settings for version 0.2, so these settings can be tried before the -migration. - -```javascript -var xml2js = require('xml2js'); -var parser = new xml2js.Parser(xml2js.defaults["0.2"]); -``` - -To get the 0.1 defaults in version 0.2 you can just use -`xml2js.defaults["0.1"]` in the same place. This provides you with enough time -to migrate to the saner way of parsing in `xml2js` 0.2. We try to make the -migration as simple and gentle as possible, but some breakage cannot be -avoided. - -So, what exactly did change and why? In 0.2 we changed some defaults to parse -the XML in a more universal and sane way. So we disabled `normalize` and `trim` -so `xml2js` does not cut out any text content. You can reenable this at will of -course. A more important change is that we return the root tag in the resulting -JavaScript structure via the `explicitRoot` setting, so you need to access the -first element. This is useful for anybody who wants to know what the root node -is and preserves more information. The last major change was to enable -`explicitArray`, so everytime it is possible that one might embed more than one -sub-tag into a tag, xml2js >= 0.2 returns an array even if the array just -includes one element. This is useful when dealing with APIs that return -variable amounts of subtags. - -Running tests, development -========================== - -[![Build Status](https://travis-ci.org/Leonidas-from-XIV/node-xml2js.svg?branch=master)](https://travis-ci.org/Leonidas-from-XIV/node-xml2js) -[![Coverage Status](https://coveralls.io/repos/Leonidas-from-XIV/node-xml2js/badge.svg?branch=)](https://coveralls.io/r/Leonidas-from-XIV/node-xml2js?branch=master) -[![Dependency Status](https://david-dm.org/Leonidas-from-XIV/node-xml2js.svg)](https://david-dm.org/Leonidas-from-XIV/node-xml2js) - -The development requirements are handled by npm, you just need to install them. -We also have a number of unit tests, they can be run using `npm test` directly -from the project root. This runs zap to discover all the tests and execute -them. - -If you like to contribute, keep in mind that `xml2js` is written in -CoffeeScript, so don't develop on the JavaScript files that are checked into -the repository for convenience reasons. Also, please write some unit test to -check your behaviour and if it is some user-facing thing, add some -documentation to this README, so people will know it exists. Thanks in advance! - -Getting support -=============== - -Please, if you have a problem with the library, first make sure you read this -README. If you read this far, thanks, you're good. Then, please make sure your -problem really is with `xml2js`. It is? Okay, then I'll look at it. Send me a -mail and we can talk. Please don't open issues, as I don't think that is the -proper forum for support problems. Some problems might as well really be bugs -in `xml2js`, if so I'll let you know to open an issue instead :) - -But if you know you really found a bug, feel free to open an issue instead. diff --git a/developer/src/common/web/utils/src/deps/xml2js/bom.js b/developer/src/common/web/utils/src/deps/xml2js/bom.js deleted file mode 100644 index 0ad6e2a4a43..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/bom.js +++ /dev/null @@ -1,8 +0,0 @@ -"use strict"; -export function stripBOM(str) { - if (str[0] === '\uFEFF') { - return str.substring(1); - } else { - return str; - } -}; diff --git a/developer/src/common/web/utils/src/deps/xml2js/builder.js b/developer/src/common/web/utils/src/deps/xml2js/builder.js deleted file mode 100644 index 5dfca61c9f6..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/builder.js +++ /dev/null @@ -1,118 +0,0 @@ -var escapeCDATA, requiresCDATA, wrapCDATA, - hasProp = {}.hasOwnProperty; - -import * as builder from 'xmlbuilder'; -import { defaults } from './defaults.js'; - -requiresCDATA = function(entry) { - return typeof entry === "string" && (entry.indexOf('&') >= 0 || entry.indexOf('>') >= 0 || entry.indexOf('<') >= 0); -}; - -wrapCDATA = function(entry) { - return ""; -}; - -escapeCDATA = function(entry) { - return entry.replace(']]>', ']]]]>'); -}; - -export class Builder { - constructor(opts) { - var key, ref, value; - this.options = {}; - ref = defaults["0.2"]; - for (key in ref) { - if (!hasProp.call(ref, key)) continue; - value = ref[key]; - this.options[key] = value; - } - for (key in opts) { - if (!hasProp.call(opts, key)) continue; - value = opts[key]; - this.options[key] = value; - } - } - - buildObject(rootObj) { - var attrkey, charkey, render, rootElement, rootName; - attrkey = this.options.attrkey; - charkey = this.options.charkey; - if ((Object.keys(rootObj).length === 1) && (this.options.rootName === defaults['0.2'].rootName)) { - rootName = Object.keys(rootObj)[0]; - rootObj = rootObj[rootName]; - } else { - rootName = this.options.rootName; - } - render = (function(_this) { - return function(element, obj) { - var attr, child, entry, index, key, value; - if (typeof obj !== 'object') { - if (_this.options.cdata && requiresCDATA(obj)) { - element.raw(wrapCDATA(obj)); - } else { - element.txt(obj); - } - } else if (Array.isArray(obj)) { - for (index in obj) { - if (!hasProp.call(obj, index)) continue; - child = obj[index]; - for (key in child) { - entry = child[key]; - element = render(element.ele(key), entry).up(); - } - } - } else { - for (key in obj) { - if (!hasProp.call(obj, key)) continue; - child = obj[key]; - if (key === attrkey) { - if (typeof child === "object") { - for (attr in child) { - value = child[attr]; - element = element.att(attr, value); - } - } - } else if (key === charkey) { - if (_this.options.cdata && requiresCDATA(child)) { - element = element.raw(wrapCDATA(child)); - } else { - element = element.txt(child); - } - } else if (Array.isArray(child)) { - for (index in child) { - if (!hasProp.call(child, index)) continue; - entry = child[index]; - if (typeof entry === 'string') { - if (_this.options.cdata && requiresCDATA(entry)) { - element = element.ele(key).raw(wrapCDATA(entry)).up(); - } else { - element = element.ele(key, entry).up(); - } - } else { - element = render(element.ele(key), entry).up(); - } - } - } else if (typeof child === "object") { - element = render(element.ele(key), child).up(); - } else { - if (typeof child === 'string' && _this.options.cdata && requiresCDATA(child)) { - element = element.ele(key).raw(wrapCDATA(child)).up(); - } else { - if (child == null) { - child = ''; - } - element = element.ele(key, child.toString()).up(); - } - } - } - } - return element; - }; - })(this); - rootElement = builder.create(rootName, this.options.xmldec, this.options.doctype, { - headless: this.options.headless, - allowSurrogateChars: this.options.allowSurrogateChars - }); - return render(rootElement, rootObj).end(this.options.renderOpts); - }; -} diff --git a/developer/src/common/web/utils/src/deps/xml2js/defaults.js b/developer/src/common/web/utils/src/deps/xml2js/defaults.js deleted file mode 100644 index d1009281792..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/defaults.js +++ /dev/null @@ -1,69 +0,0 @@ -// Generated by CoffeeScript 1.12.7 -export const defaults = { - "0.1": { - explicitCharkey: false, - trim: true, - normalize: true, - normalizeTags: false, - attrkey: "@", - charkey: "#", - explicitArray: false, - ignoreAttrs: false, - mergeAttrs: false, - explicitRoot: false, - validator: null, - xmlns: false, - explicitChildren: false, - childkey: '@@', - charsAsChildren: false, - includeWhiteChars: false, - async: false, - strict: true, - attrNameProcessors: null, - attrValueProcessors: null, - tagNameProcessors: null, - valueProcessors: null, - emptyTag: '' - }, - "0.2": { - explicitCharkey: false, - trim: false, - normalize: false, - normalizeTags: false, - attrkey: "$", - charkey: "_", - explicitArray: true, - ignoreAttrs: false, - mergeAttrs: false, - explicitRoot: true, - validator: null, - xmlns: false, - explicitChildren: false, - preserveChildrenOrder: false, - childkey: '$$', - charsAsChildren: false, - includeWhiteChars: false, - async: false, - strict: true, - attrNameProcessors: null, - attrValueProcessors: null, - tagNameProcessors: null, - valueProcessors: null, - rootName: 'root', - xmldec: { - 'version': '1.0', - 'encoding': 'UTF-8', - 'standalone': true - }, - doctype: null, - renderOpts: { - 'pretty': true, - 'indent': ' ', - 'newline': '\n' - }, - headless: false, - chunkSize: 10000, - emptyTag: '', - cdata: false - } - }; diff --git a/developer/src/common/web/utils/src/deps/xml2js/parser.js b/developer/src/common/web/utils/src/deps/xml2js/parser.js deleted file mode 100644 index 1c7f148d906..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/parser.js +++ /dev/null @@ -1,381 +0,0 @@ - var isEmpty, processItem, - bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }, - extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; }, - hasProp = {}.hasOwnProperty; - -import sax from 'sax'; -import { EventEmitter } from 'eventemitter3'; -import * as bom from './bom.js'; -import * as processors from './processors.js'; -import { setImmediate } from 'timers'; -import { defaults } from './defaults.js'; - - isEmpty = function(thing) { - return typeof thing === "object" && (thing != null) && Object.keys(thing).length === 0; - }; - - processItem = function(processors, item, key) { - var i, len, process; - for (i = 0, len = processors.length; i < len; i++) { - process = processors[i]; - item = process(item, key); - } - return item; - }; - -/** @type Class */ -export class Parser extends EventEmitter { -// export const Parser = (function(superClass) { - // extend(Parser, superClass); - - constructor(opts) { - super(); - this.parseStringPromise = bind(this.parseStringPromise, this); - this.parseString = bind(this.parseString, this); - this.reset = bind(this.reset, this); - this.assignOrPush = bind(this.assignOrPush, this); - this.processAsync = bind(this.processAsync, this); - var key, ref, value; - if (!(this instanceof Parser)) { - return new Parser(opts); - } - this.options = {}; - ref = defaults["0.2"]; - for (key in ref) { - if (!hasProp.call(ref, key)) continue; - value = ref[key]; - this.options[key] = value; - } - for (key in opts) { - if (!hasProp.call(opts, key)) continue; - value = opts[key]; - this.options[key] = value; - } - if (this.options.xmlns) { - this.options.xmlnskey = this.options.attrkey + "ns"; - } - if (this.options.normalizeTags) { - if (!this.options.tagNameProcessors) { - this.options.tagNameProcessors = []; - } - this.options.tagNameProcessors.unshift(processors.normalize); - } - this.reset(); - } - - processAsync() { - var chunk, err; - try { - if (this.remaining.length <= this.options.chunkSize) { - chunk = this.remaining; - this.remaining = ''; - this.saxParser = this.saxParser.write(chunk); - return this.saxParser.close(); - } else { - chunk = this.remaining.substr(0, this.options.chunkSize); - this.remaining = this.remaining.substr(this.options.chunkSize, this.remaining.length); - this.saxParser = this.saxParser.write(chunk); - return setImmediate(this.processAsync); - } - } catch (error1) { - err = error1; - if (!this.saxParser.errThrown) { - this.saxParser.errThrown = true; - return this.emit(err); - } - } - }; - - assignOrPush(obj, key, newValue) { - if (!(key in obj)) { - if (!this.options.explicitArray) { - return obj[key] = newValue; - } else { - return obj[key] = [newValue]; - } - } else { - if (!(obj[key] instanceof Array)) { - obj[key] = [obj[key]]; - } - return obj[key].push(newValue); - } - }; - - reset() { - var attrkey, charkey, ontext, stack; - this.removeAllListeners(); - this.saxParser = sax.parser(this.options.strict, { - trim: false, - normalize: false, - xmlns: this.options.xmlns - }); - this.saxParser.errThrown = false; - this.saxParser.onerror = (function(_this) { - return function(error) { - _this.saxParser.resume(); - if (!_this.saxParser.errThrown) { - _this.saxParser.errThrown = true; - return _this.emit("error", error); - } - }; - })(this); - this.saxParser.onend = (function(_this) { - return function() { - if (!_this.saxParser.ended) { - _this.saxParser.ended = true; - return _this.emit("end", _this.resultObject); - } - }; - })(this); - this.saxParser.ended = false; - this.EXPLICIT_CHARKEY = this.options.explicitCharkey; - this.resultObject = null; - stack = []; - attrkey = this.options.attrkey; - charkey = this.options.charkey; - this.saxParser.onopentag = (function(_this) { - return function(node) { - var key, newValue, obj, processedKey, ref; - obj = Object.create(null); - obj[charkey] = ""; - if (!_this.options.ignoreAttrs) { - ref = node.attributes; - for (key in ref) { - if (!hasProp.call(ref, key)) continue; - if (!(attrkey in obj) && !_this.options.mergeAttrs) { - obj[attrkey] = Object.create(null); - } - newValue = _this.options.attrValueProcessors ? processItem(_this.options.attrValueProcessors, node.attributes[key], key) : node.attributes[key]; - processedKey = _this.options.attrNameProcessors ? processItem(_this.options.attrNameProcessors, key) : key; - if (_this.options.mergeAttrs) { - _this.assignOrPush(obj, processedKey, newValue); - } else { - obj[attrkey][processedKey] = newValue; - } - } - } - obj["#name"] = _this.options.tagNameProcessors ? processItem(_this.options.tagNameProcessors, node.name) : node.name; - if (_this.options.xmlns) { - obj[_this.options.xmlnskey] = { - uri: node.uri, - local: node.local - }; - } - return stack.push(obj); - }; - })(this); - this.saxParser.onclosetag = (function(_this) { - return function() { - var cdata, emptyStr, key, node, nodeName, obj, objClone, old, s, xpath; - obj = stack.pop(); - nodeName = obj["#name"]; - if (!_this.options.explicitChildren || !_this.options.preserveChildrenOrder) { - delete obj["#name"]; - } - if (obj.cdata === true) { - cdata = obj.cdata; - delete obj.cdata; - } - s = stack[stack.length - 1]; - if (obj[charkey].match(/^\s*$/) && !cdata && !_this.options.includeWhiteChars) { - emptyStr = obj[charkey]; - delete obj[charkey]; - } else { - if (_this.options.trim) { - obj[charkey] = obj[charkey].trim(); - } - if (_this.options.normalize) { - obj[charkey] = obj[charkey].replace(/\s{2,}/g, " ").trim(); - } - obj[charkey] = _this.options.valueProcessors ? processItem(_this.options.valueProcessors, obj[charkey], nodeName) : obj[charkey]; - if (Object.keys(obj).length === 1 && charkey in obj && !_this.EXPLICIT_CHARKEY) { - obj = obj[charkey]; - } - } - if (isEmpty(obj)) { - if (typeof _this.options.emptyTag === 'function') { - obj = _this.options.emptyTag(); - } else { - obj = _this.options.emptyTag !== '' ? _this.options.emptyTag : emptyStr; - } - } - if (_this.options.validator != null) { - xpath = "/" + ((function() { - var i, len, results; - results = []; - for (i = 0, len = stack.length; i < len; i++) { - node = stack[i]; - results.push(node["#name"]); - } - return results; - })()).concat(nodeName).join("/"); - (function() { - var err; - try { - return obj = _this.options.validator(xpath, s && s[nodeName], obj); - } catch (error1) { - err = error1; - return _this.emit("error", err); - } - })(); - } - if (_this.options.explicitChildren && !_this.options.mergeAttrs && typeof obj === 'object') { - if (!_this.options.preserveChildrenOrder) { - node = Object.create(null); - if (_this.options.attrkey in obj) { - node[_this.options.attrkey] = obj[_this.options.attrkey]; - delete obj[_this.options.attrkey]; - } - if (!_this.options.charsAsChildren && _this.options.charkey in obj) { - node[_this.options.charkey] = obj[_this.options.charkey]; - delete obj[_this.options.charkey]; - } - if (Object.getOwnPropertyNames(obj).length > 0) { - node[_this.options.childkey] = obj; - } - obj = node; - } else if (s) { - s[_this.options.childkey] = s[_this.options.childkey] || []; - objClone = Object.create(null); - for (key in obj) { - if (!hasProp.call(obj, key)) continue; - objClone[key] = obj[key]; - } - s[_this.options.childkey].push(objClone); - delete obj["#name"]; - if (Object.keys(obj).length === 1 && charkey in obj && !_this.EXPLICIT_CHARKEY) { - obj = obj[charkey]; - } - } - } - if (stack.length > 0) { - return _this.assignOrPush(s, nodeName, obj); - } else { - if (_this.options.explicitRoot) { - old = obj; - obj = Object.create(null); - obj[nodeName] = old; - } - _this.resultObject = obj; - _this.saxParser.ended = true; - return _this.emit("end", _this.resultObject); - } - }; - })(this); - ontext = (function(_this) { - return function(text) { - var charChild, s; - s = stack[stack.length - 1]; - if (s) { - s[charkey] += text; - if (_this.options.explicitChildren && _this.options.preserveChildrenOrder && _this.options.charsAsChildren && (_this.options.includeWhiteChars || text.replace(/\\n/g, '').trim() !== '')) { - s[_this.options.childkey] = s[_this.options.childkey] || []; - charChild = { - '#name': '__text__' - }; - charChild[charkey] = text; - if (_this.options.normalize) { - charChild[charkey] = charChild[charkey].replace(/\s{2,}/g, " ").trim(); - } - s[_this.options.childkey].push(charChild); - } - return s; - } - }; - })(this); - this.saxParser.ontext = ontext; - return this.saxParser.oncdata = (function(_this) { - return function(text) { - var s; - s = ontext(text); - if (s) { - return s.cdata = true; - } - }; - })(this); - }; - - parseString(str, cb) { - var err; - if ((cb != null) && typeof cb === "function") { - this.on("end", function(result) { - this.reset(); - return cb(null, result); - }); - this.on("error", function(err) { - this.reset(); - return cb(err); - }); - } - try { - str = str.toString(); - if (str.trim() === '') { - this.emit("end", null); - return true; - } - str = bom.stripBOM(str); - if (this.options.async) { - this.remaining = str; - setImmediate(this.processAsync); - return this.saxParser; - } - return this.saxParser.write(str).close(); - } catch (error1) { - err = error1; - if (!(this.saxParser.errThrown || this.saxParser.ended)) { - if(this.listenerCount('error') > 0) { - this.emit('error', err); - } else { - throw err; - } - return this.saxParser.errThrown = true; - } else if (this.saxParser.ended) { - throw err; - } - } - }; - - parseStringPromise(str) { - return new Promise((function(_this) { - return function(resolve, reject) { - return _this.parseString(str, function(err, value) { - if (err) { - return reject(err); - } else { - return resolve(value); - } - }); - }; - })(this)); - }; - - } - - export const parseString = function(str, a, b) { - var cb, options, parser; - if (b != null) { - if (typeof b === 'function') { - cb = b; - } - if (typeof a === 'object') { - options = a; - } - } else { - if (typeof a === 'function') { - cb = a; - } - options = {}; - } - parser = new Parser(options); - return parser.parseString(str, cb); - }; - - export const parseStringPromise = function(str, a) { - var options, parser; - if (typeof a === 'object') { - options = a; - } - parser = new Parser(options); - return parser.parseStringPromise(str); - }; - diff --git a/developer/src/common/web/utils/src/deps/xml2js/processors.js b/developer/src/common/web/utils/src/deps/xml2js/processors.js deleted file mode 100644 index 2a6849088c1..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/processors.js +++ /dev/null @@ -1,31 +0,0 @@ - "use strict"; - var prefixMatch; - - prefixMatch = new RegExp(/(?!xmlns)^.*:/); - - export const normalize = function(str) { - return str.toLowerCase(); - }; - - export const firstCharLowerCase = function(str) { - return str.charAt(0).toLowerCase() + str.slice(1); - }; - - export const stripPrefix = function(str) { - return str.replace(prefixMatch, ''); - }; - - export const parseNumbers = function(str) { - if (!isNaN(str)) { - str = str % 1 === 0 ? parseInt(str, 10) : parseFloat(str); - } - return str; - }; - - export const parseBooleans = function(str) { - if (/^(?:true|false)$/i.test(str)) { - str = str.toLowerCase() === 'true'; - } - return str; - }; - diff --git a/developer/src/common/web/utils/src/deps/xml2js/xml2js.js b/developer/src/common/web/utils/src/deps/xml2js/xml2js.js deleted file mode 100644 index 8ecbe3bcf2f..00000000000 --- a/developer/src/common/web/utils/src/deps/xml2js/xml2js.js +++ /dev/null @@ -1,27 +0,0 @@ -var - extend = function(child, parent) { for (var key in parent) { if (hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; }, - hasProp = {}.hasOwnProperty; - - // import { defaults } from './defaults.js'; - import * as builder from './builder.js'; - import * as parser from './parser.js'; - import * as processors from './processors.js'; - - // export const defaults = defaults.defaults; - // export const processors = processors; - - /** @type Class */ - export class ValidationError extends Error { - constructor(message) { - super(message); - this.message = message; - } - }; - - export const Parser = parser.Parser; - - export const Builder = builder.Builder; - - export const parseString = parser.parseString; - - export const parseStringPromise = parser.parseStringPromise; diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index b0fe92be320..17ade173931 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -97,6 +97,14 @@ export class LDMLKeyboardXMLSourceFileReader { if(source?.keyboard3?.transforms) { for(const transforms of source.keyboard3.transforms) { boxXmlArray(transforms, 'transformGroup'); + // need to see if there's an empty ('') element. + // the schema allows an empty object, but the spec doesn't. + for (let i=0; i @@ -24,60 +24,81 @@ type KemanXMLOptionsBag = { /** map of options for the XML parser */ const PARSER_OPTIONS: KemanXMLOptionsBag = { 'keyboard3': { - explicitArray: false, - mergeAttrs: true, - includeWhiteChars: false, - emptyTag: {} as any - // Why "as any"? xml2js is broken: - // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means - // that an old version of `emptyTag` is used which doesn't support - // functions, but DefinitelyTyped is requiring use of function or a - // string. See also notes at - // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470 - // An alternative fix would be to pull xml2js directly from github - // rather than using the version tagged on npmjs.com. + ignoreAttributes: false, // We'd like attributes, please + attributeNamePrefix: '', // to avoid '@_' prefixes + trimValues: false, // preserve spaces, but: + htmlEntities: true, + tagValueProcessor: (tagName: string, tagValue: string /*, jPath, hasAttributes, isLeafNode*/) => { + // since trimValues: false, we need to zap any element values that would be trimmed. + // currently, the LDML spec doesn't have any element values, but this + // future-proofs us a little in that element values are allowed, just trimmed. + // if we do need elements in the future, we'd check the preserve-space attribute here. + return tagValue?.trim(); + }, }, 'keyboardTest3': { - preserveChildrenOrder: true, // needed for test data - explicitChildren: true, // needed for test data + ignorePiTags: true, + htmlEntities: true, + ignoreAttributes: false, // We'd like attributes, please + attributeNamePrefix: '', // avoid @_ + preserveOrder: true, // Gives us a 'special' format }, 'kps': { - explicitArray: false + ignorePiTags: true, + ignoreAttributes: false, + htmlEntities: true, + attributeNamePrefix: '$', // causes remapping into $: { … } objects + textNodeName: '_', + numberParseOptions: { + skipLike: /(?:)/, // parse numbers as strings + hex: null, + leadingZeros: null, + eNotation: null, + }, }, 'kpj': { - explicitArray: false, - mergeAttrs: false, - includeWhiteChars: false, - normalize: false, - emptyTag: '' + ignorePiTags: true, + textNodeName: '_', + htmlEntities: true, + ignoreAttributes: false, // We'd like attributes, please + attributeNamePrefix: '', // to avoid '@_' prefixes + numberParseOptions: { + skipLike: /(?:)/, // parse numbers as strings + hex: null, + leadingZeros: null, + eNotation: null, + }, }, 'kvks': { - explicitArray: false, - mergeAttrs: false, - includeWhiteChars: true, - normalize: false, - emptyTag: {} as any - // Why "as any"? xml2js is broken: - // https://github.com/Leonidas-from-XIV/node-xml2js/issues/648 means - // that an old version of `emptyTag` is used which doesn't support - // functions, but DefinitelyTyped is requiring use of function or a - // string. See also notes at - // https://github.com/DefinitelyTyped/DefinitelyTyped/pull/59259#issuecomment-1254405470 - // An alternative fix would be to pull xml2js directly from github - // rather than using the version tagged on npmjs.com. + ignorePiTags: true, + textNodeName: '_', + htmlEntities: true, + ignoreAttributes: false, // We'd like attributes, please + attributeNamePrefix: '$', // causes remapping into $: { … } objects + numberParseOptions: { + skipLike: /(?:)/, // parse numbers as strings + hex: null, + leadingZeros: null, + eNotation: null, + }, + trimValues: false, // preserve spaces, but: + tagValueProcessor: (tagName: string, tagValue: string, jPath: string, hasAttributes: string, isLeafNode: boolean) : string | undefined => { + if (!isLeafNode) { + return tagValue?.trim(); // trimmed value + } else { + return null; // no change to leaf nodes + } + }, }, }; const GENERATOR_OPTIONS: KemanXMLOptionsBag = { kvks: { - allowSurrogateChars: true, - attrkey: '$', - charkey: '_', - xmldec: { - version: '1.0', - encoding: 'UTF-8', - standalone: true - }, + attributeNamePrefix: '$', + ignoreAttributes: false, + format: true, + textNodeName: '_', + suppressEmptyNode: true, }, }; @@ -86,11 +107,125 @@ export class KeymanXMLReader { public constructor(public type: KeymanXMLType) { } + /** move `{ $abc: 4 }` into `{ $: { abc: 4 } }` */ + private static fixupDollarAttributes(data: any) : any { + if (typeof data === 'object') { + if (Array.isArray(data)) { + return data.map(v => KeymanXMLReader.fixupDollarAttributes(v)); + } + // object + const e : any = []; + const attrs : any = []; + Object.entries(data).forEach(([k, v]) => { + if (k[0] === '$') { + k = k.slice(1); + attrs.push([k, KeymanXMLReader.fixupDollarAttributes(v)]); + } else { + e.push([k, KeymanXMLReader.fixupDollarAttributes(v)]); + } + }); + if (attrs.length) { + e.push(['$', Object.fromEntries(attrs)]); + } + return Object.fromEntries(e); + } else { + return data; + } + } + + /** replace any empty string "" with an empty object {} */ + private static fixupEmptyStringToEmptyObject(data: any) : any { + if (data === "") { + // this is the core feature here. + return {}; + } else if (typeof data === 'object') { + if (Array.isArray(data)) { + return data.map(v => KeymanXMLReader.fixupEmptyStringToEmptyObject(v)); + } + // object + const e : any = []; + Object.entries(data).forEach(([k, v]) => { + e.push([k, KeymanXMLReader.fixupEmptyStringToEmptyObject(v)]); + }); + return Object.fromEntries(e); + } else { + return data; + } + } + + /** + * Replace: + * ```json + * [ { "info": [], ":@": { "abc": "def" } }] + * ``` + * with: + * ```json + * [{"$": { "abc": "def" }, "#name": "info" }] + * ``` + * see https://github.com/NaturalIntelligence/fast-xml-parser/blob/master/docs/v4/2.XMLparseOptions.md#preserveorder + * @param data input data + */ + private static fixupPreserveOrder(data: any): any { + + // we need to extract the root name specially + if (!Array.isArray(data)) { + throw Error(`Internal Error: XML parser preserveOrder did not yield an array.`); + } + if (data.length !== 1) { + // we ignore comments, so should only have one element + throw Error(`Internal Error: XML parser preserveOrder did not yield an array of size 1.`); + } + // the root element is special, we copy it into a property + const rootElement = KeymanXMLReader.fixupPreserveOrderObject(data[0]); + const rootElementName = rootElement['#name']; + const out: any = {}; + out[rootElementName] = rootElement; + return out; + } + + /** takes an 'object' with a property `:@` containing attrs, and one other property with the object name */ + private static fixupPreserveOrderObject(data: any): any { + const attrs = data[':@']; + const mainEntry : any = Object.entries(data).filter(([k,v]) => k !== ':@'); + const [elementName, subItems] = mainEntry[0]; + const out : any = {}; + if ( attrs ) { + out['$'] = attrs; + } + if (!elementName) { + throw Error(`could not find elementName in ${JSON.stringify(mainEntry[0])}`); + } + out['#name'] = elementName; + if (subItems && subItems.length) { + out['$$'] = subItems.map((subObject: any) => KeymanXMLReader.fixupPreserveOrderObject(subObject)); + // xml2js duplicated data here, including elements in their 'non-preserved-order' form. + // we don't read this data, but we're maintaining compatibility here with the read format. + // example: emit: […], keystroke:[…] + for (const o of out['$$']) { + const subElementName = o['#name']; + const nonPreservedElements = out[subElementName] = out[subElementName] ?? []; + const oWithoutName = {...o}; + delete oWithoutName['#name']; // #name is only there in the preserved-order form. + nonPreservedElements.push(oWithoutName); + } + } + return out; + } + public parse(data: string): any { const parser = this.parser(); - let a: any; - parser.parseString(data, (e: unknown, r: unknown) => { if (e) throw e; a = r; }); - return a; + let result = parser.parse(data, true); + if (PARSER_OPTIONS[this.type].attributeNamePrefix === '$') { + result = KeymanXMLReader.fixupDollarAttributes(result); + } + if (this.type === 'keyboard3') { + result = KeymanXMLReader.fixupEmptyStringToEmptyObject(result); + } + if (PARSER_OPTIONS[this.type].preserveOrder) { + result = KeymanXMLReader.fixupPreserveOrder(result); + } + delete result['?xml']; + return result; } public parser() { @@ -103,16 +238,49 @@ export class KeymanXMLReader { if (options.emptyTag) { options.emptyTag = {}; // TODO: xml2js likes to mutate the options here. Reset it. } - return new xml2js.Parser(options); + return new XMLParser(options); } } +/** + * Fixed prologue for writing XML + */ +const PROLOGUE = { '?xml': { '$version': '1.0', '$encoding': 'utf-8' } }; + /** wrapper for XML generation support */ export class KeymanXMLWriter { + + private static fixDataForWrite(data: any) : any { + if(typeof data === 'object') { + if (Array.isArray(data)) { + // just fixup each item of the array + return data.map(d => KeymanXMLWriter.fixDataForWrite(d)); + } + // else object + const e : any = []; + Object.entries(data).forEach(([k,v]) => { + if (k === '$') { + /* convert $: { a: 1, b: 2 } to { $a: 1, $b: 2} */ + Object.entries(v).forEach(([k,v]) => { + e.push([`\$${k}`, KeymanXMLWriter.fixDataForWrite(v)]); + }); + } else { + e.push([k, KeymanXMLWriter.fixDataForWrite(v)]); + } + }); + // reconstitute with $ elements fixed + return Object.fromEntries(e); + } else { + return data; // string or something else + } + } + write(data: any): string { const builder = this.builder(); - return builder.buildObject(data); + data = KeymanXMLWriter.fixDataForWrite(data); + return builder.build({ ...PROLOGUE, ...data }); } + constructor(public type: KeymanXMLType) { } @@ -122,7 +290,7 @@ export class KeymanXMLWriter { /* c8 ignore next 1 */ throw Error(`Internal error: unhandled XML type ${this.type}`); } - return new xml2js.Builder(Object.assign({}, options)); // Shallow clone in case the options are mutated. + return new XMLBuilder(Object.assign({}, options)); // Shallow clone in case the options are mutated. } } diff --git a/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor.kvks.json b/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor.kvks.json index 6775beb1605..a20c8b67518 100644 --- a/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor.kvks.json +++ b/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor.kvks.json @@ -1,17 +1,17 @@ { "visualkeyboard": { - "_": "\n \n \n", + "header": { - "_": "\n \n \n \n ", + "version": "10.0", "kbdname": "khmer_angkor", "flags": { - "_": "\n \n ", + "usealtgr": "" } }, "encoding": { - "_": "\n \n \n \n \n ", + "$": { "name": "unicode", "fontname": "Khmer Busra Kbd", @@ -19,7 +19,7 @@ }, "layer": [ { - "_": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ", + "$": { "shift": "RA" }, @@ -315,7 +315,7 @@ ] }, { - "_": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ", + "$": { "shift": "SRA" }, @@ -575,7 +575,7 @@ ] }, { - "_": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ", + "$": { "shift": "" }, @@ -871,13 +871,12 @@ ] }, { - "_": "\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n ", + "$": { "shift": "S" }, "key": [ { - "_": "", "$": { "vkey": "K_SPACE" } @@ -1169,4 +1168,4 @@ ] } } -} \ No newline at end of file +} diff --git a/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor2.kvks b/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor2.kvks index 05eb838178f..945477d3fee 100644 --- a/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor2.kvks +++ b/developer/src/common/web/utils/test/fixtures/xml/khmer_angkor2.kvks @@ -1,4 +1,4 @@ - +
10.0 @@ -156,7 +156,7 @@ ! - " + " ៛ % @@ -203,4 +203,4 @@ - \ No newline at end of file + diff --git a/developer/src/common/web/utils/test/kpj/test-kpj-file-reader.ts b/developer/src/common/web/utils/test/kpj/test-kpj-file-reader.ts index 4b154af0606..1caf70cd20d 100644 --- a/developer/src/common/web/utils/test/kpj/test-kpj-file-reader.ts +++ b/developer/src/common/web/utils/test/kpj/test-kpj-file-reader.ts @@ -15,9 +15,7 @@ describe('kpj-file-reader', function () { const input = fs.readFileSync(path); const reader = new KPJFileReader(callbacks); const kpj = reader.read(input); - assert.doesNotThrow(() => { - reader.validate(kpj); - }); + reader.validate(kpj); assert.equal(kpj.KeymanDeveloperProject.Options.BuildPath, '$PROJECTPATH\\build'); assert.equal(kpj.KeymanDeveloperProject.Options.CheckFilenameConventions, 'False'); assert.equal(kpj.KeymanDeveloperProject.Options.CompilerWarningsAsErrors, 'True'); diff --git a/developer/src/common/web/utils/tsconfig.json b/developer/src/common/web/utils/tsconfig.json index 3be972dbcb6..6f96f75ee70 100644 --- a/developer/src/common/web/utils/tsconfig.json +++ b/developer/src/common/web/utils/tsconfig.json @@ -8,6 +8,5 @@ }, "include": [ "src/**/*.ts", - "src/deps/xml2js/*.js", ], -} \ No newline at end of file +} diff --git a/package-lock.json b/package-lock.json index 7cbb8d2da2a..8cf9c5d5d83 100644 --- a/package-lock.json +++ b/package-lock.json @@ -327,6 +327,7 @@ "@keymanapp/common-types": "*", "@sentry/node": "^7.57.0", "eventemitter3": "^5.0.0", + "fast-xml-parser": "^4.5.0", "restructure": "^3.0.1", "sax": ">=0.6.0", "semver": "^7.5.4", @@ -336,7 +337,6 @@ "@types/git-diff": "^2.0.3", "@types/node": "^20.4.1", "@types/semver": "^7.3.12", - "@types/xml2js": "^0.4.5", "c8": "^7.12.0", "git-diff": "^2.0.6", "mocha": "^8.4.0", @@ -4668,14 +4668,6 @@ "@types/node": "*" } }, - "node_modules/@types/xml2js": { - "version": "0.4.9", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/yargs": { "version": "17.0.26", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.26.tgz", @@ -8739,6 +8731,28 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", + "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "license": "MIT", + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -13553,6 +13567,12 @@ "resolved": "resources/tools/strip-emoji", "link": true }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", + "license": "MIT" + }, "node_modules/supports-color": { "version": "7.2.0", "license": "MIT", From 03a24a150619f1677133240faef45eec9ec389f9 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Thu, 3 Oct 2024 12:41:08 -0500 Subject: [PATCH 2/3] feat(developer): fast-xml-parser: fix for ldml empty attributes - because of the form to="" in ldml, we need to distinguish attributes and sub-elements in the ldml xml parsing - use an attributePrefix, and fixup the object tree afterwards Fixes: #12208 --- .../src/common/web/utils/src/xml-utils.ts | 42 ++++++++++++------- .../xml/tran_fail-matches-nothing-1.xml | 13 ++++++ .../xml/tran_fail-matches-nothing-1.xml.json | 19 +++++++++ .../common/web/utils/test/test-xml-utils.ts | 1 + 4 files changed, 61 insertions(+), 14 deletions(-) create mode 100644 developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml create mode 100644 developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml.json diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 02d28dfc90c..9ed252256e6 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -25,7 +25,7 @@ type KemanXMLOptionsBag = { const PARSER_OPTIONS: KemanXMLOptionsBag = { 'keyboard3': { ignoreAttributes: false, // We'd like attributes, please - attributeNamePrefix: '', // to avoid '@_' prefixes + attributeNamePrefix: '@__', // We'll use this to convert attributes to strings and subobjects to arrays, when empty. trimValues: false, // preserve spaces, but: htmlEntities: true, tagValueProcessor: (tagName: string, tagValue: string /*, jPath, hasAttributes, isLeafNode*/) => { @@ -133,19 +133,35 @@ export class KeymanXMLReader { } } - /** replace any empty string "" with an empty object {} */ + /** + * Requires attribute prefix @__ (double underscore) + * For attributes, just remove @__ and continue. + * For objects, replace any empty string "" with an empty object {} */ private static fixupEmptyStringToEmptyObject(data: any) : any { - if (data === "") { - // this is the core feature here. - return {}; - } else if (typeof data === 'object') { + if (typeof data === 'object') { + // For arrays of objects, we map "" to {} + // "" means an empty object if (Array.isArray(data)) { - return data.map(v => KeymanXMLReader.fixupEmptyStringToEmptyObject(v)); + return data.map(v => { + if (v === '') { + return {}; + } else { + return KeymanXMLReader.fixupEmptyStringToEmptyObject(v); + } + }); } - // object - const e : any = []; + // otherwise: remove @__ for attributes, remap objects + const e: any = []; Object.entries(data).forEach(([k, v]) => { - e.push([k, KeymanXMLReader.fixupEmptyStringToEmptyObject(v)]); + if (k.startsWith('@__')) { + e.push([k.substring(3), KeymanXMLReader.fixupEmptyStringToEmptyObject(v)]); + } else { + if (v === '') { + e.push([k, {}]); + } else { + e.push([k, KeymanXMLReader.fixupEmptyStringToEmptyObject(v)]); + } + } }); return Object.fromEntries(e); } else { @@ -217,11 +233,9 @@ export class KeymanXMLReader { let result = parser.parse(data, true); if (PARSER_OPTIONS[this.type].attributeNamePrefix === '$') { result = KeymanXMLReader.fixupDollarAttributes(result); - } - if (this.type === 'keyboard3') { + } else if (PARSER_OPTIONS[this.type].attributeNamePrefix === '@__') { result = KeymanXMLReader.fixupEmptyStringToEmptyObject(result); - } - if (PARSER_OPTIONS[this.type].preserveOrder) { + } else if (PARSER_OPTIONS[this.type].preserveOrder) { result = KeymanXMLReader.fixupPreserveOrder(result); } delete result['?xml']; diff --git a/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml b/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml new file mode 100644 index 00000000000..5b5e2bba5bd --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml.json b/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml.json new file mode 100644 index 00000000000..477499477d8 --- /dev/null +++ b/developer/src/common/web/utils/test/fixtures/xml/tran_fail-matches-nothing-1.xml.json @@ -0,0 +1,19 @@ +{ + "keyboard3": { + "conformsTo": "45", + "xmlns": "https://schemas.unicode.org/cldr/45/keyboard3", + "locale": "mt", + "info": { + "name": "fail-matches-nothing" + }, + "keys": {}, + "transforms": { + "type": "simple", + "transformGroup": { + "transform": { + "from": "" + } + } + } + } +} \ No newline at end of file diff --git a/developer/src/common/web/utils/test/test-xml-utils.ts b/developer/src/common/web/utils/test/test-xml-utils.ts index fbe32f8da4f..1cc9724b278 100644 --- a/developer/src/common/web/utils/test/test-xml-utils.ts +++ b/developer/src/common/web/utils/test/test-xml-utils.ts @@ -32,6 +32,7 @@ const read_cases: Case[] = [ 'k_020_fr.xml', 'strs_invalid-illegal.xml', 'tran_fail-empty.xml', + 'tran_fail-matches-nothing-1.xml', ], }, { type: 'keyboardTest3', From 9a4c7e5820a27487bfe5363d030a04f619756d85 Mon Sep 17 00:00:00 2001 From: "Steven R. Loomis" Date: Mon, 7 Oct 2024 13:05:50 -0500 Subject: [PATCH 3/3] feat(developer): fast-xml-parser: typo fix in xml-utils.ts Fixes: #12208 --- developer/src/common/web/utils/src/xml-utils.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/developer/src/common/web/utils/src/xml-utils.ts b/developer/src/common/web/utils/src/xml-utils.ts index 9ed252256e6..999da59fde3 100644 --- a/developer/src/common/web/utils/src/xml-utils.ts +++ b/developer/src/common/web/utils/src/xml-utils.ts @@ -17,12 +17,12 @@ export type KeymanXMLType = ; /** Bag of options, maximally one for each KeymanXMLType */ -type KemanXMLOptionsBag = { +type KeymanXMLOptionsBag = { [key in KeymanXMLType]?: any }; /** map of options for the XML parser */ -const PARSER_OPTIONS: KemanXMLOptionsBag = { +const PARSER_OPTIONS: KeymanXMLOptionsBag = { 'keyboard3': { ignoreAttributes: false, // We'd like attributes, please attributeNamePrefix: '@__', // We'll use this to convert attributes to strings and subobjects to arrays, when empty. @@ -92,7 +92,7 @@ const PARSER_OPTIONS: KemanXMLOptionsBag = { }, }; -const GENERATOR_OPTIONS: KemanXMLOptionsBag = { +const GENERATOR_OPTIONS: KeymanXMLOptionsBag = { kvks: { attributeNamePrefix: '$', ignoreAttributes: false,