Merge branch 'epic/shared-fonts' into chore/merge-master-into-shared-…

…fonts

common/schemas/kps/README.md

@@ -2,6 +2,19 @@
 
 Master version: https://github.com/keymanapp/api.keyman.com/blob/master/schemas/kps/17.0/kps.xsd
 
+See https://github.com/keymanapp/
+
+## 2024-11-29 18.0
+* Version 18.0 adds:
+  - Files/File/Source - a remote source for the file, or 'local'
+* Version 18.0 modifies:
+  - Files/File/Name - may be either a local file, relative to .kps (or absolute
+    path, not recommended) or a permanent URL on github.com
+* Version 18.0 deprecates (for later removal):
+  - Files/File/CopyLocation
+  - Files/File/Description
+  - Files/File/FileType
+
 ## 2023-10-19 17.0
 * Version 17.0 adds:
   - LicenseFile - a .md file, usually named LICENSE.md
@@ -10,7 +23,7 @@ Master version: https://github.com/keymanapp/api.keyman.com/blob/master/schemas/
   - RelatedPackages - a list of other packages which relate to this one, or are deprecated by it
   - Keyboards/Keyboard/Examples - a list of typing examples for the keyboard
   - Keyboarsd/Keyboard/WebOSKFonts - a list of font filenames (not necessarily in package) suitable for rendering the on screen keyboard
-  - Keyboarsd/Keyboard/WebDisplayFonts - a list of font filenames (not necessarily in package) suitable for use with the keyboard
+  - Keyboards/Keyboard/WebDisplayFonts - a list of font filenames (not necessarily in package) suitable for use with the keyboard
 * Version 17.0 removes:
   - LexicalModels/LexicalModel/Version - version information is not stored in the models, but only in the package metadata (was unused)
 

common/schemas/kps/kps.xsd

@@ -6,7 +6,10 @@
 
   Supports KPS files for Keyman Developer 7+
 
-  Expects FileVersion 7.0
+  Expects FileVersion:
+  - 7.0 - minimum version
+  - 12.0 - supports lexical models, kmp.json
+  - 18.0 - supports Files/File/Source
 -->
 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
 
@@ -89,9 +92,10 @@
               <xs:complexType>
                 <xs:all>
                   <xs:element minOccurs="0" maxOccurs="1" name="Name" type="xs:string" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="Description" type="xs:string" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="CopyLocation" type="km-file-copy-location" />
-                  <xs:element minOccurs="0" maxOccurs="1" name="FileType" type="xs:string" />
+                  <xs:element minOccurs="0" maxOccurs="1" name="Description" type="xs:string" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="CopyLocation" type="km-file-copy-location" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="FileType" type="xs:string" /> <!-- 18.0: deprecated -->
+                  <xs:element minOccurs="0" maxOccurs="1" name="Source" type="xs:string" />
                 </xs:all>
               </xs:complexType>
             </xs:element>

developer/docs/help/reference/file-types/kps.md

@@ -17,11 +17,172 @@ Details:
 Distributed with keyboard:
 :   No. This is a development file and should not be distributed.
 
-## File format
+# File format
 
+## Schema
+
+The schema for .kps is documented at:
+https://github.com/keymanapp/keyman/tree/master/common/schemas/kps
+
+## Strings
 The optional `<Strings>` section of the file can be included to
 customise the text in the bootstrap installer. The default strings are
 found in the file:
 [strings.xml](https://github.com/keymanapp/keyman/blob/stable-14.0/windows/src/desktop/setup/locale/en/strings.xml).
 You can add your own strings for a given package which are used when
 compiling as a bundled installer.
+
+
+## File.Name
+
+The `Name` element may be either a local filename, or a GitHub raw permanent
+URL, conforming to the pattern:
+
+```
+<Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+```
+
+## File.Source
+
+The `<Source>` element documents the original source of the file, from
+one of several supported sources:
+* flo - fonts.languagetechnology.org
+* github
+* noto - in the future
+
+While the `<Name>` element must refer to either a local file or to a permanent
+GitHub file (as recognized by the presence of a commit hash), the `<Source>`
+element typically will refer to a file which may change over time.
+
+The `<Source>` element is supported in version 18.0 and later of .kps
+format.
+
+### Local file system
+
+The `<Source>` element must not be included for files sourced from the local
+file system, where `<Name>` references a file on the local filesystem.
+
+Note: the `<Source>` element is optional for other sources as well; the absence
+of the element does not mean that the `<Name>` element is necessarily
+referencing a local file -- that is determined by pattern match.
+
+```xml
+<File>
+  <Name>[relative-or-absolute-path/]{filename}</Name>
+</File>
+```
+
+Note: absolute paths are not recommended because they are not portable to
+other computers.
+
+#### Example
+
+```xml
+<File>
+  <Name>Alkalami-Regular.ttf</Name>
+</File>
+```
+
+```xml
+<File>
+  <Name>../../shared/fonts/Alkalami-Regular.ttf</Name>
+</File>
+```
+
+### fonts.languagetechnology.org `flo:<id>`
+
+References a font by ID from fonts.languagetechnology.org. The resource must be
+resolved to a permanent URL. If the `<Name>` field is missing, the compiler will
+return an error, reporting the URL it discovers, so the author should be able to
+paste that URL in.
+
+If the URL in the `<Name>` field differs from the resolved URL given by FLO,
+then a hint will be issued by the compiler, allowing the author to easily update
+to a new version of the resource at their convenience. It is anticipated that
+this may be scripted on the keyboards repository in the future.
+
+```xml
+<File>
+  <Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+  <Source>flo:{id}</Source>
+</File>
+```
+
+#### Example
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>flo:alkalami</Source>
+</File>
+```
+
+### GitHub
+
+A reference to a resource located on GitHub. The resource must be resolvable to
+a permanent URL. If the `<Name>` field is missing, the compiler will return an
+error, reporting the URL it discovers, so the author should be able to paste
+that URL in.
+
+If the URL in the `<Name>` field differs from the resolved URL from the
+`<Source>` element, then a hint will be issued by the compiler, allowing the
+author to easily update to a new version of the resource at their convenience.
+It is anticipated that this may be scripted on the keyboards repository in the
+future.
+
+```xml
+<File>
+  <Name>https://github.com/{owner}/{repo}/raw/{commit}/[path/]{filename}</Name>
+  <Source>https://github.com/{owner}/{repo}/(raw|blob)/([refs/heads/]{branch}|[refs/tags/]{tag}|{commit})/[path/]{filename}</Source>
+</File>
+```
+
+#### Examples
+
+A GitHub raw reference, from a branch:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+Including `/refs/heads/` is also acceptable (as often found in GitHub UI):
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/refs/heads/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+A GitHub blob reference, as copied from a URL in GitHub:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/blob/main/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+A GitHub raw reference, from a commit, resolves identically. Note that as the
+`<Source>` element includes a commit hash, the referenced file can never change
+and so there is no real benefit in including it:
+
+```xml
+<File>
+  <Name>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Name>
+  <Source>https://github.com/silnrsi/fonts/raw/b88c7af5d16681bd137156929ff8baec82526560/fonts/sil/alkalami/Alkalami-Regular.ttf</Source>
+</File>
+```
+
+### noto
+
+In a future version.
+
+## Additional file format notes
+
+The `CopyLocation`, `Description`, and `FileType` elements are deprecated as of
+v18.0 and totally ignored by the compiler and end-user apps.
+

developer/docs/help/reference/kmc/cli/reference.md

@@ -493,9 +493,8 @@ following sources:
   `author.bcp47.uniq` id pattern, or a keyboard id pattern (where period `.` is
   not permitted)
 * A GitHub repository or subfolder within a repository that matches the Keyman
-  keyboard/model repository layout. The branch name is optional, and will use
-  the default branch from the repository if omitted. For example,
-  `github:keyman-keyboards/khmer_angkor:main:/khmer_angkor.kpj`
+  keyboard/model repository layout. For example,
+  `https://github.com/keyman-keyboards/khmer_angkor/tree/main/khmer_angkor.kpj`
 
 `-o, --out-path <filename>`
 

developer/src/common/web/test-helpers/index.ts

@@ -119,7 +119,10 @@ export class TestCompilerCallbacks implements CompilerCallbacks {
   }
 
   get path(): CompilerPathCallbacks {
-    return path;
+    return {
+      ...path,
+      isAbsolute: path.win32.isAbsolute
+    };
   }
 
   get fs(): CompilerFileSystemCallbacks {

developer/src/common/web/utils/src/cloud-urls.ts

@@ -0,0 +1,18 @@
+/**
+ * Matches a Keyman keyboard resource, based on the permanent home page for the
+ * keyboard on keyman.com, `https://keyman.com/keyboards/<id>`
+ */
+export const KEYMANCOM_CLOUD_URI = /^(?:http(?:s)?:\/\/)?keyman\.com\/keyboards\/(?<id>[a-z0-9_.-]+)/i;
+
+/**
+ * Matches a `cloud:<id>` URI for a Keyman resource (e.g. keyboard or lexical
+ * model)
+ */
+export const CLOUD_URI = /^cloud:(?<id>.+)$/i;
+
+
+export interface CloudUriRegexMatchArray extends RegExpMatchArray {
+  groups?: {
+    id?: string;
+  }
+}

developer/src/common/web/utils/src/common-messages.ts

@@ -56,4 +56,10 @@ export class CommonTypesMessages {
     this.ERROR_InvalidPackageFile,
     `Package source file is invalid: ${(o.e ?? 'unknown error').toString()}`
   );
+
+  static ERROR_UnsupportedPackageFileVersion = SevError | 0x000A;
+  static Error_UnsupportedPackageFileVersion = (o:{version:string}) => m(
+    this.ERROR_UnsupportedPackageFileVersion,
+    `Package source file is an unsupported version '${def(o.version)}'`,
+  );
 };

developer/src/common/web/utils/src/github-urls.ts

@@ -0,0 +1,35 @@
+/**
+ * Matches only a GitHub permanent raw URI with a commit hash, without any other
+ * components; note hash is called branch to match other URI formats
+ */
+export const GITHUB_STABLE_SOURCE = /^https:\/\/github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/raw\/(?<branch>[a-f0-9]{40})\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw 'user content' URI which can be
+ * translated to a permanent URI with a commit hash
+ */
+export const GITHUB_RAW_URI = /^https:\/\/raw\.githubusercontent\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw URI which can be translated to a
+ * permanent URI with a commit hash
+ */
+export const GITHUB_URI = /^https:\/\/github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw URI which can be translated to a
+ * permanent URI with a commit hash, with the http[s] protocol optional, for
+ * matching user-supplied URLs. groups are: `owner`, `repo`, `branch`, and
+ * `path`.
+ */
+export const GITHUB_URI_OPTIONAL_PROTOCOL = /^(?:http(?:s)?:\/\/)?github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)(?:\/(?:(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.*))?)?$/;
+
+
+export interface GitHubRegexMatchArray extends RegExpMatchArray {
+  groups?: {
+    owner?: string;
+    repo?: string;
+    branch?: string;
+    path?: string;
+  }
+}

developer/src/common/web/utils/src/index.ts

@@ -66,3 +66,6 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j
 export { CommonTypesMessages } from './common-messages.js';
 export * as SourceFilenamePatterns from './source-filename-patterns.js';
 export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js';
+
+export * as GitHubUrls from './github-urls.js';
+export * as CloudUrls from './cloud-urls.js';

developer/src/common/web/utils/src/types/kps/kps-file-reader.ts

@@ -12,7 +12,8 @@ import boxXmlArray = util.boxXmlArray;
 import { CommonTypesMessages } from "../../common-messages.js";
 import { CompilerCallbacks } from "../../compiler-callbacks.js";
 import { KeymanXMLReader } from "../../xml-utils.js";
-import { KpsPackage } from "./kps-file.js";
+import { KPS_FILE_VERSIONS, KpsPackage } from "./kps-file.js";
+
 
 export class KpsFileReader {
   constructor(private callbacks: CompilerCallbacks) {
@@ -38,6 +39,11 @@ export class KpsFileReader {
       return null;
     }
 
+    if(!KPS_FILE_VERSIONS.includes(kpsPackage.Package?.System?.FileVersion)) {
+      this.callbacks.reportMessage(CommonTypesMessages.Error_UnsupportedPackageFileVersion({version: kpsPackage.Package?.System?.FileVersion}));
+      return null;
+    }
+
     return this.boxArrays(kpsPackage);
   }
 

developer/src/common/web/utils/src/types/kps/kps-file.ts

@@ -16,6 +16,21 @@
 // * Strings element is not yet checked to be correct
 //
 
+/** minimum supported file version, for keyboard packages */
+export const KPS_FILE_MINIMUM_VERSION_KEYBOARD_SUPPORT = '7.0';
+
+/** minimum version for lexical model packages */
+export const KPS_FILE_MINIMUM_VERSION_LEXICAL_MODEL_SUPPORT = '12.0';
+
+/** minimum version that supports remote references */
+export const KPS_FILE_MINIMUM_VERSION_REMOTE_SUPPORT = '18.0';
+
+export const KPS_FILE_VERSIONS = [
+  KPS_FILE_MINIMUM_VERSION_KEYBOARD_SUPPORT,
+  KPS_FILE_MINIMUM_VERSION_LEXICAL_MODEL_SUPPORT,
+  KPS_FILE_MINIMUM_VERSION_REMOTE_SUPPORT,
+];
+
 export interface KpsPackage {
   /**
    * <Package> -- the root element.
@@ -77,6 +92,8 @@ export interface KpsFileContentFile {
   CopyLocation?: string;
   /** @deprecated */
   FileType?: string;
+  /** Source location, e.g. flo:<id>, URL */
+  Source?: string;
 }
 
 export interface KpsFileLexicalModel {

developer/src/common/web/utils/test/TestCompilerCallbacks.ts

@@ -25,7 +25,10 @@ export class TestCompilerCallbacks implements CompilerCallbacks {
   messages: CompilerEvent[] = [];
 
   get path(): CompilerPathCallbacks {
-    return path;
+    return {
+      ...path,
+      isAbsolute: path.win32.isAbsolute
+    };
   }
 
   get fs(): CompilerFileSystemCallbacks {