Merge pull request #12754 from keymanapp/change/developer/12746-kmc-c…

…opy-full-gh-url

change(developer): use full github url in kmc copy parameters

developer/docs/help/reference/kmc/cli/reference.md

@@ -493,9 +493,8 @@ following sources:
   `author.bcp47.uniq` id pattern, or a keyboard id pattern (where period `.` is
   not permitted)
 * A GitHub repository or subfolder within a repository that matches the Keyman
-  keyboard/model repository layout. The branch name is optional, and will use
-  the default branch from the repository if omitted. For example,
-  `github:keyman-keyboards/khmer_angkor:main:/khmer_angkor.kpj`
+  keyboard/model repository layout. For example,
+  `https://github.com/keyman-keyboards/khmer_angkor/tree/main/khmer_angkor.kpj`
 
 `-o, --out-path <filename>`
 

developer/src/common/web/utils/src/cloud-urls.ts

@@ -0,0 +1,18 @@
+/**
+ * Matches a Keyman keyboard resource, based on the permanent home page for the
+ * keyboard on keyman.com, `https://keyman.com/keyboards/<id>`
+ */
+export const KEYMANCOM_CLOUD_URI = /^(?:http(?:s)?:\/\/)?keyman\.com\/keyboards\/(?<id>[a-z0-9_.-]+)/i;
+
+/**
+ * Matches a `cloud:<id>` URI for a Keyman resource (e.g. keyboard or lexical
+ * model)
+ */
+export const CLOUD_URI = /^cloud:(?<id>.+)$/i;
+
+
+export interface CloudUriRegexMatchArray extends RegExpMatchArray {
+  groups?: {
+    id?: string;
+  }
+}

developer/src/common/web/utils/src/github-urls.ts

@@ -0,0 +1,35 @@
+/**
+ * Matches only a GitHub permanent raw URI with a commit hash, without any other
+ * components; note hash is called branch to match other URI formats
+ */
+export const GITHUB_STABLE_SOURCE = /^https:\/\/github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/raw\/(?<branch>[a-f0-9]{40})\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw 'user content' URI which can be
+ * translated to a permanent URI with a commit hash
+ */
+export const GITHUB_RAW_URI = /^https:\/\/raw\.githubusercontent\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw URI which can be translated to a
+ * permanent URI with a commit hash
+ */
+export const GITHUB_URI = /^https:\/\/github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)\/(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.+)$/;
+
+/**
+ * Matches any GitHub git resource raw URI which can be translated to a
+ * permanent URI with a commit hash, with the http[s] protocol optional, for
+ * matching user-supplied URLs. groups are: `owner`, `repo`, `branch`, and
+ * `path`.
+ */
+export const GITHUB_URI_OPTIONAL_PROTOCOL = /^(?:http(?:s)?:\/\/)?github\.com\/(?<owner>[a-zA-Z0-9-]+)\/(?<repo>[\w\.-]+)(?:\/(?:(?:raw|blob|tree)\/(?:refs\/(?:heads|tags)\/)?(?<branch>[^/]+)\/(?<path>.*))?)?$/;
+
+
+export interface GitHubRegexMatchArray extends RegExpMatchArray {
+  groups?: {
+    owner?: string;
+    repo?: string;
+    branch?: string;
+    path?: string;
+  }
+}

developer/src/common/web/utils/src/index.ts

@@ -66,3 +66,6 @@ export { UrlSubpathCompilerCallback } from './utils/UrlSubpathCompilerCallback.j
 export { CommonTypesMessages } from './common-messages.js';
 export * as SourceFilenamePatterns from './source-filename-patterns.js';
 export { KeymanXMLType, KeymanXMLWriter, KeymanXMLReader } from './xml-utils.js';
+
+export * as GitHubUrls from './github-urls.js';
+export * as CloudUrls from './cloud-urls.js';

developer/src/kmc-copy/src/KeymanProjectCopier.ts

@@ -4,7 +4,7 @@
  * Copy a keyboard or lexical model project
  */
 
-import { CompilerCallbacks, CompilerLogLevel, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanDeveloperProject, KeymanDeveloperProjectOptions, KPJFileReader, KPJFileWriter, KpsFileReader, KpsFileWriter } from "@keymanapp/developer-utils";
+import { CloudUrls, GitHubUrls, CompilerCallbacks, CompilerLogLevel, KeymanCompiler, KeymanCompilerArtifact, KeymanCompilerArtifacts, KeymanCompilerResult, KeymanDeveloperProject, KeymanDeveloperProjectOptions, KPJFileReader, KPJFileWriter, KpsFileReader, KpsFileWriter } from "@keymanapp/developer-utils";
 import { KeymanFileTypes } from "@keymanapp/common-types";
 
 import { CopierMessages } from "./copier-messages.js";
@@ -86,6 +86,9 @@ export class KeymanProjectCopier implements KeymanCompiler {
   relocateExternalFiles: boolean = false; // TODO-COPY: support
 
   public async init(callbacks: CompilerCallbacks, options: CopierOptions): Promise<boolean> {
+    if(!callbacks || !options) {
+      return false;
+    }
     this.callbacks = callbacks;
     this.options = options;
     this.cloudSource = new KeymanCloudSource(this.callbacks);
@@ -97,7 +100,7 @@ export class KeymanProjectCopier implements KeymanCompiler {
    * artifacts on success. The files are passed in by name, and the compiler
    * will use callbacks as passed to the {@link KeymanProjectCopier.init}
    * function to read any input files by disk.
-   * @param   source  Source file or folder to copy. Can be a local file or folder, github:repo[:path], or cloud:id
+   * @param   source  Source file or folder to copy. Can be a local file or folder, https://github.com/.../repo[/path], or cloud:id
    * @returns         Binary artifacts on success, null on failure.
    */
   public async run(source: string): Promise<CopierResult> {
@@ -151,10 +154,10 @@ export class KeymanProjectCopier implements KeymanCompiler {
    * @returns  path to .kpj (either local or remote)
    */
   private async getSourceProject(source: string): Promise<string | GitHubRef> {
-    if(source.startsWith('github:')) {
-      // `github:owner/repo:path/to/kpj`, referencing a .kpj file
+    if(source.match(GitHubUrls.GITHUB_URI_OPTIONAL_PROTOCOL) || source.match(GitHubUrls.GITHUB_RAW_URI)) {
+      // `[https://]github.com/owner/repo/[tree|blob|raw]/[refs/...]/branch/path/to/kpj`, referencing a .kpj file
       return await this.getGitHubSourceProject(source);
-    } else if(source.startsWith('cloud:')) {
+    } else if(source.match(CloudUrls.CLOUD_URI) || source.match(CloudUrls.KEYMANCOM_CLOUD_URI)) {
       // `cloud:id`, referencing a Keyman Cloud keyboard
       return await this.getCloudSourceProject(source);
     } else if(this.callbacks.fs.existsSync(source) && source.endsWith(KeymanFileTypes.Source.Project) && !this.callbacks.isDirectory(source)) {
@@ -194,64 +197,69 @@ export class KeymanProjectCopier implements KeymanCompiler {
 
   /**
    * Resolve path to GitHub source, which must be in the following format:
-   *   `github:owner/repo[:branch]:path/to/kpj`
+   *   `[https://]github.com/owner/repo/branch/path/to/kpj`
    * The path must be fully qualified, referencing the .kpj file; it
    * cannot just be the folder where the .kpj is found
    * @param source
    * @returns a promise: GitHub reference to the source for the keyboard, or null on failure
    */
   private async getGitHubSourceProject(source: string): Promise<GitHubRef> {
-    const parts = source.split(':');
-    if(parts.length < 3 || parts.length > 4 || !parts[1].match(/^[a-z0-9-]+\/[a-z0-9._-]+$/i)) {
-      // https://stackoverflow.com/questions/59081778/rules-for-special-characters-in-github-repository-name
-      this.callbacks.reportMessage(CopierMessages.Error_InvalidGitHubSource({source}));
-      return null;
+    const parts: GitHubUrls.GitHubRegexMatchArray =
+      GitHubUrls.GITHUB_URI_OPTIONAL_PROTOCOL.exec(source) ??
+      GitHubUrls.GITHUB_RAW_URI.exec(source);
+    if(!parts) {
+      throw new Error('Expected GITHUB_URI_OPTIONAL_PROTOCOL or GITHUB_RAW_URI to match');
     }
 
-    const origin = parts[1].split('/');
-
-    const ref: GitHubRef = new GitHubRef({
-      owner: origin[0],
-      repo: origin[1],
-      branch: null,
-      path: null
-    });
+    const ref: GitHubRef = new GitHubRef(parts);
 
-    if(parts.length == 4) {
-      ref.branch = parts[2];
-      ref.path = parts[3];
-    } else {
+    if(!ref.branch) {
       ref.branch = await this.cloudSource.getDefaultBranchFromGitHub(ref);
       if(!ref.branch) {
         this.callbacks.reportMessage(CopierMessages.Error_CouldNotFindDefaultBranchOnGitHub({ref: ref.toString()}));
         return null;
       }
-      ref.path = parts[2];
-
+    }
+    if(!ref.path) {
+      ref.path = '/';
     }
     if(!ref.path.startsWith('/')) {
       ref.path = '/' + ref.path;
     }
 
+    if(ref.path != '/') {
+      if(!ref.path.endsWith('.kpj')) {
+        // Assumption, project filename matches folder name
+        if(ref.path.endsWith('/')) {
+          ref.path = ref.path.substring(0, ref.path.length-1);
+        }
+        ref.path = ref.path + '/' + this.callbacks.path.basename(ref.path) + '.kpj';
+      }
+    }
+
     return ref;
   }
 
   /**
    * Resolve path to Keyman Cloud source (which is on GitHub), which must be in
    * the following format:
-   *   `cloud:keyboard_id|model_id`
+   *   `cloud:keyboard_id`, or
+   *   `cloud:model_id`, or
+   *   `https://keyman.com/keyboards/keyboard_id`
    * The `keyboard_id` parameter should be a valid id (a-z0-9_), as found at
-   * https://keyman.com/keyboards; alternativel if it is a model_id, it should
+   * https://keyman.com/keyboards; alternatively if it is a model_id, it should
    * have the format author.bcp47.uniq
    * @param source
    * @returns a promise: GitHub reference to the source for the keyboard, or null on failure
    */
   private async getCloudSourceProject(source: string): Promise<GitHubRef> {
-    const parts = source.split(':');
-    const id = parts[1];
+    const parts = CloudUrls.CLOUD_URI.exec(source) ?? CloudUrls.KEYMANCOM_CLOUD_URI.exec(source);
+    if(!parts) {
+      throw new Error('Expected CLOUD_URI or KEYMANCOM_CLOUD_URI to match');
+    }
 
+    const id: string = parts.groups.id;
     const isModel = /^[^.]+\.[^.]+\.[^.]+$/.test(id);
-
     const remote = await this.cloudSource.getSourceFromKeymanCloud(id, isModel);
     if(!remote) {
       return null;
@@ -687,4 +695,11 @@ export class KeymanProjectCopier implements KeymanCompiler {
     return true;
   }
   /* c8 ignore stop */
+
+  /** @internal */
+  public unitTestEndPoints = {
+    getGithubSourceProject: this.getGitHubSourceProject.bind(this),
+    getCloudSourceProject: this.getCloudSourceProject.bind(this)
+  };
+
 }

developer/src/kmc-copy/src/cloud.ts

@@ -3,7 +3,7 @@
  *
  * GitHub and Keyman Cloud interface wrappers
  */
-import { CompilerCallbacks } from "@keymanapp/developer-utils";
+import { CompilerCallbacks, GitHubUrls } from "@keymanapp/developer-utils";
 import { CopierMessages } from "./copier-messages.js";
 import { KeymanFileTypes } from "@keymanapp/common-types";
 
@@ -12,17 +12,24 @@ export class GitHubRef {
   public repo: string;
   public branch: string;
   public path: string;
-  constructor(owner: string | GitHubRef, repo?: string, branch?: string, path?: string) {
+  constructor(owner: string | GitHubRef | GitHubUrls.GitHubRegexMatchArray, repo?: string, branch?: string, path?: string) {
     if(typeof owner == 'string') {
       this.owner = owner;
       this.repo = repo;
       this.branch = branch;
       this.path = path;
-    } else {
+    } else if("groups" in owner) {
+      this.owner = owner.groups.owner;
+      this.repo = owner.groups.repo;
+      this.branch = owner.groups.branch;
+      this.path = owner.groups.path;
+    } else if("owner" in owner) {
       this.owner = owner.owner;
       this.repo = owner.repo;
       this.branch = owner.branch;
       this.path = owner.path;
+    } else {
+      throw new Error(`Unrecognized GitHubRef '${owner}'`)
     }
   }
   toString() {

developer/src/kmc-copy/src/copier-messages.ts

@@ -117,18 +117,7 @@ export class CopierMessages {
     `Dry run requested. No changes have been saved`
   );
 
-  static ERROR_InvalidGitHubSource = SevError | 0x0011;
-  static Error_InvalidGitHubSource = (o:{source: string}) => m(
-    this.ERROR_InvalidGitHubSource,
-    `Source project specification '${def(o.source)}' is not a valid GitHub reference`,
-    `The source project specification for GitHub sources must match the pattern:
-        github:\\<owner/repo>[:\\<branch>]:\\<path>
-    The path must include the .kpj filename and may optionally begin with a forward slash.
-    The following are valid examples:
-        github:keymanapp/keyboards:master:release/k/khmer_angkor/khmer_angkor.kpj
-        github:keymanapp/keyboards:release/k/khmer_angkor/khmer_angkor.kpj
-        github:keymanapp/keyboards:/release/k/khmer_angkor/khmer_angkor.kpj`
-  );
+  // 0x0011 unused
 
   static ERROR_CannotDownloadFolderFromGitHub = SevError | 0x0012;
   static Error_CannotDownloadFolderFromGitHub = (o:{ref: string, message?: string, cause?: string}) => m(

developer/src/kmc-copy/test/copier.tests.ts

@@ -12,6 +12,7 @@ import { assert } from 'chai';
 import { TestCompilerCallbacks } from '@keymanapp/developer-test-helpers';
 import { KeymanProjectCopier } from '../src/KeymanProjectCopier.js';
 import { makePathToFixture } from './helpers/index.js';
+import { GitHubRef } from './cloud.js';
 
 const { TEST_SAVE_ARTIFACTS, TEST_SAVE_FIXTURES } = env;
 let outputRoot: string = '/an/imaginary/root/';
@@ -374,7 +375,7 @@ describe('KeymanProjectCopier', function() {
 
     // armenian_mnemonic selected because (a) small, and (b) has v2.0 project, so
     // that exercises the folder retrieval as well
-    const result = await copier.run('github:keymanapp/keyboards:release/a/armenian_mnemonic/armenian_mnemonic.kpj');
+    const result = await copier.run('github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj');
 
     // We should have no messages and a successful result
     assert.isOk(result);
@@ -416,6 +417,81 @@ describe('KeymanProjectCopier', function() {
     });
   });
 
+  // Keyman Cloud patterns
+
+  const cloud_khmer_angkor: GitHubRef = { branch: 'master', owner: 'keymanapp', repo: 'keyboards', path: '/release/k/khmer_angkor/khmer_angkor.kpj' };
+  const cloud_nrc_en_mtnt: GitHubRef = { branch: 'master', owner: 'keymanapp', repo: 'lexical-models', path: '/release/nrc/nrc.en.mtnt/nrc.en.mtnt.kpj' };
+  const cloud_urls: [string,GitHubRef][] = [
+    ['cloud:khmer_angkor', cloud_khmer_angkor],
+    ['https://keyman.com/keyboards/khmer_angkor', cloud_khmer_angkor],
+    ['https://keyman.com/keyboards/khmer_angkor/', cloud_khmer_angkor],
+    ['keyman.com/keyboards/khmer_angkor/', cloud_khmer_angkor],
+    ['http://keyman.com/keyboards/khmer_angkor#abc', cloud_khmer_angkor],
+    ['cloud:nrc.en.mtnt', cloud_nrc_en_mtnt],
+  ];
+
+  cloud_urls.forEach(url => {
+    it(`should parse URL '${url[0]}' and figure out the .kpj`, async function() {
+      // url -->
+      const copier = new KeymanProjectCopier();
+      assert.isTrue(await copier.init(callbacks, {
+        dryRun: false,
+        outPath: ''
+      }));
+
+      const ref = await copier.unitTestEndPoints.getCloudSourceProject(url[0]);
+      assert.isNotNull(ref);
+      assert.deepEqual(ref, url[1]);
+    });
+  })
+
+
+  // GitHub patterns that should match as inputs for kmc-copy source
+
+  const armenian_mnemonic_urls = [ {
+    branch: 'master', urls: [
+    'github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic',
+    'http://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic',
+    'https://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic',
+    'https://github.com/keymanapp/keyboards/tree/master/release/a/armenian_mnemonic/',
+    'https://github.com/keymanapp/keyboards/tree/refs/heads/master/release/a/armenian_mnemonic',
+    'https://github.com/keymanapp/keyboards/tree/refs/heads/master/release/a/armenian_mnemonic/',
+    'https://github.com/keymanapp/keyboards/raw/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+    'https://github.com/keymanapp/keyboards/raw/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+    'https://github.com/keymanapp/keyboards/blob/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+    'https://github.com/keymanapp/keyboards/blob/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+
+    // And similar patterns for raw.githubusercontent.com
+
+    'https://raw.githubusercontent.com/keymanapp/keyboards/refs/heads/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+    'https://raw.githubusercontent.com/keymanapp/keyboards/master/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+  ]}, {
+    branch: '78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2', urls: [
+    'https://github.com/keymanapp/keyboards/blob/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+    'https://github.com/keymanapp/keyboards/tree/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic',
+    'https://github.com/keymanapp/keyboards/tree/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/',
+    'https://raw.githubusercontent.com/keymanapp/keyboards/78b6f98e5db4a249cc4231f8744f5fe4e5fd29f2/release/a/armenian_mnemonic/armenian_mnemonic.kpj',
+  ]}];
+
+  armenian_mnemonic_urls.forEach(({branch,urls}) => urls.forEach(url => {
+    it(`should parse URL '${url}' and figure out the .kpj`, async function() {
+      // url -->
+      const copier = new KeymanProjectCopier();
+      assert.isTrue(await copier.init(callbacks, {
+        dryRun: false,
+        outPath: ''
+      }));
+
+      const ref = await copier.unitTestEndPoints.getGithubSourceProject(url);
+      assert.deepEqual(ref, {
+        branch,
+        owner: 'keymanapp',
+        repo: 'keyboards',
+        path: '/release/a/armenian_mnemonic/armenian_mnemonic.kpj'
+      });
+    });
+  }));
+
   // TODO-COPY: additional tests
   it.skip('should copy a disorganized project into current structure', async function() {});
   it.skip('should copy a standalone .kmn into a new project', async function() {});