From 640d3e50964fd452c9dc8a6286accb96d10d1885 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 17 Sep 2024 14:58:11 +0700 Subject: [PATCH 01/11] test(developer): add markdown link check test for product documentation Adds check-markdown package that parses .md files, looks for links and images, and verifies that they are valid links (for internal links). --- android/build.sh | 7 +- developer/src/build.sh | 3 + ios/build.sh | 8 +- linux/build.sh | 5 + mac/build.sh | 9 +- package-lock.json | 94 +++++++++++++++++++ package.json | 1 + resources/shellHelperFunctions.sh | 4 + resources/tools/check-markdown/.gitignore | 1 + resources/tools/check-markdown/build.sh | 26 +++++ resources/tools/check-markdown/package.json | 11 +++ .../tools/check-markdown/src/check-link.ts | 59 ++++++++++++ .../tools/check-markdown/src/find-files.ts | 9 ++ resources/tools/check-markdown/src/index.ts | 49 ++++++++++ .../tools/check-markdown/src/parse-files.ts | 30 ++++++ resources/tools/check-markdown/src/types.ts | 18 ++++ resources/tools/check-markdown/tsconfig.json | 9 ++ windows/src/build.sh | 9 +- 18 files changed, 345 insertions(+), 7 deletions(-) create mode 100644 resources/tools/check-markdown/.gitignore create mode 100755 resources/tools/check-markdown/build.sh create mode 100644 resources/tools/check-markdown/package.json create mode 100644 resources/tools/check-markdown/src/check-link.ts create mode 100644 resources/tools/check-markdown/src/find-files.ts create mode 100644 resources/tools/check-markdown/src/index.ts create mode 100644 resources/tools/check-markdown/src/parse-files.ts create mode 100644 resources/tools/check-markdown/src/types.ts create mode 100644 resources/tools/check-markdown/tsconfig.json diff --git a/android/build.sh b/android/build.sh index d876d882d77..ef75bb6b46a 100755 --- a/android/build.sh +++ b/android/build.sh @@ -18,11 +18,13 @@ builder_describe \ configure \ build \ test \ + "@/resources/tools/check-markdown test:help" \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ --ci+ \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ ":app=KMAPro Keyman for Android" \ + ":help Online documentation" \ ":sample1=Samples/KMSample1 Sample app: KMSample1" \ ":sample2=Samples/KMSample2 Sample app: KMSample2" \ ":keyboardharness=Tests/KeyboardHarness Test app: KeyboardHarness" \ @@ -42,4 +44,7 @@ if builder_start_action clean; then builder_finish_action success clean fi -builder_run_child_actions configure build test publish +builder_run_child_actions configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/android/docs/help" + +builder_run_child_actions publish \ No newline at end of file diff --git a/developer/src/build.sh b/developer/src/build.sh index c85851f1f65..f5e106164f3 100755 --- a/developer/src/build.sh +++ b/developer/src/build.sh @@ -15,11 +15,13 @@ builder_describe \ configure \ build \ test \ + "@/resources/tools/check-markdown test:help" \ "api Analyze API and prepare API documentation" \ "publish Prepare files for distribution, publish symbols, publish or pack npm packages, and build installer" \ "install Install built programs locally" \ ":common Developer common files" \ ":ext Third party components" \ + ":help Online documentation" \ ":kmcmplib Compiler - .kmn compiler" \ ":kmc-analyze Compiler - Analysis Tools" \ ":kmc-keyboard-info Compiler - .keyboard_info Module" \ @@ -125,6 +127,7 @@ fi #------------------------------------------------------------------------------------------------------------------- builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/developer/docs/help" #------------------------------------------------------------------------------------------------------------------- diff --git a/ios/build.sh b/ios/build.sh index 21a0be28277..1d650faa651 100755 --- a/ios/build.sh +++ b/ios/build.sh @@ -10,14 +10,17 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" # Please note that this build script (understandably) assumes that it is running on Mac OS X. -verify_on_mac +# verify_on_mac builder_describe "Builds Keyman Engine and the Keyman app for use on iOS devices - iPhone and iPad." \ "clean" \ "configure" \ "build" \ + "test" \ + "@/resources/tools/check-markdown test:help" \ ":engine Builds KeymanEngine.xcframework, usable by our main app and by third-party apps" \ ":app=keyman Builds the Keyman app for iOS platforms" \ + ":help Online documentation" \ ":sample1=Samples/KMSample1 Builds the first KeymanEngine sample app" \ ":sample2=Samples/KMSample2 Builds the second KeymanEngine sample app" \ ":fv=../oem/firstvoices/ios Builds OEM FirstVoices for iOS platforms" \ @@ -25,4 +28,5 @@ builder_describe "Builds Keyman Engine and the Keyman app for use on iOS devices builder_parse "$@" -builder_run_child_actions clean configure build \ No newline at end of file +builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/ios/docs/help" diff --git a/linux/build.sh b/linux/build.sh index 1012b7eb1f0..6b97c287f5a 100755 --- a/linux/build.sh +++ b/linux/build.sh @@ -6,12 +6,16 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + ################################ Main script ################################ builder_describe \ "Build Keyman for Linux." \ + "@/resources/tools/check-markdown test:help" \ ":config=keyman-config keyman-config" \ ":engine=ibus-keyman ibus-keyman" \ + ":help Online documentation" \ ":service=keyman-system-service keyman-system-service" \ "clean" \ "configure" \ @@ -36,3 +40,4 @@ test_action() { } builder_run_action test test_action +builder_run_action test:help check-markdown "$KEYMAN_ROOT/linux/docs/help" diff --git a/mac/build.sh b/mac/build.sh index fbca48bd05f..e052106edde 100755 --- a/mac/build.sh +++ b/mac/build.sh @@ -12,6 +12,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe "Builds Keyman for macOS." \ "@/core:mac" \ + "@/resources/tools/check-markdown test:help" \ "clean" \ "configure" \ "build" \ @@ -20,11 +21,12 @@ builder_describe "Builds Keyman for macOS." \ "install Installs result of Keyman4MacIM locally." \ ":engine KeymanEngine4Mac" \ ":app Keyman4MacIM" \ + ":help Online documentation" \ ":testapp Keyman4Mac (test harness)" \ "--quick,-q Bypasses notarization for $(builder_term install)" # Please note that this build script (understandably) assumes that it is running on Mac OS X. -verify_on_mac +# verify_on_mac builder_parse "$@" @@ -89,8 +91,8 @@ UPLOAD_SENTRY=false # Import local environment variables for build # -# /mac/localenv.sh can be used to define CERTIFICATE_ID, -# APPSTORECONNECT_PROVIDER, APPSTORECONNECT_USERNAME, +# /mac/localenv.sh can be used to define CERTIFICATE_ID, +# APPSTORECONNECT_PROVIDER, APPSTORECONNECT_USERNAME, # APPSTORECONNECT_PASSWORD, DEVELOPMENT_TEAM variables; # see /mac/README.md for details. # @@ -304,6 +306,7 @@ builder_run_action build:testapp do_build_testapp builder_run_action test:engine execBuildCommand $ENGINE_NAME "xcodebuild -project \"$KME4M_PROJECT_PATH\" $BUILD_OPTIONS test -scheme $ENGINE_NAME" builder_run_action test:app execBuildCommand "$IM_NAME-tests" "xcodebuild test -workspace \"$KMIM_WORKSPACE_PATH\" $CODESIGNING_SUPPRESSION $BUILD_OPTIONS -scheme Keyman SYMROOT=\"$KM4MIM_BASE_PATH/build\"" +builder_run_action test:help check-markdown "$KEYMAN_ROOT/mac/docs/help" builder_run_action install do_install diff --git a/package-lock.json b/package-lock.json index 912bb6727f3..c5f10117dcc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "root", "workspaces": [ "resources/gosh", + "resources/tools/check-markdown", "resources/tools/strip-emoji", "resources/build/version", "core/include/ldml", @@ -6484,6 +6485,10 @@ "url": "https://github.com/chalk/chalk-template?sponsor=1" } }, + "node_modules/check-markdown": { + "resolved": "resources/tools/check-markdown", + "link": true + }, "node_modules/chokidar": { "version": "3.5.1", "dev": true, @@ -15290,6 +15295,95 @@ "gosh": "gosh.js" } }, + "resources/tools/check-markdown": { + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "chalk": "^2.4.2", + "marked": "^14.1.2" + } + }, + "resources/tools/check-markdown/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "resources/tools/check-markdown/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "resources/tools/check-markdown/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/marked": { + "version": "14.1.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-14.1.2.tgz", + "integrity": "sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "resources/tools/check-markdown/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "resources/tools/strip-emoji": { "name": "stripemoji", "version": "1.0.0", diff --git a/package.json b/package.json index 2c06c1652c8..51228544692 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "scripts": {}, "workspaces": [ "resources/gosh", + "resources/tools/check-markdown", "resources/tools/strip-emoji", "resources/build/version", "core/include/ldml", diff --git a/resources/shellHelperFunctions.sh b/resources/shellHelperFunctions.sh index a3f419382e9..6f8e1b52681 100755 --- a/resources/shellHelperFunctions.sh +++ b/resources/shellHelperFunctions.sh @@ -301,3 +301,7 @@ _select_node_version_with_nvm() { builder_die "Attempted to select node.js version $REQUIRED_NODE_VERSION but found $CURRENT_NODE_VERSION instead" fi } + +check-markdown() { + node "$KEYMAN_ROOT/resources/tools/check-markdown" --root "$1" +} \ No newline at end of file diff --git a/resources/tools/check-markdown/.gitignore b/resources/tools/check-markdown/.gitignore new file mode 100644 index 00000000000..d16386367f7 --- /dev/null +++ b/resources/tools/check-markdown/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/resources/tools/check-markdown/build.sh b/resources/tools/check-markdown/build.sh new file mode 100755 index 00000000000..4194129a25f --- /dev/null +++ b/resources/tools/check-markdown/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +################################ Main script ################################ + +builder_describe "Check markdown internal links" \ + "clean" \ + "configure" \ + "build" + +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + +builder_parse "$@" + +builder_run_action clean rm -rf build/ +builder_run_action configure verify_npm_setup +builder_run_action build tsc --build +# builder_run_action test mocha diff --git a/resources/tools/check-markdown/package.json b/resources/tools/check-markdown/package.json new file mode 100644 index 00000000000..16c85f65a49 --- /dev/null +++ b/resources/tools/check-markdown/package.json @@ -0,0 +1,11 @@ +{ + "name": "check-markdown", + "version": "1.0.0", + "type": "module", + "main": "build/index.js", + "license": "MIT", + "devDependencies": { + "marked": "^14.1.2", + "chalk": "^2.4.2" + } +} diff --git a/resources/tools/check-markdown/src/check-link.ts b/resources/tools/check-markdown/src/check-link.ts new file mode 100644 index 00000000000..1eb858fe8c3 --- /dev/null +++ b/resources/tools/check-markdown/src/check-link.ts @@ -0,0 +1,59 @@ +import * as fs from 'node:fs'; +import { posix as path } from 'node:path'; + +import { Tokens } from 'marked'; + +import { LinkRef, LinkRefMessage } from './types.js'; + +export function checkLinks(root: string, links: LinkRef[]) { + let result = true; + for(const file of links) { + for(const link of file.links) { + // verify that the file exists in filesystem at expected location + result = checkLink(root, file.file, link, file.messages) && result; + } + } + return result; +} + +function checkLink(root: string, file: string, token: Tokens.Link | Tokens.Image, messages: LinkRefMessage[]) { + const parsed = token.href.split('#'); + const href = parsed[0]; + // const anchor = parsed.length > 1 ? parsed[1] : ''; + + if(href.startsWith('https:') || href.startsWith('http:')) { + messages.push({token, type:'info', message: 'External link'}); + return true; + + } + if(href.startsWith('/')) { + messages.push({token, type:'info', message: 'Absolute path'}); + return true; + } + + const p = path.normalize(path.join(path.dirname(file), href)); + if(p.startsWith('../') || p == '..') { + messages.push({token, type:'info', message: 'Relative path outside root'}); + return true; + } + + let fullPath = path.join(root, p); + if(token.type == 'link') { + if(fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) { + fullPath = path.join(fullPath, 'index.md'); + } else if(fullPath.endsWith('.md')) { + messages.push({token, type:'warning', message: 'Link should not have a .md extension'}); + } else { + fullPath = fullPath + '.md'; + } + } + + if(!fs.existsSync(fullPath)) { + fullPath = path.relative(root, fullPath); + messages.push({token, type:'error', message: `Link target '${fullPath}' does not exist`}); + return false; + } + + // TODO: check anchor + return true; +} diff --git a/resources/tools/check-markdown/src/find-files.ts b/resources/tools/check-markdown/src/find-files.ts new file mode 100644 index 00000000000..76c7f1b5775 --- /dev/null +++ b/resources/tools/check-markdown/src/find-files.ts @@ -0,0 +1,9 @@ +import * as fs from 'node:fs'; + +export function findFiles(root: string) { + const files: string[] = fs.readdirSync(root, { + recursive: true, encoding: 'utf-8' + }).map(file => file.replace(/\\/g, '/')); + + return files; +} diff --git a/resources/tools/check-markdown/src/index.ts b/resources/tools/check-markdown/src/index.ts new file mode 100644 index 00000000000..4ae8a8e1185 --- /dev/null +++ b/resources/tools/check-markdown/src/index.ts @@ -0,0 +1,49 @@ +import { posix as path } from 'node:path'; + +import chalk from 'chalk'; +import { Command } from 'commander'; + +import { severityColors, type LinkRef } from './types.js'; +import { checkLinks } from './check-link.js'; +import { findFiles } from './find-files.js'; +import { parseFiles } from './parse-files.js'; + +const program = new Command(); +const command = program + .description('Markdown link and sanity checker') + .option('-r, --root ', 'Root path to check') + .option('-v, --verbose', 'Report on external links and warnings') + .action(run); + +program.parse(process.argv); + +function run() { + const root = command.opts().root; + const verbose = program.opts().verbose; + const files = findFiles(root); + const links = parseFiles(root, files); + const result = checkLinks(root, links); + + for(const file of links) { + if(file.messages.length) { + reportMessages(root, result && verbose /* only give verbose output if no errors */, file); + } + } + + process.exit(result ? 0 : 1); +}{ + +} +function reportMessages(root: string, verbose: boolean, file: LinkRef) { + for(const message of file.messages) { + if(message.type == 'error' || verbose) { + process.stdout.write( + chalk.cyan(path.join(root, file.file)) + ' - ' + + severityColors[message.type](message.type) + ': ' + + message.message + + chalk.grey(' [' + message.token.text + '](' + message.token.href + ')') + + '\n' + ); + } + } +} diff --git a/resources/tools/check-markdown/src/parse-files.ts b/resources/tools/check-markdown/src/parse-files.ts new file mode 100644 index 00000000000..ea98db1bab9 --- /dev/null +++ b/resources/tools/check-markdown/src/parse-files.ts @@ -0,0 +1,30 @@ +import * as fs from 'node:fs'; +import { posix as path } from 'node:path'; + +import { marked, Token } from 'marked'; + +import { type LinkRef } from './types.js'; + +let refLinks: any[] = []; + +const walkTokens = (token: Token) => { + if (token.type === 'link' || token.type === 'image') { + refLinks.push(token); + } +}; + +marked.use( { walkTokens }); + +export function parseFiles(root: string, files: string[]): LinkRef[] { + const links: LinkRef[] = []; + for (const file of files) { + const fullPath = path.join(root, file); + if (fs.statSync(fullPath).isFile() && fullPath.endsWith('.md')) { + refLinks = []; + marked.parse(fs.readFileSync(fullPath, 'utf-8')); + links.push({ file, links: refLinks, messages: [] }); + } + } + + return links; +} diff --git a/resources/tools/check-markdown/src/types.ts b/resources/tools/check-markdown/src/types.ts new file mode 100644 index 00000000000..f5aa8233e15 --- /dev/null +++ b/resources/tools/check-markdown/src/types.ts @@ -0,0 +1,18 @@ +import chalk from 'chalk'; +import { Token, Tokens } from 'marked'; + +export type MessageType = 'info' | 'warning' | 'error'; + +export const severityColors: {[value in MessageType]: chalk.Chalk} = { + 'info': chalk.reset, + 'warning': chalk.hex('FFA500'), // orange + 'error': chalk.redBright, +}; + +export interface LinkRefMessage { + type: MessageType; + message: string; + token: Tokens.Link | Tokens.Image; +}; + +export interface LinkRef {file: string, links: Token[], messages: LinkRefMessage[]}; diff --git a/resources/tools/check-markdown/tsconfig.json b/resources/tools/check-markdown/tsconfig.json new file mode 100644 index 00000000000..74778aad7c3 --- /dev/null +++ b/resources/tools/check-markdown/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.base.json", + + "compilerOptions": { + "baseUrl": "./", + "outDir": "./build/", + "rootDir": "src/", + }, +} diff --git a/windows/src/build.sh b/windows/src/build.sh index 4ef6da65b86..c6768a6d92c 100755 --- a/windows/src/build.sh +++ b/windows/src/build.sh @@ -5,9 +5,13 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + builder_describe \ "Keyman for Windows" \ \ + "@/resources/tools/check-markdown test:help" \ + \ clean \ configure \ build \ @@ -17,6 +21,7 @@ builder_describe \ \ ":engine Keyman Engine for Windows" \ ":desktop Keyman for Windows" \ + ":help Online documentation" \ ":components=global/delphi Delphi components" \ ":support Support tools" \ ":test=test/unit-tests Shared unit tests" \ @@ -24,4 +29,6 @@ builder_describe \ builder_parse "$@" -builder_run_child_actions clean configure build test publish install +builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/windows/docs/help" +builder_run_child_actions publish install From b209cd2505ea9e0ce881cea7fd54f1b48ed53010 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 26 Sep 2024 08:36:56 -0700 Subject: [PATCH 02/11] chore(common): check-markdown tweaks --- resources/tools/check-markdown/build.sh | 2 +- resources/tools/check-markdown/src/index.ts | 9 ++++++--- resources/tools/check-markdown/src/types.ts | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/resources/tools/check-markdown/build.sh b/resources/tools/check-markdown/build.sh index 4194129a25f..607be1dea63 100755 --- a/resources/tools/check-markdown/build.sh +++ b/resources/tools/check-markdown/build.sh @@ -20,7 +20,7 @@ builder_describe_outputs \ builder_parse "$@" -builder_run_action clean rm -rf build/ +builder_run_action clean rm -rf build/ tsconfig.tsbuildinfo builder_run_action configure verify_npm_setup builder_run_action build tsc --build # builder_run_action test mocha diff --git a/resources/tools/check-markdown/src/index.ts b/resources/tools/check-markdown/src/index.ts index 4ae8a8e1185..cf7e452bcff 100644 --- a/resources/tools/check-markdown/src/index.ts +++ b/resources/tools/check-markdown/src/index.ts @@ -8,10 +8,12 @@ import { checkLinks } from './check-link.js'; import { findFiles } from './find-files.js'; import { parseFiles } from './parse-files.js'; +const color = chalk.default; + const program = new Command(); const command = program .description('Markdown link and sanity checker') - .option('-r, --root ', 'Root path to check') + .requiredOption('-r, --root ', 'Root path to check') .option('-v, --verbose', 'Report on external links and warnings') .action(run); @@ -20,6 +22,7 @@ program.parse(process.argv); function run() { const root = command.opts().root; const verbose = program.opts().verbose; + const files = findFiles(root); const links = parseFiles(root, files); const result = checkLinks(root, links); @@ -38,10 +41,10 @@ function reportMessages(root: string, verbose: boolean, file: LinkRef) { for(const message of file.messages) { if(message.type == 'error' || verbose) { process.stdout.write( - chalk.cyan(path.join(root, file.file)) + ' - ' + + color.cyan(path.join(root, file.file)) + ' - ' + severityColors[message.type](message.type) + ': ' + message.message + - chalk.grey(' [' + message.token.text + '](' + message.token.href + ')') + + color.grey(' [' + message.token.text + '](' + message.token.href + ')') + '\n' ); } diff --git a/resources/tools/check-markdown/src/types.ts b/resources/tools/check-markdown/src/types.ts index f5aa8233e15..7f310674cc7 100644 --- a/resources/tools/check-markdown/src/types.ts +++ b/resources/tools/check-markdown/src/types.ts @@ -1,12 +1,14 @@ import chalk from 'chalk'; import { Token, Tokens } from 'marked'; +const color = chalk.default; + export type MessageType = 'info' | 'warning' | 'error'; export const severityColors: {[value in MessageType]: chalk.Chalk} = { - 'info': chalk.reset, - 'warning': chalk.hex('FFA500'), // orange - 'error': chalk.redBright, + 'info': color.reset, + 'warning': color.hex('FFA500'), // orange + 'error': color.redBright, }; export interface LinkRefMessage { From cccf3d5dc94fe5b4da3dd6e0abf776655ef3704d Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Tue, 8 Oct 2024 15:00:57 +0700 Subject: [PATCH 03/11] chore(common): avoid adding .md extension to files that exist already --- resources/tools/check-markdown/src/check-link.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/tools/check-markdown/src/check-link.ts b/resources/tools/check-markdown/src/check-link.ts index 1eb858fe8c3..9a99c33a7db 100644 --- a/resources/tools/check-markdown/src/check-link.ts +++ b/resources/tools/check-markdown/src/check-link.ts @@ -43,7 +43,8 @@ function checkLink(root: string, file: string, token: Tokens.Link | Tokens.Image fullPath = path.join(fullPath, 'index.md'); } else if(fullPath.endsWith('.md')) { messages.push({token, type:'warning', message: 'Link should not have a .md extension'}); - } else { + } else if(!fs.existsSync(fullPath)) { + // TODO: consider testing other file extensions in future? fullPath = fullPath + '.md'; } } From 016a6f0225538d0973b79320f370c324386339e3 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 9 Oct 2024 08:12:21 +0700 Subject: [PATCH 04/11] chore(common): address review comments --- android/build.sh | 2 +- developer/src/build.sh | 2 +- ios/build.sh | 7 +------ mac/build.sh | 4 ---- resources/tools/check-markdown/src/index.ts | 6 +++--- 5 files changed, 6 insertions(+), 15 deletions(-) diff --git a/android/build.sh b/android/build.sh index ef75bb6b46a..859fcc56ff6 100755 --- a/android/build.sh +++ b/android/build.sh @@ -14,11 +14,11 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Build Keyman Engine for Android, Keyman for Android, and FirstVoices Android app." \ + "@/resources/tools/check-markdown test:help" \ clean \ configure \ build \ test \ - "@/resources/tools/check-markdown test:help" \ "publish Publishes symbols to Sentry and the APKs to the Play Store." \ --ci+ \ --upload-sentry+ \ diff --git a/developer/src/build.sh b/developer/src/build.sh index f2408d7a6a7..bbae5f629ef 100755 --- a/developer/src/build.sh +++ b/developer/src/build.sh @@ -11,11 +11,11 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Keyman Developer" \ + "@/resources/tools/check-markdown test:help" \ clean \ configure \ build \ test \ - "@/resources/tools/check-markdown test:help" \ "api Analyze API and prepare API documentation" \ "publish Prepare files for distribution, publish symbols, publish or pack npm packages, and build installer" \ "install Install built programs locally" \ diff --git a/ios/build.sh b/ios/build.sh index 1d650faa651..fe007760369 100755 --- a/ios/build.sh +++ b/ios/build.sh @@ -1,23 +1,18 @@ #!/usr/bin/env bash - ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE -# Include our resource functions; they're pretty useful! . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -# Please note that this build script (understandably) assumes that it is running on Mac OS X. -# verify_on_mac - builder_describe "Builds Keyman Engine and the Keyman app for use on iOS devices - iPhone and iPad." \ + "@/resources/tools/check-markdown test:help" \ "clean" \ "configure" \ "build" \ "test" \ - "@/resources/tools/check-markdown test:help" \ ":engine Builds KeymanEngine.xcframework, usable by our main app and by third-party apps" \ ":app=keyman Builds the Keyman app for iOS platforms" \ ":help Online documentation" \ diff --git a/mac/build.sh b/mac/build.sh index e052106edde..8f17ba868d2 100755 --- a/mac/build.sh +++ b/mac/build.sh @@ -5,7 +5,6 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE -# Include our resource functions; they're pretty useful! . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" . "$KEYMAN_ROOT/resources/build/build-help.inc.sh" . "$KEYMAN_ROOT/mac/mac-utils.inc.sh" @@ -25,9 +24,6 @@ builder_describe "Builds Keyman for macOS." \ ":testapp Keyman4Mac (test harness)" \ "--quick,-q Bypasses notarization for $(builder_term install)" -# Please note that this build script (understandably) assumes that it is running on Mac OS X. -# verify_on_mac - builder_parse "$@" # Default is release build of Engine and (code-signed) Input Method diff --git a/resources/tools/check-markdown/src/index.ts b/resources/tools/check-markdown/src/index.ts index cf7e452bcff..0ca8f25320c 100644 --- a/resources/tools/check-markdown/src/index.ts +++ b/resources/tools/check-markdown/src/index.ts @@ -25,15 +25,15 @@ function run() { const files = findFiles(root); const links = parseFiles(root, files); - const result = checkLinks(root, links); + const checkLinksSucceeded = checkLinks(root, links); for(const file of links) { if(file.messages.length) { - reportMessages(root, result && verbose /* only give verbose output if no errors */, file); + reportMessages(root, checkLinksSucceeded && verbose /* only give verbose output if no errors */, file); } } - process.exit(result ? 0 : 1); + process.exit(checkLinksSucceeded ? 0 : 1); }{ } From eed9b8128486b3325ba20e2bc86523e2b1422a52 Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Wed, 9 Oct 2024 08:16:16 +0700 Subject: [PATCH 05/11] chore(common): address review comments --- resources/tools/check-markdown/README.md | 20 +++++++++++++++++++ .../tools/check-markdown/src/check-link.ts | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 resources/tools/check-markdown/README.md diff --git a/resources/tools/check-markdown/README.md b/resources/tools/check-markdown/README.md new file mode 100644 index 00000000000..0f01787c105 --- /dev/null +++ b/resources/tools/check-markdown/README.md @@ -0,0 +1,20 @@ +# check-markdown + +This tool is used to test the validity of internal links within product +documentation, e.g. `/android/docs/help/**/*.md`. + +It currently tests that: + +1. Markdown can be parsed +2. Links to other files in the same section exist (with or without .md extension) +3. Images exist + +It will also optionally report on: + +1. External absolute links (starting with http/https) +2. Relative links outside the root of the help documentation +3. Unnecessary use of .md extension in links + +We could extend it to include: + +1. Checks for anchor validity diff --git a/resources/tools/check-markdown/src/check-link.ts b/resources/tools/check-markdown/src/check-link.ts index 9a99c33a7db..08df0b001d2 100644 --- a/resources/tools/check-markdown/src/check-link.ts +++ b/resources/tools/check-markdown/src/check-link.ts @@ -24,8 +24,8 @@ function checkLink(root: string, file: string, token: Tokens.Link | Tokens.Image if(href.startsWith('https:') || href.startsWith('http:')) { messages.push({token, type:'info', message: 'External link'}); return true; - } + if(href.startsWith('/')) { messages.push({token, type:'info', message: 'Absolute path'}); return true; From 044256f2915d0dbc7c7696988e3424a93b920dac Mon Sep 17 00:00:00 2001 From: Eberhard Beilharz Date: Thu, 10 Oct 2024 11:57:45 +0200 Subject: [PATCH 06/11] chore(linux): improve output if dpkg-gensymbols fails and run other tests --- .github/workflows/api-verification.yml | 2 +- linux/scripts/verify_api.inc.sh | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/api-verification.yml b/.github/workflows/api-verification.yml index de7e967b24f..443adb471f3 100644 --- a/.github/workflows/api-verification.yml +++ b/.github/workflows/api-verification.yml @@ -67,7 +67,7 @@ jobs: - name: "Verify API for libkeymancore*.so (${{ steps.environment_step.outputs.GIT_BRANCH }}, branch ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}, by ${{ steps.environment_step.outputs.GIT_USER }})" run: | - echo "Verify API for libkeymancore*.so (${{ steps.environment_step.outputs.GIT_BRANCH }}, branch ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}, by ${{ steps.environment_step.outputs.GIT_USER }})" >> $GITHUB_STEP_SUMMARY + echo "Verify API for libkeymancore*.so (${{ steps.environment_step.outputs.GIT_BRANCH }}, branch ${{ steps.environment_step.outputs.GIT_BASE_BRANCH }}, by ${{ steps.environment_step.outputs.GIT_USER }}):" >> $GITHUB_STEP_SUMMARY BIN_PACKAGE=$(ls "${GITHUB_WORKSPACE}/artifacts/" | grep "${PKG_NAME}[0-9]*_${{ steps.environment_step.outputs.VERSION }}-1${{ steps.environment_step.outputs.PRERELEASE_TAG }}+$(lsb_release -c -s)1_amd64.deb") cd ${{ github.workspace }}/keyman/linux diff --git a/linux/scripts/verify_api.inc.sh b/linux/scripts/verify_api.inc.sh index adaa9187c27..8334bc84b89 100644 --- a/linux/scripts/verify_api.inc.sh +++ b/linux/scripts/verify_api.inc.sh @@ -32,8 +32,12 @@ check_api_not_changed() { trap "rm -rf \"${tmpDir}\"" ERR dpkg -x "${BIN_PKG}" "${tmpDir}" mkdir -p debian/tmp/DEBIAN - dpkg-gensymbols -v"${VERSION}" -p"${PKG_NAME}" -e"${tmpDir}"/usr/lib/x86_64-linux-gnu/"${LIB_NAME}".so* -c4 - output_ok "${LIB_NAME} API didn't change" + if dpkg-gensymbols -v"${VERSION}" -p"${PKG_NAME}" -e"${tmpDir}"/usr/lib/x86_64-linux-gnu/"${LIB_NAME}".so* -c4; then + output_ok "${LIB_NAME} API didn't change" + else + output_error "${LIB_NAME} API changed" + EXIT_CODE=4 + fi cd "${REPO_ROOT}/linux" rm -rf "${tmpDir}" trap ERR From 7dc787b96bcf9e86d6370d7c78bbfd6a196afadc Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Fri, 11 Oct 2024 14:11:45 +0700 Subject: [PATCH 07/11] fix(developer): use TextDecoder to convert Uint8Array to string `toString()` only works when the Uint8Array is actually a Node Buffer. --- developer/src/common/web/utils/src/types/kpj/kpj-file-reader.ts | 2 +- .../src/common/web/utils/src/types/kvks/kvks-file-reader.ts | 2 +- .../utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts | 2 +- developer/src/kmc-package/src/compiler/kmp-compiler.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/developer/src/common/web/utils/src/types/kpj/kpj-file-reader.ts b/developer/src/common/web/utils/src/types/kpj/kpj-file-reader.ts index f86879b0068..8baeaef612b 100644 --- a/developer/src/common/web/utils/src/types/kpj/kpj-file-reader.ts +++ b/developer/src/common/web/utils/src/types/kpj/kpj-file-reader.ts @@ -14,7 +14,7 @@ export class KPJFileReader { let data: KPJFile; data = new KeymanXMLReader('kpj') - .parse(file.toString()); + .parse(new TextDecoder().decode(file)); data = this.boxArrays(data); if(data.KeymanDeveloperProject?.Files?.File?.length) { diff --git a/developer/src/common/web/utils/src/types/kvks/kvks-file-reader.ts b/developer/src/common/web/utils/src/types/kvks/kvks-file-reader.ts index 9f2ca18f887..1048d7dc888 100644 --- a/developer/src/common/web/utils/src/types/kvks/kvks-file-reader.ts +++ b/developer/src/common/web/utils/src/types/kvks/kvks-file-reader.ts @@ -22,7 +22,7 @@ export default class KVKSFileReader { try { source = new KeymanXMLReader('kvks') - .parse(file.toString()) as KVKSourceFile; + .parse(new TextDecoder().decode(file)) as KVKSourceFile; } catch(e) { if(file.byteLength > 4 && file.subarray(0,3).every((v,i) => v == KVK_HEADER_IDENTIFIER_BYTES[i])) { throw new Error('File appears to be a binary .kvk file', {cause: e}); diff --git a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts index 17ade173931..92669716ea1 100644 --- a/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts +++ b/developer/src/common/web/utils/src/types/ldml-keyboard/ldml-keyboard-xml-reader.ts @@ -302,7 +302,7 @@ export class LDMLKeyboardXMLSourceFileReader { loadTestDataUnboxed(file: Uint8Array): any { const source = new KeymanXMLReader('keyboardTest3') - .parse(file.toString()) as any; + .parse(new TextDecoder().decode(file)) as any; return source; } diff --git a/developer/src/kmc-package/src/compiler/kmp-compiler.ts b/developer/src/kmc-package/src/compiler/kmp-compiler.ts index 1708d69704c..e256c37b88d 100644 --- a/developer/src/kmc-package/src/compiler/kmp-compiler.ts +++ b/developer/src/kmc-package/src/compiler/kmp-compiler.ts @@ -183,7 +183,7 @@ export class KmpCompiler implements KeymanCompiler { try { a = new KeymanXMLReader('kps') - .parse(data.toString()) as KpsFile.KpsPackage; + .parse(data) as KpsFile.KpsPackage; } catch(e) { this.callbacks.reportMessage(PackageCompilerMessages.Error_InvalidPackageFile({e})); } From bb9ccfdaa9847fa1195dc6d59e4ea8e7f215ffb9 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Fri, 11 Oct 2024 13:02:22 -0400 Subject: [PATCH 08/11] auto: increment master version to 18.0.127 --- HISTORY.md | 5 +++++ VERSION.md | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index b442c74e502..e3002e327c6 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,10 @@ # Keyman Version History +## 18.0.126 alpha 2024-10-11 + +* test(common): add markdown link check test for product documentation (#12472) +* chore(linux): improve output if `dpkg-gensymbols` fails and run other tests (#12527) + ## 18.0.125 alpha 2024-10-10 * chore(common): allow to run `build.sh` scripts in `bashdb` debugger (#12518) diff --git a/VERSION.md b/VERSION.md index 689f7b264ea..af094556ac4 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.126 \ No newline at end of file +18.0.127 \ No newline at end of file From 8ad0ea4fb280bff385e944affeced30807c37892 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Sat, 12 Oct 2024 14:02:25 -0400 Subject: [PATCH 09/11] auto: increment master version to 18.0.128 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index e3002e327c6..1451c5ed027 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 18.0.127 alpha 2024-10-12 + +* fix(developer): use TextDecoder to convert Uint8Array to string (#12537) + ## 18.0.126 alpha 2024-10-11 * test(common): add markdown link check test for product documentation (#12472) diff --git a/VERSION.md b/VERSION.md index af094556ac4..be701dd746a 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.127 \ No newline at end of file +18.0.128 \ No newline at end of file From c6893c701ef81250728488fbca652b312eb43c9e Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Mon, 14 Oct 2024 05:10:54 +0700 Subject: [PATCH 10/11] docs: add refs to Keyman MIME types Fixes: #10967 --- developer/docs/help/reference/file-types/kmp.md | 2 ++ developer/docs/help/reference/file-types/kmx.md | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/developer/docs/help/reference/file-types/kmp.md b/developer/docs/help/reference/file-types/kmp.md index b79bd663be4..034de23b4d5 100644 --- a/developer/docs/help/reference/file-types/kmp.md +++ b/developer/docs/help/reference/file-types/kmp.md @@ -21,6 +21,8 @@ Details: Developer will also include a [metadata](metadata) file in the package. Lexical model packages contain one lexical model instead of keyboards. +: .kmp file format has registered MIME type + [`application/vnd.keyman.kmp+zip`](https://www.iana.org/assignments/media-types/application/vnd.keyman.kmp+zip). Distributed with keyboard: : A Keyman keyboard Package file (.KMP) can include keyboard files diff --git a/developer/docs/help/reference/file-types/kmx.md b/developer/docs/help/reference/file-types/kmx.md index a4351688a52..062c18a2a9d 100644 --- a/developer/docs/help/reference/file-types/kmx.md +++ b/developer/docs/help/reference/file-types/kmx.md @@ -3,7 +3,7 @@ title: KMX files --- Used by: -: Keyman Desktop, +: Keyman Core in Keyman for Windows, Keyman for macOS, and Keyman for Linux. @@ -16,6 +16,8 @@ Details: for the keyboard (.ICO/.BMP). Note that .KMX files are not used for Keyman for Android and Keyman for iPhone and iPad. +: .KMX file format has registered MIME type + [`application/vnd.keyman.kmx`](https://www.iana.org/assignments/media-types/application/vnd.keyman.kmx). Distributed with keyboard: : This is the keyboard file. It must be distributed with your package From d0a70d006f3fed7d7c4af97d51acfc39e0162f18 Mon Sep 17 00:00:00 2001 From: Keyman Build Agent Date: Mon, 14 Oct 2024 14:02:10 -0400 Subject: [PATCH 11/11] auto: increment master version to 18.0.129 --- HISTORY.md | 4 ++++ VERSION.md | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 1451c5ed027..e46274077fe 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,5 +1,9 @@ # Keyman Version History +## 18.0.128 alpha 2024-10-14 + +* docs(developer): add refs to Keyman MIME types (#12540) + ## 18.0.127 alpha 2024-10-12 * fix(developer): use TextDecoder to convert Uint8Array to string (#12537) diff --git a/VERSION.md b/VERSION.md index be701dd746a..494d2a8106d 100644 --- a/VERSION.md +++ b/VERSION.md @@ -1 +1 @@ -18.0.128 \ No newline at end of file +18.0.129 \ No newline at end of file