diff --git a/android/build.sh b/android/build.sh index d876d882d77..859fcc56ff6 100755 --- a/android/build.sh +++ b/android/build.sh @@ -14,6 +14,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Build Keyman Engine for Android, Keyman for Android, and FirstVoices Android app." \ + "@/resources/tools/check-markdown test:help" \ clean \ configure \ build \ @@ -23,6 +24,7 @@ builder_describe \ --upload-sentry+ \ ":engine=KMEA Keyman Engine for Android" \ ":app=KMAPro Keyman for Android" \ + ":help Online documentation" \ ":sample1=Samples/KMSample1 Sample app: KMSample1" \ ":sample2=Samples/KMSample2 Sample app: KMSample2" \ ":keyboardharness=Tests/KeyboardHarness Test app: KeyboardHarness" \ @@ -42,4 +44,7 @@ if builder_start_action clean; then builder_finish_action success clean fi -builder_run_child_actions configure build test publish +builder_run_child_actions configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/android/docs/help" + +builder_run_child_actions publish \ No newline at end of file diff --git a/developer/src/build.sh b/developer/src/build.sh index cde9b7fc233..bbae5f629ef 100755 --- a/developer/src/build.sh +++ b/developer/src/build.sh @@ -11,6 +11,7 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" builder_describe \ "Keyman Developer" \ + "@/resources/tools/check-markdown test:help" \ clean \ configure \ build \ @@ -20,6 +21,7 @@ builder_describe \ "install Install built programs locally" \ ":common Developer common files" \ ":ext Third party components" \ + ":help Online documentation" \ ":kmcmplib Compiler - .kmn compiler" \ ":kmc-analyze Compiler - Analysis Tools" \ ":kmc-keyboard-info Compiler - .keyboard_info Module" \ @@ -125,6 +127,7 @@ fi #------------------------------------------------------------------------------------------------------------------- builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/developer/docs/help" #------------------------------------------------------------------------------------------------------------------- diff --git a/ios/build.sh b/ios/build.sh index 21a0be28277..fe007760369 100755 --- a/ios/build.sh +++ b/ios/build.sh @@ -1,23 +1,21 @@ #!/usr/bin/env bash - ## START STANDARD BUILD SCRIPT INCLUDE # adjust relative paths as necessary THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE -# Include our resource functions; they're pretty useful! . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" -# Please note that this build script (understandably) assumes that it is running on Mac OS X. -verify_on_mac - builder_describe "Builds Keyman Engine and the Keyman app for use on iOS devices - iPhone and iPad." \ + "@/resources/tools/check-markdown test:help" \ "clean" \ "configure" \ "build" \ + "test" \ ":engine Builds KeymanEngine.xcframework, usable by our main app and by third-party apps" \ ":app=keyman Builds the Keyman app for iOS platforms" \ + ":help Online documentation" \ ":sample1=Samples/KMSample1 Builds the first KeymanEngine sample app" \ ":sample2=Samples/KMSample2 Builds the second KeymanEngine sample app" \ ":fv=../oem/firstvoices/ios Builds OEM FirstVoices for iOS platforms" \ @@ -25,4 +23,5 @@ builder_describe "Builds Keyman Engine and the Keyman app for use on iOS devices builder_parse "$@" -builder_run_child_actions clean configure build \ No newline at end of file +builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/ios/docs/help" diff --git a/linux/build.sh b/linux/build.sh index 1012b7eb1f0..6b97c287f5a 100755 --- a/linux/build.sh +++ b/linux/build.sh @@ -6,12 +6,16 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + ################################ Main script ################################ builder_describe \ "Build Keyman for Linux." \ + "@/resources/tools/check-markdown test:help" \ ":config=keyman-config keyman-config" \ ":engine=ibus-keyman ibus-keyman" \ + ":help Online documentation" \ ":service=keyman-system-service keyman-system-service" \ "clean" \ "configure" \ @@ -36,3 +40,4 @@ test_action() { } builder_run_action test test_action +builder_run_action test:help check-markdown "$KEYMAN_ROOT/linux/docs/help" diff --git a/mac/build.sh b/mac/build.sh index fbca48bd05f..8f17ba868d2 100755 --- a/mac/build.sh +++ b/mac/build.sh @@ -5,13 +5,13 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE -# Include our resource functions; they're pretty useful! . "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" . "$KEYMAN_ROOT/resources/build/build-help.inc.sh" . "$KEYMAN_ROOT/mac/mac-utils.inc.sh" builder_describe "Builds Keyman for macOS." \ "@/core:mac" \ + "@/resources/tools/check-markdown test:help" \ "clean" \ "configure" \ "build" \ @@ -20,12 +20,10 @@ builder_describe "Builds Keyman for macOS." \ "install Installs result of Keyman4MacIM locally." \ ":engine KeymanEngine4Mac" \ ":app Keyman4MacIM" \ + ":help Online documentation" \ ":testapp Keyman4Mac (test harness)" \ "--quick,-q Bypasses notarization for $(builder_term install)" -# Please note that this build script (understandably) assumes that it is running on Mac OS X. -verify_on_mac - builder_parse "$@" # Default is release build of Engine and (code-signed) Input Method @@ -89,8 +87,8 @@ UPLOAD_SENTRY=false # Import local environment variables for build # -# /mac/localenv.sh can be used to define CERTIFICATE_ID, -# APPSTORECONNECT_PROVIDER, APPSTORECONNECT_USERNAME, +# /mac/localenv.sh can be used to define CERTIFICATE_ID, +# APPSTORECONNECT_PROVIDER, APPSTORECONNECT_USERNAME, # APPSTORECONNECT_PASSWORD, DEVELOPMENT_TEAM variables; # see /mac/README.md for details. # @@ -304,6 +302,7 @@ builder_run_action build:testapp do_build_testapp builder_run_action test:engine execBuildCommand $ENGINE_NAME "xcodebuild -project \"$KME4M_PROJECT_PATH\" $BUILD_OPTIONS test -scheme $ENGINE_NAME" builder_run_action test:app execBuildCommand "$IM_NAME-tests" "xcodebuild test -workspace \"$KMIM_WORKSPACE_PATH\" $CODESIGNING_SUPPRESSION $BUILD_OPTIONS -scheme Keyman SYMROOT=\"$KM4MIM_BASE_PATH/build\"" +builder_run_action test:help check-markdown "$KEYMAN_ROOT/mac/docs/help" builder_run_action install do_install diff --git a/package-lock.json b/package-lock.json index 8cf9c5d5d83..38d3d0ae136 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,6 +7,7 @@ "name": "root", "workspaces": [ "resources/gosh", + "resources/tools/check-markdown", "resources/tools/strip-emoji", "resources/build/version", "core/include/ldml", @@ -6476,6 +6477,10 @@ "url": "https://github.com/chalk/chalk-template?sponsor=1" } }, + "node_modules/check-markdown": { + "resolved": "resources/tools/check-markdown", + "link": true + }, "node_modules/chokidar": { "version": "3.5.1", "dev": true, @@ -15310,6 +15315,95 @@ "gosh": "gosh.js" } }, + "resources/tools/check-markdown": { + "version": "1.0.0", + "license": "MIT", + "devDependencies": { + "chalk": "^2.4.2", + "marked": "^14.1.2" + } + }, + "resources/tools/check-markdown/node_modules/ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^1.9.0" + }, + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + }, + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-name": "1.1.3" + } + }, + "resources/tools/check-markdown/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true, + "license": "MIT" + }, + "resources/tools/check-markdown/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "resources/tools/check-markdown/node_modules/marked": { + "version": "14.1.2", + "resolved": "https://registry.npmjs.org/marked/-/marked-14.1.2.tgz", + "integrity": "sha512-f3r0yqpz31VXiDB/wj9GaOB0a2PRLQl6vJmXiFrniNwjkKdvakqJRULhjFKJpxOchlCRiG5fcacoUZY5Xa6PEQ==", + "dev": true, + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 18" + } + }, + "resources/tools/check-markdown/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "resources/tools/strip-emoji": { "name": "stripemoji", "version": "1.0.0", diff --git a/package.json b/package.json index 2c06c1652c8..51228544692 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "scripts": {}, "workspaces": [ "resources/gosh", + "resources/tools/check-markdown", "resources/tools/strip-emoji", "resources/build/version", "core/include/ldml", diff --git a/resources/shellHelperFunctions.sh b/resources/shellHelperFunctions.sh index a3f419382e9..6f8e1b52681 100755 --- a/resources/shellHelperFunctions.sh +++ b/resources/shellHelperFunctions.sh @@ -301,3 +301,7 @@ _select_node_version_with_nvm() { builder_die "Attempted to select node.js version $REQUIRED_NODE_VERSION but found $CURRENT_NODE_VERSION instead" fi } + +check-markdown() { + node "$KEYMAN_ROOT/resources/tools/check-markdown" --root "$1" +} \ No newline at end of file diff --git a/resources/tools/check-markdown/.gitignore b/resources/tools/check-markdown/.gitignore new file mode 100644 index 00000000000..d16386367f7 --- /dev/null +++ b/resources/tools/check-markdown/.gitignore @@ -0,0 +1 @@ +build/ \ No newline at end of file diff --git a/resources/tools/check-markdown/README.md b/resources/tools/check-markdown/README.md new file mode 100644 index 00000000000..0f01787c105 --- /dev/null +++ b/resources/tools/check-markdown/README.md @@ -0,0 +1,20 @@ +# check-markdown + +This tool is used to test the validity of internal links within product +documentation, e.g. `/android/docs/help/**/*.md`. + +It currently tests that: + +1. Markdown can be parsed +2. Links to other files in the same section exist (with or without .md extension) +3. Images exist + +It will also optionally report on: + +1. External absolute links (starting with http/https) +2. Relative links outside the root of the help documentation +3. Unnecessary use of .md extension in links + +We could extend it to include: + +1. Checks for anchor validity diff --git a/resources/tools/check-markdown/build.sh b/resources/tools/check-markdown/build.sh new file mode 100755 index 00000000000..607be1dea63 --- /dev/null +++ b/resources/tools/check-markdown/build.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +## START STANDARD BUILD SCRIPT INCLUDE +# adjust relative paths as necessary +THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" +. "${THIS_SCRIPT%/*}/../../../resources/build/builder.inc.sh" +## END STANDARD BUILD SCRIPT INCLUDE + +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + +################################ Main script ################################ + +builder_describe "Check markdown internal links" \ + "clean" \ + "configure" \ + "build" + +builder_describe_outputs \ + configure /node_modules \ + build build/index.js + +builder_parse "$@" + +builder_run_action clean rm -rf build/ tsconfig.tsbuildinfo +builder_run_action configure verify_npm_setup +builder_run_action build tsc --build +# builder_run_action test mocha diff --git a/resources/tools/check-markdown/package.json b/resources/tools/check-markdown/package.json new file mode 100644 index 00000000000..16c85f65a49 --- /dev/null +++ b/resources/tools/check-markdown/package.json @@ -0,0 +1,11 @@ +{ + "name": "check-markdown", + "version": "1.0.0", + "type": "module", + "main": "build/index.js", + "license": "MIT", + "devDependencies": { + "marked": "^14.1.2", + "chalk": "^2.4.2" + } +} diff --git a/resources/tools/check-markdown/src/check-link.ts b/resources/tools/check-markdown/src/check-link.ts new file mode 100644 index 00000000000..08df0b001d2 --- /dev/null +++ b/resources/tools/check-markdown/src/check-link.ts @@ -0,0 +1,60 @@ +import * as fs from 'node:fs'; +import { posix as path } from 'node:path'; + +import { Tokens } from 'marked'; + +import { LinkRef, LinkRefMessage } from './types.js'; + +export function checkLinks(root: string, links: LinkRef[]) { + let result = true; + for(const file of links) { + for(const link of file.links) { + // verify that the file exists in filesystem at expected location + result = checkLink(root, file.file, link, file.messages) && result; + } + } + return result; +} + +function checkLink(root: string, file: string, token: Tokens.Link | Tokens.Image, messages: LinkRefMessage[]) { + const parsed = token.href.split('#'); + const href = parsed[0]; + // const anchor = parsed.length > 1 ? parsed[1] : ''; + + if(href.startsWith('https:') || href.startsWith('http:')) { + messages.push({token, type:'info', message: 'External link'}); + return true; + } + + if(href.startsWith('/')) { + messages.push({token, type:'info', message: 'Absolute path'}); + return true; + } + + const p = path.normalize(path.join(path.dirname(file), href)); + if(p.startsWith('../') || p == '..') { + messages.push({token, type:'info', message: 'Relative path outside root'}); + return true; + } + + let fullPath = path.join(root, p); + if(token.type == 'link') { + if(fs.existsSync(fullPath) && fs.statSync(fullPath).isDirectory()) { + fullPath = path.join(fullPath, 'index.md'); + } else if(fullPath.endsWith('.md')) { + messages.push({token, type:'warning', message: 'Link should not have a .md extension'}); + } else if(!fs.existsSync(fullPath)) { + // TODO: consider testing other file extensions in future? + fullPath = fullPath + '.md'; + } + } + + if(!fs.existsSync(fullPath)) { + fullPath = path.relative(root, fullPath); + messages.push({token, type:'error', message: `Link target '${fullPath}' does not exist`}); + return false; + } + + // TODO: check anchor + return true; +} diff --git a/resources/tools/check-markdown/src/find-files.ts b/resources/tools/check-markdown/src/find-files.ts new file mode 100644 index 00000000000..76c7f1b5775 --- /dev/null +++ b/resources/tools/check-markdown/src/find-files.ts @@ -0,0 +1,9 @@ +import * as fs from 'node:fs'; + +export function findFiles(root: string) { + const files: string[] = fs.readdirSync(root, { + recursive: true, encoding: 'utf-8' + }).map(file => file.replace(/\\/g, '/')); + + return files; +} diff --git a/resources/tools/check-markdown/src/index.ts b/resources/tools/check-markdown/src/index.ts new file mode 100644 index 00000000000..0ca8f25320c --- /dev/null +++ b/resources/tools/check-markdown/src/index.ts @@ -0,0 +1,52 @@ +import { posix as path } from 'node:path'; + +import chalk from 'chalk'; +import { Command } from 'commander'; + +import { severityColors, type LinkRef } from './types.js'; +import { checkLinks } from './check-link.js'; +import { findFiles } from './find-files.js'; +import { parseFiles } from './parse-files.js'; + +const color = chalk.default; + +const program = new Command(); +const command = program + .description('Markdown link and sanity checker') + .requiredOption('-r, --root ', 'Root path to check') + .option('-v, --verbose', 'Report on external links and warnings') + .action(run); + +program.parse(process.argv); + +function run() { + const root = command.opts().root; + const verbose = program.opts().verbose; + + const files = findFiles(root); + const links = parseFiles(root, files); + const checkLinksSucceeded = checkLinks(root, links); + + for(const file of links) { + if(file.messages.length) { + reportMessages(root, checkLinksSucceeded && verbose /* only give verbose output if no errors */, file); + } + } + + process.exit(checkLinksSucceeded ? 0 : 1); +}{ + +} +function reportMessages(root: string, verbose: boolean, file: LinkRef) { + for(const message of file.messages) { + if(message.type == 'error' || verbose) { + process.stdout.write( + color.cyan(path.join(root, file.file)) + ' - ' + + severityColors[message.type](message.type) + ': ' + + message.message + + color.grey(' [' + message.token.text + '](' + message.token.href + ')') + + '\n' + ); + } + } +} diff --git a/resources/tools/check-markdown/src/parse-files.ts b/resources/tools/check-markdown/src/parse-files.ts new file mode 100644 index 00000000000..ea98db1bab9 --- /dev/null +++ b/resources/tools/check-markdown/src/parse-files.ts @@ -0,0 +1,30 @@ +import * as fs from 'node:fs'; +import { posix as path } from 'node:path'; + +import { marked, Token } from 'marked'; + +import { type LinkRef } from './types.js'; + +let refLinks: any[] = []; + +const walkTokens = (token: Token) => { + if (token.type === 'link' || token.type === 'image') { + refLinks.push(token); + } +}; + +marked.use( { walkTokens }); + +export function parseFiles(root: string, files: string[]): LinkRef[] { + const links: LinkRef[] = []; + for (const file of files) { + const fullPath = path.join(root, file); + if (fs.statSync(fullPath).isFile() && fullPath.endsWith('.md')) { + refLinks = []; + marked.parse(fs.readFileSync(fullPath, 'utf-8')); + links.push({ file, links: refLinks, messages: [] }); + } + } + + return links; +} diff --git a/resources/tools/check-markdown/src/types.ts b/resources/tools/check-markdown/src/types.ts new file mode 100644 index 00000000000..7f310674cc7 --- /dev/null +++ b/resources/tools/check-markdown/src/types.ts @@ -0,0 +1,20 @@ +import chalk from 'chalk'; +import { Token, Tokens } from 'marked'; + +const color = chalk.default; + +export type MessageType = 'info' | 'warning' | 'error'; + +export const severityColors: {[value in MessageType]: chalk.Chalk} = { + 'info': color.reset, + 'warning': color.hex('FFA500'), // orange + 'error': color.redBright, +}; + +export interface LinkRefMessage { + type: MessageType; + message: string; + token: Tokens.Link | Tokens.Image; +}; + +export interface LinkRef {file: string, links: Token[], messages: LinkRefMessage[]}; diff --git a/resources/tools/check-markdown/tsconfig.json b/resources/tools/check-markdown/tsconfig.json new file mode 100644 index 00000000000..74778aad7c3 --- /dev/null +++ b/resources/tools/check-markdown/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../../tsconfig.base.json", + + "compilerOptions": { + "baseUrl": "./", + "outDir": "./build/", + "rootDir": "src/", + }, +} diff --git a/windows/src/build.sh b/windows/src/build.sh index 4ef6da65b86..c6768a6d92c 100755 --- a/windows/src/build.sh +++ b/windows/src/build.sh @@ -5,9 +5,13 @@ THIS_SCRIPT="$(readlink -f "${BASH_SOURCE[0]}")" . "${THIS_SCRIPT%/*}/../../resources/build/builder.inc.sh" ## END STANDARD BUILD SCRIPT INCLUDE +. "$KEYMAN_ROOT/resources/shellHelperFunctions.sh" + builder_describe \ "Keyman for Windows" \ \ + "@/resources/tools/check-markdown test:help" \ + \ clean \ configure \ build \ @@ -17,6 +21,7 @@ builder_describe \ \ ":engine Keyman Engine for Windows" \ ":desktop Keyman for Windows" \ + ":help Online documentation" \ ":components=global/delphi Delphi components" \ ":support Support tools" \ ":test=test/unit-tests Shared unit tests" \ @@ -24,4 +29,6 @@ builder_describe \ builder_parse "$@" -builder_run_child_actions clean configure build test publish install +builder_run_child_actions clean configure build test +builder_run_action test:help check-markdown "$KEYMAN_ROOT/windows/docs/help" +builder_run_child_actions publish install