diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f7b303f54..26a096660 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: node-version: ${{ matrix.version }} cache: yarn - run: yarn --frozen-lockfile - - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client --lines 80 --per-file yarn test:mocha + - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client -x src/convert.ts --lines 80 --per-file yarn test:mocha - run: yarn test:tsc - run: | echo ::add-matcher::.github/eslint.json diff --git a/bin/observable.ts b/bin/observable.ts index aec619c8a..f286d9cd3 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -62,7 +62,7 @@ else if (values.help) { /** Commands that use Clack formatting. When handling CliErrors, clack.outro() * will be used for these commands. */ -const CLACKIFIED_COMMANDS = ["create", "deploy", "login"]; +const CLACKIFIED_COMMANDS = ["create", "deploy", "login", "convert"]; try { switch (command) { @@ -78,6 +78,7 @@ try { logout sign-out of Observable deploy deploy a project to Observable whoami check authentication status + convert convert an Observable notebook to Markdown help print usage information version print the version` ); @@ -177,6 +178,17 @@ try { await import("../src/observableApiAuth.js").then((auth) => auth.whoami()); break; } + case "convert": { + const { + positionals, + values: {output, force} + } = helpArgs(command, { + options: {output: {type: "string", default: "."}, force: {type: "boolean", short: "f"}}, + allowPositionals: true + }); + await import("../src/convert.js").then((convert) => convert.convert(positionals, {output: output!, force})); + break; + } default: { console.error(`observable: unknown command '${command}'. See 'observable help'.`); process.exit(1); diff --git a/src/convert.ts b/src/convert.ts new file mode 100644 index 000000000..372d3ec3b --- /dev/null +++ b/src/convert.ts @@ -0,0 +1,146 @@ +import {existsSync} from "node:fs"; +import {utimes, writeFile} from "node:fs/promises"; +import {join} from "node:path"; +import * as clack from "@clack/prompts"; +import wrapAnsi from "wrap-ansi"; +import type {ClackEffects} from "./clack.js"; +import {CliError} from "./error.js"; +import {prepareOutput} from "./files.js"; +import {getObservableUiOrigin} from "./observableApiClient.js"; +import {type TtyEffects, bold, cyan, faint, inverse, link, reset, defaultEffects as ttyEffects} from "./tty.js"; + +export interface ConvertEffects extends TtyEffects { + clack: ClackEffects; + prepareOutput(outputPath: string): Promise; + existsSync(outputPath: string): boolean; + writeFile(outputPath: string, contents: Buffer | string): Promise; + touch(outputPath: string, date: Date | string | number): Promise; +} + +const defaultEffects: ConvertEffects = { + ...ttyEffects, + clack, + async prepareOutput(outputPath: string): Promise { + await prepareOutput(outputPath); + }, + existsSync(outputPath: string): boolean { + return existsSync(outputPath); + }, + async writeFile(outputPath: string, contents: Buffer | string): Promise { + await writeFile(outputPath, contents); + }, + async touch(outputPath: string, date: Date | string | number): Promise { + await utimes(outputPath, (date = new Date(date)), date); + } +}; + +export async function convert( + inputs: string[], + {output, force = false, files: includeFiles = true}: {output: string; force?: boolean; files?: boolean}, + effects: ConvertEffects = defaultEffects +): Promise { + const {clack} = effects; + clack.intro(`${inverse(" observable convert ")}`); + let n = 0; + for (const input of inputs) { + let start = Date.now(); + let s = clack.spinner(); + const url = resolveInput(input); + const name = inferFileName(url); + const path = join(output, name); + if (await maybeFetch(path, force, effects)) { + s.start(`Downloading ${bold(path)}`); + const response = await fetch(url); + if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); + const {nodes, files, update_time} = await response.json(); + s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await effects.prepareOutput(path); + await effects.writeFile(path, convertNodes(nodes)); + await effects.touch(path, update_time); + n++; + if (includeFiles) { + for (const file of files) { + const path = join(output, file.name); + if (await maybeFetch(path, force, effects)) { + start = Date.now(); + s = clack.spinner(); + s.start(`Downloading ${bold(file.name)}`); + const response = await fetch(file.download_url); + if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); + const buffer = Buffer.from(await response.arrayBuffer()); + s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await effects.prepareOutput(path); + await effects.writeFile(path, buffer); + await effects.touch(path, file.create_time); + n++; + } + } + } + } + } + clack.note( + wrapAnsi( + "Due to syntax differences between Observable notebooks and " + + "Observable Framework, converted notebooks may require further " + + "changes to function correctly. To learn more about JavaScript " + + "in Framework, please read:\n\n" + + reset(cyan(link("https://observablehq.com/framework/javascript"))), + Math.min(64, effects.outputColumns) + ), + "Note" + ); + clack.outro( + `${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted; ${n} file${n === 1 ? "" : "s"} written` + ); +} + +async function maybeFetch(path: string, force: boolean, effects: ConvertEffects): Promise { + const {clack} = effects; + if (effects.existsSync(path) && !force) { + const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); + if (!choice) return false; + if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false}); + } + return true; +} + +export function convertNodes(nodes): string { + let string = ""; + let first = true; + for (const node of nodes) { + if (first) first = false; + else string += "\n"; + string += convertNode(node); + } + return string; +} + +export function convertNode(node): string { + let string = ""; + if (node.mode !== "md") string += `\`\`\`${node.mode}${node.pinned ? " echo" : ""}\n`; + string += `${node.value}\n`; + if (node.mode !== "md") string += "```\n"; + return string; +} + +export function inferFileName(input: string): string { + return new URL(input).pathname.replace(/^\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md"; +} + +export function resolveInput(input: string): string { + let url: URL; + if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin()); + else if (isSlugSpecifier(input)) url = new URL(`/${input}`, getObservableUiOrigin()); + else url = new URL(input); + url.host = `api.${url.host}`; + url.pathname = `/document${url.pathname.replace(/^\/d\//, "/")}`; + return String(url); +} + +function isIdSpecifier(string: string) { + return /^([0-9a-f]{16})(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string); +} + +function isSlugSpecifier(string: string) { + return /^(?:@([0-9a-z_-]+))\/([0-9a-z_-]+(?:\/[0-9]+)?)(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string); +} diff --git a/test/convert-test.ts b/test/convert-test.ts new file mode 100644 index 000000000..f734bb6fe --- /dev/null +++ b/test/convert-test.ts @@ -0,0 +1,94 @@ +import assert from "node:assert"; +import {convertNode, convertNodes, inferFileName, resolveInput} from "../src/convert.js"; + +describe("convertNodes", () => { + it("converts multiple nodes", () => { + assert.strictEqual( + convertNodes([ + {mode: "md", value: "Hello, world!"}, + {mode: "js", value: "1 + 2"} + ]), + "Hello, world!\n\n```js\n1 + 2\n```\n" + ); + }); +}); + +describe("convertNode", () => { + it("passes through Markdown, adding a newline", () => { + assert.strictEqual(convertNode({mode: "md", value: "Hello, world!"}), "Hello, world!\n"); + assert.strictEqual(convertNode({mode: "md", value: "# Hello, world!"}), "# Hello, world!\n"); + assert.strictEqual(convertNode({mode: "md", value: "# Hello, ${'world'}!"}), "# Hello, ${'world'}!\n"); + }); + it("wraps JavaScript in a fenced code block", () => { + assert.strictEqual(convertNode({mode: "js", value: "1 + 2"}), "```js\n1 + 2\n```\n"); + }); + it("converts pinned to echo", () => { + assert.strictEqual(convertNode({mode: "js", pinned: true, value: "1 + 2"}), "```js echo\n1 + 2\n```\n"); + }); +}); + +describe("inferFileName", () => { + it("infers a suitable file name based on identifier", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/1111111111111111"), "1111111111111111.md"); + }); + it("infers a suitable file name based on slug", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart"), "bar-chart.md"); + }); + it("handles a slug with a suffix", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart/2"), "bar-chart,2.md"); + }); + it("handles a different origin", () => { + assert.strictEqual(inferFileName("https://api.example.com/document/@d3/bar-chart"), "bar-chart.md"); + }); +}); + +/* prettier-ignore */ +describe("resolveInput", () => { + it("resolves document identifiers", () => { + assert.strictEqual(resolveInput("1111111111111111"), "https://api.observablehq.com/document/1111111111111111"); + assert.strictEqual(resolveInput("1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef"); + }); + it("resolves document slugs", () => { + assert.strictEqual(resolveInput("@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2"); + }); + it("resolves document versions", () => { + assert.strictEqual(resolveInput("1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0"); + }); + it("resolves urls", () => { + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0"); + }); + it("preserves the specified host", () => { + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef"), "https://api.example.com/document/1234567890abcdef"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@123"), "https://api.example.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@latest"), "https://api.example.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef~0"), "https://api.example.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart"), "https://api.example.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@123"), "https://api.example.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@latest"), "https://api.example.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart~0"), "https://api.example.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2"), "https://api.example.com/document/@d3/bar-chart/2"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@123"), "https://api.example.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@latest"), "https://api.example.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2~0"), "https://api.example.com/document/@d3/bar-chart/2~0"); + }); +});