From ba7538314a3d7d3a0d085382a2f402c3d910a829 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Mon, 12 Feb 2024 16:49:07 -0800 Subject: [PATCH 01/16] observable convert --- bin/observable.ts | 6 ++++ src/convert.ts | 46 ++++++++++++++++++++++++++ test/convert-test.ts | 79 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 131 insertions(+) create mode 100644 src/convert.ts create mode 100644 test/convert-test.ts diff --git a/bin/observable.ts b/bin/observable.ts index aec619c8a..8660d88f8 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -78,6 +78,7 @@ try { logout sign-out of Observable deploy deploy a project to Observable whoami check authentication status + convert convert an Observable notebook to Markdown help print usage information version print the version` ); @@ -177,6 +178,11 @@ try { await import("../src/observableApiAuth.js").then((auth) => auth.whoami()); break; } + case "convert": { + const {positionals} = helpArgs(command, {allowPositionals: true}); + await import("../src/convert.js").then((convert) => convert.convert(positionals)); + break; + } default: { console.error(`observable: unknown command '${command}'. See 'observable help'.`); process.exit(1); diff --git a/src/convert.ts b/src/convert.ts new file mode 100644 index 000000000..a85d36fe5 --- /dev/null +++ b/src/convert.ts @@ -0,0 +1,46 @@ +import {getObservableUiOrigin} from "./observableApiClient.js"; + +export async function convert(inputs: string[]): Promise { + for (const input of inputs.map(resolveInput)) { + const response = await fetch(input); + if (!response.ok) throw new Error(`error fetching ${input}: ${response.status}`); + process.stdout.write(convertNodes((await response.json()).nodes)); + } +} + +export function convertNodes(nodes): string { + let string = ""; + let first = true; + for (const node of nodes) { + if (first) first = false; + else string += "\n"; + string += convertNode(node); + } + return string; +} + +export function convertNode(node): string { + let string = ""; + if (node.mode !== "md") string += `\`\`\`${node.mode}${node.pinned ? " echo" : ""}\n`; + string += `${node.value}\n`; + if (node.mode !== "md") string += "```\n"; + return string; +} + +export function resolveInput(input: string): string { + let url: URL; + if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin()); + else if (isSlugSpecifier(input)) url = new URL(`/${input}`, getObservableUiOrigin()); + else url = new URL(input); + url.host = `api.${url.host}`; + url.pathname = `/document${url.pathname.replace(/^\/d\//, "/")}`; + return String(url); +} + +function isIdSpecifier(string: string) { + return /^([0-9a-f]{16})(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string); +} + +function isSlugSpecifier(string: string) { + return /^(?:@([0-9a-z_-]+))\/([0-9a-z_-]+(?:\/[0-9]+)?)(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string); +} diff --git a/test/convert-test.ts b/test/convert-test.ts new file mode 100644 index 000000000..1d2cf78ec --- /dev/null +++ b/test/convert-test.ts @@ -0,0 +1,79 @@ +import assert from "node:assert"; +import {convertNode, convertNodes, resolveInput} from "../src/convert.js"; + +describe("convertNodes", () => { + it("converts multiple nodes", () => { + assert.strictEqual( + convertNodes([ + {mode: "md", value: "Hello, world!"}, + {mode: "js", value: "1 + 2"} + ]), + "Hello, world!\n\n```js\n1 + 2\n```\n" + ); + }); +}); + +describe("convertNode", () => { + it("passes through Markdown, adding a newline", () => { + assert.strictEqual(convertNode({mode: "md", value: "Hello, world!"}), "Hello, world!\n"); + assert.strictEqual(convertNode({mode: "md", value: "# Hello, world!"}), "# Hello, world!\n"); + assert.strictEqual(convertNode({mode: "md", value: "# Hello, ${'world'}!"}), "# Hello, ${'world'}!\n"); + }); + it("wraps JavaScript in a fenced code block", () => { + assert.strictEqual(convertNode({mode: "js", value: "1 + 2"}), "```js\n1 + 2\n```\n"); + }); + it("converts pinned to echo", () => { + assert.strictEqual(convertNode({mode: "js", pinned: true, value: "1 + 2"}), "```js echo\n1 + 2\n```\n"); + }); +}); + +/* prettier-ignore */ +describe("resolveInput", () => { + it("resolves document identifiers", () => { + assert.strictEqual(resolveInput("1111111111111111"), "https://api.observablehq.com/document/1111111111111111"); + assert.strictEqual(resolveInput("1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef"); + }); + it("resolves document slugs", () => { + assert.strictEqual(resolveInput("@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2"); + }); + it("resolves document versions", () => { + assert.strictEqual(resolveInput("1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0"); + }); + it("resolves urls", () => { + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0"); + }); + it("preserves the specified host", () => { + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef"), "https://api.example.com/document/1234567890abcdef"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@123"), "https://api.example.com/document/1234567890abcdef@123"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@latest"), "https://api.example.com/document/1234567890abcdef@latest"); + assert.strictEqual(resolveInput("https://example.com/1234567890abcdef~0"), "https://api.example.com/document/1234567890abcdef~0"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart"), "https://api.example.com/document/@d3/bar-chart"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@123"), "https://api.example.com/document/@d3/bar-chart@123"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@latest"), "https://api.example.com/document/@d3/bar-chart@latest"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart~0"), "https://api.example.com/document/@d3/bar-chart~0"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2"), "https://api.example.com/document/@d3/bar-chart/2"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@123"), "https://api.example.com/document/@d3/bar-chart/2@123"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@latest"), "https://api.example.com/document/@d3/bar-chart/2@latest"); + assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2~0"), "https://api.example.com/document/@d3/bar-chart/2~0"); + }); +}); From 4e3ade04c5ccb88c8ab09ee07834e5076f065285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Tue, 13 Feb 2024 19:49:00 +0100 Subject: [PATCH 02/16] convert --output specifies the output directory, defaults to . convert the file names (scatterplot/2 should be "scatterplot,2") nicer log effects --- bin/observable.ts | 10 ++++++++-- src/convert.ts | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/bin/observable.ts b/bin/observable.ts index 8660d88f8..fa0991b57 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -179,8 +179,14 @@ try { break; } case "convert": { - const {positionals} = helpArgs(command, {allowPositionals: true}); - await import("../src/convert.js").then((convert) => convert.convert(positionals)); + const { + positionals, + values: {output} + } = helpArgs(command, { + options: {output: {type: "string", default: "."}}, + allowPositionals: true + }); + await import("../src/convert.js").then((convert) => convert.convert(positionals, String(output))); break; } default: { diff --git a/src/convert.ts b/src/convert.ts index a85d36fe5..c10c0d879 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,10 +1,22 @@ +import {join} from "node:path"; +import {type BuildEffects, FileBuildEffects} from "./build.js"; +import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; +import {faint} from "./tty.js"; -export async function convert(inputs: string[]): Promise { +export async function convert( + inputs: string[], + output: string, + effects: BuildEffects = new FileBuildEffects(output) +): Promise { for (const input of inputs.map(resolveInput)) { + effects.output.write(`${faint("loading")} ${input} ${faint("→")} `); const response = await fetch(input); if (!response.ok) throw new Error(`error fetching ${input}: ${response.status}`); - process.stdout.write(convertNodes((await response.json()).nodes)); + const name = input.replace(/^https:\/\/api\.observablehq\.com\/document(\/@[^/]+)?\//, "").replace(/\//g, ","); + const destination = join(output, `${name}.md`); + await prepareOutput(destination); + await effects.writeFile(destination, convertNodes((await response.json()).nodes)); } } From ea47f76b532f78eb6571ef3bbd6d22c6a89be7aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Tue, 13 Feb 2024 21:53:26 +0100 Subject: [PATCH 03/16] download attachments skip documents and files we already have --- bin/observable.ts | 4 +++- src/convert.ts | 49 +++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/bin/observable.ts b/bin/observable.ts index fa0991b57..3232bbb08 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -186,7 +186,9 @@ try { options: {output: {type: "string", default: "."}}, allowPositionals: true }); - await import("../src/convert.js").then((convert) => convert.convert(positionals, String(output))); + await import("../src/convert.js").then((convert) => + convert.convert(positionals, {output: String(output), files: true}) + ); break; } default: { diff --git a/src/convert.ts b/src/convert.ts index c10c0d879..69aa98a62 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,22 +1,59 @@ +import {access} from "node:fs/promises"; import {join} from "node:path"; import {type BuildEffects, FileBuildEffects} from "./build.js"; +import {isEnoent} from "./error.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; import {faint} from "./tty.js"; +async function readyOutput(dir, name) { + const destination = join(dir, name); + try { + await access(destination, 0); + return false; + } catch (error) { + if (isEnoent(error)) { + await prepareOutput(destination); + return true; + } + throw error; + } +} + export async function convert( inputs: string[], - output: string, + {output, files: download_files}: {output: string; files: boolean}, effects: BuildEffects = new FileBuildEffects(output) ): Promise { for (const input of inputs.map(resolveInput)) { - effects.output.write(`${faint("loading")} ${input} ${faint("→")} `); + effects.output.write( + `${faint("reading")} ${input.replace("https://api.observablehq.com/document/", "")} ${faint("→")} ` + ); const response = await fetch(input); if (!response.ok) throw new Error(`error fetching ${input}: ${response.status}`); - const name = input.replace(/^https:\/\/api\.observablehq\.com\/document(\/@[^/]+)?\//, "").replace(/\//g, ","); - const destination = join(output, `${name}.md`); - await prepareOutput(destination); - await effects.writeFile(destination, convertNodes((await response.json()).nodes)); + const name = + input.replace(/^https:\/\/api\.observablehq\.com\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md"; + const ready = await readyOutput(output, name); + if (!ready) { + effects.logger.warn(faint("skip"), name); + } else { + const {nodes, files} = await response.json(); + await effects.writeFile(name, convertNodes(nodes)); + if (download_files && files) { + for (const file of files) { + effects.output.write(`${faint("attachment")} ${file.name} ${faint("→")} `); + const ready = await readyOutput(output, file.name); + if (!ready) { + effects.logger.warn(faint("skip"), file.name); + } else { + const response = await fetch(file.download_url); + if (!response.ok) throw new Error(`error fetching ${file}: ${response.status}`); + await effects.writeFile(file.name, Buffer.from(await response.arrayBuffer())); + // TODO touch create_time: "2024-02-12T23:29:35.968Z"; + } + } + } + } } } From e87a4dcc56b17ccf3bbcfd9ddac2217e64122c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Tue, 13 Feb 2024 22:02:53 +0100 Subject: [PATCH 04/16] touch (preserve dates) --- src/convert.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index 69aa98a62..9a94d8b78 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,4 +1,5 @@ -import {access} from "node:fs/promises"; +import {utimesSync} from "node:fs"; +import {access, utimes} from "node:fs/promises"; import {join} from "node:path"; import {type BuildEffects, FileBuildEffects} from "./build.js"; import {isEnoent} from "./error.js"; @@ -37,8 +38,10 @@ export async function convert( if (!ready) { effects.logger.warn(faint("skip"), name); } else { - const {nodes, files} = await response.json(); + const {nodes, files, update_time} = await response.json(); await effects.writeFile(name, convertNodes(nodes)); + const ts = new Date(update_time); + await utimes(join(output, name), ts, ts); // touch if (download_files && files) { for (const file of files) { effects.output.write(`${faint("attachment")} ${file.name} ${faint("→")} `); @@ -49,7 +52,8 @@ export async function convert( const response = await fetch(file.download_url); if (!response.ok) throw new Error(`error fetching ${file}: ${response.status}`); await effects.writeFile(file.name, Buffer.from(await response.arrayBuffer())); - // TODO touch create_time: "2024-02-12T23:29:35.968Z"; + const ts = new Date(file.create_time); + await utimes(join(output, file.name), ts, ts); // touch } } } From ecd7c73126ea8b312426d12216acc06a48fcad65 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 16:23:38 -0800 Subject: [PATCH 05/16] Update bin/observable.ts --- bin/observable.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/observable.ts b/bin/observable.ts index 3232bbb08..a97aed8e7 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -187,7 +187,7 @@ try { allowPositionals: true }); await import("../src/convert.js").then((convert) => - convert.convert(positionals, {output: String(output), files: true}) + convert.convert(positionals, {output: output!, files: true}) ); break; } From f7d2cdb9e4af2b73f425ebe0e3eee456125f58d7 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 17:11:31 -0800 Subject: [PATCH 06/16] clack, --force --- bin/observable.ts | 8 ++-- src/convert.ts | 112 +++++++++++++++++++++++++------------------ test/convert-test.ts | 17 ++++++- 3 files changed, 86 insertions(+), 51 deletions(-) diff --git a/bin/observable.ts b/bin/observable.ts index a97aed8e7..c8251eda7 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -62,7 +62,7 @@ else if (values.help) { /** Commands that use Clack formatting. When handling CliErrors, clack.outro() * will be used for these commands. */ -const CLACKIFIED_COMMANDS = ["create", "deploy", "login"]; +const CLACKIFIED_COMMANDS = ["create", "deploy", "login", "convert"]; try { switch (command) { @@ -181,13 +181,13 @@ try { case "convert": { const { positionals, - values: {output} + values: {output, force} } = helpArgs(command, { - options: {output: {type: "string", default: "."}}, + options: {output: {type: "string", default: "."}, force: {type: "boolean", short: "f"}}, allowPositionals: true }); await import("../src/convert.js").then((convert) => - convert.convert(positionals, {output: output!, files: true}) + convert.convert(positionals, {output: output!, force, files: true}) ); break; } diff --git a/src/convert.ts b/src/convert.ts index 9a94d8b78..d750afd7b 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -1,64 +1,76 @@ -import {utimesSync} from "node:fs"; -import {access, utimes} from "node:fs/promises"; +import {existsSync} from "node:fs"; +import {utimes, writeFile} from "node:fs/promises"; import {join} from "node:path"; -import {type BuildEffects, FileBuildEffects} from "./build.js"; -import {isEnoent} from "./error.js"; +import * as clack from "@clack/prompts"; +import type {ClackEffects} from "./clack.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; -import {faint} from "./tty.js"; +import {bold, faint, inverse, red} from "./tty.js"; -async function readyOutput(dir, name) { - const destination = join(dir, name); - try { - await access(destination, 0); - return false; - } catch (error) { - if (isEnoent(error)) { - await prepareOutput(destination); - return true; - } - throw error; - } +export interface ConvertEffects { + clack: ClackEffects; + prepareOutput(outputPath: string): Promise; + existsSync(outputPath: string): boolean; + writeFile(outputPath: string, contents: Buffer | string): Promise; } +const defaultEffects: ConvertEffects = { + clack, + async prepareOutput(outputPath: string): Promise { + await prepareOutput(outputPath); + }, + existsSync(outputPath: string): boolean { + return existsSync(outputPath); + }, + async writeFile(outputPath: string, contents: Buffer | string): Promise { + await writeFile(outputPath, contents); + } +}; + export async function convert( inputs: string[], - {output, files: download_files}: {output: string; files: boolean}, - effects: BuildEffects = new FileBuildEffects(output) + {output, force = false, files: includeFiles}: {output: string; force?: boolean; files: boolean}, + effects: ConvertEffects = defaultEffects ): Promise { - for (const input of inputs.map(resolveInput)) { - effects.output.write( - `${faint("reading")} ${input.replace("https://api.observablehq.com/document/", "")} ${faint("→")} ` - ); - const response = await fetch(input); - if (!response.ok) throw new Error(`error fetching ${input}: ${response.status}`); - const name = - input.replace(/^https:\/\/api\.observablehq\.com\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md"; - const ready = await readyOutput(output, name); - if (!ready) { - effects.logger.warn(faint("skip"), name); - } else { + let success = 0; + clack.intro(`${inverse(" observable convert ")}`); + for (const input of inputs) { + let start = Date.now(); + let s = clack.spinner(); + try { + const url = resolveInput(input); + s.start(`Fetching ${url}`); + const response = await fetch(url); + if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); // TODO pretty const {nodes, files, update_time} = await response.json(); - await effects.writeFile(name, convertNodes(nodes)); - const ts = new Date(update_time); - await utimes(join(output, name), ts, ts); // touch - if (download_files && files) { + const name = inferFileName(url); + const path = join(output, name); + await effects.prepareOutput(path); + if (!force && effects.existsSync(path)) throw new Error(`${path} already exists`); + await effects.writeFile(path, convertNodes(nodes)); + await touch(path, update_time); + s.stop(`Converted ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + if (includeFiles) { for (const file of files) { - effects.output.write(`${faint("attachment")} ${file.name} ${faint("→")} `); - const ready = await readyOutput(output, file.name); - if (!ready) { - effects.logger.warn(faint("skip"), file.name); - } else { - const response = await fetch(file.download_url); - if (!response.ok) throw new Error(`error fetching ${file}: ${response.status}`); - await effects.writeFile(file.name, Buffer.from(await response.arrayBuffer())); - const ts = new Date(file.create_time); - await utimes(join(output, file.name), ts, ts); // touch - } + start = Date.now(); + s = clack.spinner(); + s.start(`Downloading ${file.name}`); + const response = await fetch(file.download_url); + if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); + const filePath = join(output, file.name); + await effects.prepareOutput(filePath); + if (!force && effects.existsSync(filePath)) throw new Error(`${filePath} already exists`); + await effects.writeFile(filePath, Buffer.from(await response.arrayBuffer())); + await touch(filePath, file.create_time); + s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); } } + ++success; + } catch (error) { + s.stop(`Converting ${input} failed: ${red(String(error))}`); } } + clack.outro(`${success} of ${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted`); } export function convertNodes(nodes): string { @@ -80,6 +92,14 @@ export function convertNode(node): string { return string; } +export function inferFileName(input: string): string { + return new URL(input).pathname.replace(/^\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md"; +} + +async function touch(path: string, time: Date | string | number) { + await utimes(path, (time = new Date(time)), time); +} + export function resolveInput(input: string): string { let url: URL; if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin()); diff --git a/test/convert-test.ts b/test/convert-test.ts index 1d2cf78ec..f734bb6fe 100644 --- a/test/convert-test.ts +++ b/test/convert-test.ts @@ -1,5 +1,5 @@ import assert from "node:assert"; -import {convertNode, convertNodes, resolveInput} from "../src/convert.js"; +import {convertNode, convertNodes, inferFileName, resolveInput} from "../src/convert.js"; describe("convertNodes", () => { it("converts multiple nodes", () => { @@ -27,6 +27,21 @@ describe("convertNode", () => { }); }); +describe("inferFileName", () => { + it("infers a suitable file name based on identifier", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/1111111111111111"), "1111111111111111.md"); + }); + it("infers a suitable file name based on slug", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart"), "bar-chart.md"); + }); + it("handles a slug with a suffix", () => { + assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart/2"), "bar-chart,2.md"); + }); + it("handles a different origin", () => { + assert.strictEqual(inferFileName("https://api.example.com/document/@d3/bar-chart"), "bar-chart.md"); + }); +}); + /* prettier-ignore */ describe("resolveInput", () => { it("resolves document identifiers", () => { From 3e34be280ced385b5228a1f6ba21286cf003dea8 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 17:13:11 -0800 Subject: [PATCH 07/16] effects.touch --- src/convert.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index d750afd7b..a125b241c 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -12,6 +12,7 @@ export interface ConvertEffects { prepareOutput(outputPath: string): Promise; existsSync(outputPath: string): boolean; writeFile(outputPath: string, contents: Buffer | string): Promise; + touch(outputPath: string, date: Date | string | number): Promise; } const defaultEffects: ConvertEffects = { @@ -24,6 +25,9 @@ const defaultEffects: ConvertEffects = { }, async writeFile(outputPath: string, contents: Buffer | string): Promise { await writeFile(outputPath, contents); +}, + async touch(outputPath: string, date: Date | string | number): Promise { + await utimes(outputPath, (date = new Date(date)), date); } }; @@ -48,7 +52,7 @@ export async function convert( await effects.prepareOutput(path); if (!force && effects.existsSync(path)) throw new Error(`${path} already exists`); await effects.writeFile(path, convertNodes(nodes)); - await touch(path, update_time); + await effects.touch(path, update_time); s.stop(`Converted ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); if (includeFiles) { for (const file of files) { @@ -61,7 +65,7 @@ export async function convert( await effects.prepareOutput(filePath); if (!force && effects.existsSync(filePath)) throw new Error(`${filePath} already exists`); await effects.writeFile(filePath, Buffer.from(await response.arrayBuffer())); - await touch(filePath, file.create_time); + await effects.touch(filePath, file.create_time); s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); } } @@ -96,10 +100,6 @@ export function inferFileName(input: string): string { return new URL(input).pathname.replace(/^\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md"; } -async function touch(path: string, time: Date | string | number) { - await utimes(path, (time = new Date(time)), time); -} - export function resolveInput(input: string): string { let url: URL; if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin()); From b71d7a06b3fc8f8a3c02cd3c8f1ad395104abace Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 18:08:19 -0800 Subject: [PATCH 08/16] prompt to overwrite --- src/convert.ts | 80 +++++++++++++++++++++++++++++--------------------- 1 file changed, 46 insertions(+), 34 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index a125b241c..5bff55a42 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -3,9 +3,10 @@ import {utimes, writeFile} from "node:fs/promises"; import {join} from "node:path"; import * as clack from "@clack/prompts"; import type {ClackEffects} from "./clack.js"; +import {CliError} from "./error.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; -import {bold, faint, inverse, red} from "./tty.js"; +import {bold, faint, inverse, yellow} from "./tty.js"; export interface ConvertEffects { clack: ClackEffects; @@ -25,7 +26,7 @@ const defaultEffects: ConvertEffects = { }, async writeFile(outputPath: string, contents: Buffer | string): Promise { await writeFile(outputPath, contents); -}, + }, async touch(outputPath: string, date: Date | string | number): Promise { await utimes(outputPath, (date = new Date(date)), date); } @@ -36,45 +37,56 @@ export async function convert( {output, force = false, files: includeFiles}: {output: string; force?: boolean; files: boolean}, effects: ConvertEffects = defaultEffects ): Promise { - let success = 0; + const {clack} = effects; clack.intro(`${inverse(" observable convert ")}`); for (const input of inputs) { let start = Date.now(); let s = clack.spinner(); - try { - const url = resolveInput(input); - s.start(`Fetching ${url}`); - const response = await fetch(url); - if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); // TODO pretty - const {nodes, files, update_time} = await response.json(); - const name = inferFileName(url); - const path = join(output, name); - await effects.prepareOutput(path); - if (!force && effects.existsSync(path)) throw new Error(`${path} already exists`); - await effects.writeFile(path, convertNodes(nodes)); - await effects.touch(path, update_time); - s.stop(`Converted ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - if (includeFiles) { - for (const file of files) { - start = Date.now(); - s = clack.spinner(); - s.start(`Downloading ${file.name}`); - const response = await fetch(file.download_url); - if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); - const filePath = join(output, file.name); - await effects.prepareOutput(filePath); - if (!force && effects.existsSync(filePath)) throw new Error(`${filePath} already exists`); - await effects.writeFile(filePath, Buffer.from(await response.arrayBuffer())); - await effects.touch(filePath, file.create_time); - s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - } + const url = resolveInput(input); + const name = inferFileName(url); + const path = join(output, name); + s.start(`Downloading ${bold(path)}`); + const response = await fetch(url); + if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); + const {nodes, files, update_time} = await response.json(); + s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await maybeWrite(path, convertNodes(nodes), force, effects); + await effects.touch(path, update_time); + if (includeFiles) { + for (const file of files) { + start = Date.now(); + s = clack.spinner(); + s.start(`Downloading ${bold(file.name)}`); + const filePath = join(output, file.name); + const response = await fetch(file.download_url); + if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); + const buffer = Buffer.from(await response.arrayBuffer()); + s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await maybeWrite(filePath, buffer, force, effects); + await effects.touch(filePath, file.create_time); } - ++success; - } catch (error) { - s.stop(`Converting ${input} failed: ${red(String(error))}`); } } - clack.outro(`${success} of ${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted`); + clack.outro(`${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted`); +} + +async function maybeWrite( + path: string, + contents: Buffer | string, + force: boolean, + effects: ConvertEffects +): Promise { + const {clack} = effects; + if (effects.existsSync(path) && !force) { + const choice = await clack.confirm({ + message: `${bold(path)} already exists; replace?`, + initialValue: false + }); + if (!choice) clack.outro(yellow("Cancelled convert")); + if (clack.isCancel(choice) || !choice) throw new CliError("Cancelled convert", {print: false}); + } + await effects.prepareOutput(path); + await effects.writeFile(path, contents); } export function convertNodes(nodes): string { From e160294c8b5146106569a38526145ff4407d20dc Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 18:09:01 -0800 Subject: [PATCH 09/16] files is optional --- bin/observable.ts | 4 +--- src/convert.ts | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/bin/observable.ts b/bin/observable.ts index c8251eda7..f286d9cd3 100644 --- a/bin/observable.ts +++ b/bin/observable.ts @@ -186,9 +186,7 @@ try { options: {output: {type: "string", default: "."}, force: {type: "boolean", short: "f"}}, allowPositionals: true }); - await import("../src/convert.js").then((convert) => - convert.convert(positionals, {output: output!, force, files: true}) - ); + await import("../src/convert.js").then((convert) => convert.convert(positionals, {output: output!, force})); break; } default: { diff --git a/src/convert.ts b/src/convert.ts index 5bff55a42..11c042341 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -34,7 +34,7 @@ const defaultEffects: ConvertEffects = { export async function convert( inputs: string[], - {output, force = false, files: includeFiles}: {output: string; force?: boolean; files: boolean}, + {output, force = false, files: includeFiles = true}: {output: string; force?: boolean; files?: boolean}, effects: ConvertEffects = defaultEffects ): Promise { const {clack} = effects; From ad0d83c41255bafca34ba958457f3cfc056c6880 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Tue, 13 Feb 2024 18:10:25 -0800 Subject: [PATCH 10/16] disable coverage for convert --- .github/workflows/test.yml | 2 +- src/convert.ts | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f7b303f54..26a096660 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: node-version: ${{ matrix.version }} cache: yarn - run: yarn --frozen-lockfile - - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client --lines 80 --per-file yarn test:mocha + - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client -x src/convert.ts --lines 80 --per-file yarn test:mocha - run: yarn test:tsc - run: | echo ::add-matcher::.github/eslint.json diff --git a/src/convert.ts b/src/convert.ts index 11c042341..a2f8f3666 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -78,10 +78,7 @@ async function maybeWrite( ): Promise { const {clack} = effects; if (effects.existsSync(path) && !force) { - const choice = await clack.confirm({ - message: `${bold(path)} already exists; replace?`, - initialValue: false - }); + const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); if (!choice) clack.outro(yellow("Cancelled convert")); if (clack.isCancel(choice) || !choice) throw new CliError("Cancelled convert", {print: false}); } From 3165026168eea01829e665ef22a5932abfa13c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 14 Feb 2024 16:34:05 +0100 Subject: [PATCH 11/16] skip instead of cancelling ctrl-c still stops, as expected --- src/convert.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index a2f8f3666..829e1c9e4 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -39,6 +39,7 @@ export async function convert( ): Promise { const {clack} = effects; clack.intro(`${inverse(" observable convert ")}`); + let n = 0; for (const input of inputs) { let start = Date.now(); let s = clack.spinner(); @@ -50,7 +51,7 @@ export async function convert( if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); const {nodes, files, update_time} = await response.json(); s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - await maybeWrite(path, convertNodes(nodes), force, effects); + if ((await maybeWrite(path, convertNodes(nodes), force, effects)) === undefined) n++; await effects.touch(path, update_time); if (includeFiles) { for (const file of files) { @@ -62,12 +63,14 @@ export async function convert( if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); const buffer = Buffer.from(await response.arrayBuffer()); s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - await maybeWrite(filePath, buffer, force, effects); + if ((await maybeWrite(filePath, buffer, force, effects)) === undefined) n++; await effects.touch(filePath, file.create_time); } } } - clack.outro(`${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted`); + clack.outro( + `${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted; ${n} file${n === 1 ? "" : "s"} written` + ); } async function maybeWrite( @@ -75,12 +78,15 @@ async function maybeWrite( contents: Buffer | string, force: boolean, effects: ConvertEffects -): Promise { +): Promise { const {clack} = effects; if (effects.existsSync(path) && !force) { const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); - if (!choice) clack.outro(yellow("Cancelled convert")); - if (clack.isCancel(choice) || !choice) throw new CliError("Cancelled convert", {print: false}); + if (!choice) { + clack.outro(yellow("Skipping…")); + return true; + } + if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false}); } await effects.prepareOutput(path); await effects.writeFile(path, contents); From 06fb5a6f9ec834631ec668d6cc49a7b18c4fda18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 14 Feb 2024 16:42:27 +0100 Subject: [PATCH 12/16] ask for overwrite before downloading --- src/convert.ts | 61 +++++++++++++++++++++++++------------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index 829e1c9e4..7a4648e70 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -46,25 +46,33 @@ export async function convert( const url = resolveInput(input); const name = inferFileName(url); const path = join(output, name); - s.start(`Downloading ${bold(path)}`); - const response = await fetch(url); - if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); - const {nodes, files, update_time} = await response.json(); - s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - if ((await maybeWrite(path, convertNodes(nodes), force, effects)) === undefined) n++; - await effects.touch(path, update_time); - if (includeFiles) { - for (const file of files) { - start = Date.now(); - s = clack.spinner(); - s.start(`Downloading ${bold(file.name)}`); - const filePath = join(output, file.name); - const response = await fetch(file.download_url); - if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); - const buffer = Buffer.from(await response.arrayBuffer()); - s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); - if ((await maybeWrite(filePath, buffer, force, effects)) === undefined) n++; - await effects.touch(filePath, file.create_time); + if (await maybeFetch(path, force, effects)) { + s.start(`Downloading ${bold(path)}`); + const response = await fetch(url); + if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`); + const {nodes, files, update_time} = await response.json(); + s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await effects.prepareOutput(path); + await effects.writeFile(path, convertNodes(nodes)); + await effects.touch(path, update_time); + n++; + if (includeFiles) { + for (const file of files) { + const path = join(output, file.name); + if (await maybeFetch(path, force, effects)) { + start = Date.now(); + s = clack.spinner(); + s.start(`Downloading ${bold(file.name)}`); + const response = await fetch(file.download_url); + if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`); + const buffer = Buffer.from(await response.arrayBuffer()); + s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`); + await effects.prepareOutput(path); + await effects.writeFile(path, buffer); + await effects.touch(path, file.create_time); + n++; + } + } } } } @@ -73,23 +81,14 @@ export async function convert( ); } -async function maybeWrite( - path: string, - contents: Buffer | string, - force: boolean, - effects: ConvertEffects -): Promise { +async function maybeFetch(path: string, force: boolean, effects: ConvertEffects): Promise { const {clack} = effects; if (effects.existsSync(path) && !force) { const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); - if (!choice) { - clack.outro(yellow("Skipping…")); - return true; - } + if (!choice) return clack.outro(yellow("Skipping…")), false; if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false}); } - await effects.prepareOutput(path); - await effects.writeFile(path, contents); + return true; } export function convertNodes(nodes): string { From 8576935002c4875520953975b8f4d0365bbb454d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Wed, 14 Feb 2024 16:44:11 +0100 Subject: [PATCH 13/16] clack.warn --- src/convert.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index 7a4648e70..434f644b0 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -6,7 +6,7 @@ import type {ClackEffects} from "./clack.js"; import {CliError} from "./error.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; -import {bold, faint, inverse, yellow} from "./tty.js"; +import {bold, faint, inverse} from "./tty.js"; export interface ConvertEffects { clack: ClackEffects; @@ -85,7 +85,7 @@ async function maybeFetch(path: string, force: boolean, effects: ConvertEffects) const {clack} = effects; if (effects.existsSync(path) && !force) { const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); - if (!choice) return clack.outro(yellow("Skipping…")), false; + if (!choice) return clack.log.warn("Skipping…"), false; if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false}); } return true; From 723a59739b364a1a7b9294a81042eead69935cd0 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Wed, 14 Feb 2024 15:18:15 -0800 Subject: [PATCH 14/16] remove superfluous warn --- src/convert.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/convert.ts b/src/convert.ts index 434f644b0..0b289a4b8 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -85,7 +85,7 @@ async function maybeFetch(path: string, force: boolean, effects: ConvertEffects) const {clack} = effects; if (effects.existsSync(path) && !force) { const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false}); - if (!choice) return clack.log.warn("Skipping…"), false; + if (!choice) return false; if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false}); } return true; From 22089b752afa9fe71b9930426e8abc5176ceafd8 Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Wed, 14 Feb 2024 15:31:33 -0800 Subject: [PATCH 15/16] caveat emptor --- src/convert.ts | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/convert.ts b/src/convert.ts index 0b289a4b8..63e0f3b05 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -2,13 +2,14 @@ import {existsSync} from "node:fs"; import {utimes, writeFile} from "node:fs/promises"; import {join} from "node:path"; import * as clack from "@clack/prompts"; +import wrapAnsi from "wrap-ansi"; import type {ClackEffects} from "./clack.js"; import {CliError} from "./error.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; -import {bold, faint, inverse} from "./tty.js"; +import {type TtyEffects, bold, faint, inverse, defaultEffects as ttyEffects, yellow, link, cyan, reset} from "./tty.js"; -export interface ConvertEffects { +export interface ConvertEffects extends TtyEffects { clack: ClackEffects; prepareOutput(outputPath: string): Promise; existsSync(outputPath: string): boolean; @@ -17,6 +18,7 @@ export interface ConvertEffects { } const defaultEffects: ConvertEffects = { + ...ttyEffects, clack, async prepareOutput(outputPath: string): Promise { await prepareOutput(outputPath); @@ -76,6 +78,17 @@ export async function convert( } } } + clack.note( + wrapAnsi( + "Due to syntax differences between Observable notebooks and " + + "Observable Framework, converted notebooks may require further " + + "changes to function correctly. To learn more about JavaScript " + + "in Framework, please read:\n\n" + + reset(cyan(link("https://observablehq.com/framework/javascript"))), + Math.min(64, effects.outputColumns) + ), + "Note" + ); clack.outro( `${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted; ${n} file${n === 1 ? "" : "s"} written` ); From 33b3a734b006d924f56a00cf802fe2b64efb20ce Mon Sep 17 00:00:00 2001 From: Mike Bostock Date: Wed, 14 Feb 2024 15:33:42 -0800 Subject: [PATCH 16/16] fix imports --- src/convert.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/convert.ts b/src/convert.ts index 63e0f3b05..372d3ec3b 100644 --- a/src/convert.ts +++ b/src/convert.ts @@ -7,7 +7,7 @@ import type {ClackEffects} from "./clack.js"; import {CliError} from "./error.js"; import {prepareOutput} from "./files.js"; import {getObservableUiOrigin} from "./observableApiClient.js"; -import {type TtyEffects, bold, faint, inverse, defaultEffects as ttyEffects, yellow, link, cyan, reset} from "./tty.js"; +import {type TtyEffects, bold, cyan, faint, inverse, link, reset, defaultEffects as ttyEffects} from "./tty.js"; export interface ConvertEffects extends TtyEffects { clack: ClackEffects;