observable convert (#764)

* observable convert * convert --output specifies the output directory, defaults to . convert the file names (scatterplot/2 should be "scatterplot,2") nicer log effects * download attachments skip documents and files we already have * touch (preserve dates) * Update bin/observable.ts * clack, --force * effects.touch * prompt to overwrite * files is optional * disable coverage for convert --------- Co-authored-by: Philippe Rivière <[email protected]>
observablehq · Feb 14, 2024 · 5bc5510 · 5bc5510
1 parent ca4218a
commit 5bc5510
Show file tree

Hide file tree

Showing 4 changed files with 254 additions and 2 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -19,7 +19,7 @@ jobs:
           node-version: ${{ matrix.version }}
           cache: yarn
       - run: yarn --frozen-lockfile
-      - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client --lines 80 --per-file yarn test:mocha
+      - run: yarn c8 --check-coverage -x src/**/*.d.ts -x src/preview.ts -x src/observableApiConfig.ts -x src/client -x src/convert.ts --lines 80 --per-file yarn test:mocha
       - run: yarn test:tsc
       - run: |
           echo ::add-matcher::.github/eslint.json

diff --git a/bin/observable.ts b/bin/observable.ts
@@ -62,7 +62,7 @@ else if (values.help) {
 
 /** Commands that use Clack formatting. When handling CliErrors, clack.outro()
  * will be used for these commands. */
-const CLACKIFIED_COMMANDS = ["create", "deploy", "login"];
+const CLACKIFIED_COMMANDS = ["create", "deploy", "login", "convert"];
 
 try {
   switch (command) {
@@ -78,6 +78,7 @@ try {
   logout       sign-out of Observable
   deploy       deploy a project to Observable
   whoami       check authentication status
+  convert      convert an Observable notebook to Markdown
   help         print usage information
   version      print the version`
       );
@@ -177,6 +178,17 @@ try {
       await import("../src/observableApiAuth.js").then((auth) => auth.whoami());
       break;
     }
+    case "convert": {
+      const {
+        positionals,
+        values: {output, force}
+      } = helpArgs(command, {
+        options: {output: {type: "string", default: "."}, force: {type: "boolean", short: "f"}},
+        allowPositionals: true
+      });
+      await import("../src/convert.js").then((convert) => convert.convert(positionals, {output: output!, force}));
+      break;
+    }
     default: {
       console.error(`observable: unknown command '${command}'. See 'observable help'.`);
       process.exit(1);

diff --git a/src/convert.ts b/src/convert.ts
@@ -0,0 +1,146 @@
+import {existsSync} from "node:fs";
+import {utimes, writeFile} from "node:fs/promises";
+import {join} from "node:path";
+import * as clack from "@clack/prompts";
+import wrapAnsi from "wrap-ansi";
+import type {ClackEffects} from "./clack.js";
+import {CliError} from "./error.js";
+import {prepareOutput} from "./files.js";
+import {getObservableUiOrigin} from "./observableApiClient.js";
+import {type TtyEffects, bold, cyan, faint, inverse, link, reset, defaultEffects as ttyEffects} from "./tty.js";
+
+export interface ConvertEffects extends TtyEffects {
+  clack: ClackEffects;
+  prepareOutput(outputPath: string): Promise<void>;
+  existsSync(outputPath: string): boolean;
+  writeFile(outputPath: string, contents: Buffer | string): Promise<void>;
+  touch(outputPath: string, date: Date | string | number): Promise<void>;
+}
+
+const defaultEffects: ConvertEffects = {
+  ...ttyEffects,
+  clack,
+  async prepareOutput(outputPath: string): Promise<void> {
+    await prepareOutput(outputPath);
+  },
+  existsSync(outputPath: string): boolean {
+    return existsSync(outputPath);
+  },
+  async writeFile(outputPath: string, contents: Buffer | string): Promise<void> {
+    await writeFile(outputPath, contents);
+  },
+  async touch(outputPath: string, date: Date | string | number): Promise<void> {
+    await utimes(outputPath, (date = new Date(date)), date);
+  }
+};
+
+export async function convert(
+  inputs: string[],
+  {output, force = false, files: includeFiles = true}: {output: string; force?: boolean; files?: boolean},
+  effects: ConvertEffects = defaultEffects
+): Promise<void> {
+  const {clack} = effects;
+  clack.intro(`${inverse(" observable convert ")}`);
+  let n = 0;
+  for (const input of inputs) {
+    let start = Date.now();
+    let s = clack.spinner();
+    const url = resolveInput(input);
+    const name = inferFileName(url);
+    const path = join(output, name);
+    if (await maybeFetch(path, force, effects)) {
+      s.start(`Downloading ${bold(path)}`);
+      const response = await fetch(url);
+      if (!response.ok) throw new Error(`error fetching ${url}: ${response.status}`);
+      const {nodes, files, update_time} = await response.json();
+      s.stop(`Downloaded ${bold(path)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`);
+      await effects.prepareOutput(path);
+      await effects.writeFile(path, convertNodes(nodes));
+      await effects.touch(path, update_time);
+      n++;
+      if (includeFiles) {
+        for (const file of files) {
+          const path = join(output, file.name);
+          if (await maybeFetch(path, force, effects)) {
+            start = Date.now();
+            s = clack.spinner();
+            s.start(`Downloading ${bold(file.name)}`);
+            const response = await fetch(file.download_url);
+            if (!response.ok) throw new Error(`error fetching ${file.download_url}: ${response.status}`);
+            const buffer = Buffer.from(await response.arrayBuffer());
+            s.stop(`Downloaded ${bold(file.name)} ${faint(`in ${(Date.now() - start).toLocaleString("en-US")}ms`)}`);
+            await effects.prepareOutput(path);
+            await effects.writeFile(path, buffer);
+            await effects.touch(path, file.create_time);
+            n++;
+          }
+        }
+      }
+    }
+  }
+  clack.note(
+    wrapAnsi(
+      "Due to syntax differences between Observable notebooks and " +
+        "Observable Framework, converted notebooks may require further " +
+        "changes to function correctly. To learn more about JavaScript " +
+        "in Framework, please read:\n\n" +
+        reset(cyan(link("https://observablehq.com/framework/javascript"))),
+      Math.min(64, effects.outputColumns)
+    ),
+    "Note"
+  );
+  clack.outro(
+    `${inputs.length} notebook${inputs.length === 1 ? "" : "s"} converted; ${n} file${n === 1 ? "" : "s"} written`
+  );
+}
+
+async function maybeFetch(path: string, force: boolean, effects: ConvertEffects): Promise<boolean> {
+  const {clack} = effects;
+  if (effects.existsSync(path) && !force) {
+    const choice = await clack.confirm({message: `${bold(path)} already exists; replace?`, initialValue: false});
+    if (!choice) return false;
+    if (clack.isCancel(choice)) throw new CliError("Stopped convert", {print: false});
+  }
+  return true;
+}
+
+export function convertNodes(nodes): string {
+  let string = "";
+  let first = true;
+  for (const node of nodes) {
+    if (first) first = false;
+    else string += "\n";
+    string += convertNode(node);
+  }
+  return string;
+}
+
+export function convertNode(node): string {
+  let string = "";
+  if (node.mode !== "md") string += `\`\`\`${node.mode}${node.pinned ? " echo" : ""}\n`;
+  string += `${node.value}\n`;
+  if (node.mode !== "md") string += "```\n";
+  return string;
+}
+
+export function inferFileName(input: string): string {
+  return new URL(input).pathname.replace(/^\/document(\/@[^/]+)?\//, "").replace(/\//g, ",") + ".md";
+}
+
+export function resolveInput(input: string): string {
+  let url: URL;
+  if (isIdSpecifier(input)) url = new URL(`/d/${input}`, getObservableUiOrigin());
+  else if (isSlugSpecifier(input)) url = new URL(`/${input}`, getObservableUiOrigin());
+  else url = new URL(input);
+  url.host = `api.${url.host}`;
+  url.pathname = `/document${url.pathname.replace(/^\/d\//, "/")}`;
+  return String(url);
+}
+
+function isIdSpecifier(string: string) {
+  return /^([0-9a-f]{16})(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string);
+}
+
+function isSlugSpecifier(string: string) {
+  return /^(?:@([0-9a-z_-]+))\/([0-9a-z_-]+(?:\/[0-9]+)?)(?:@(\d+)|~(\d+)|@(\w+))?$/.test(string);
+}
diff --git a/test/convert-test.ts b/test/convert-test.ts
@@ -0,0 +1,94 @@
+import assert from "node:assert";
+import {convertNode, convertNodes, inferFileName, resolveInput} from "../src/convert.js";
+
+describe("convertNodes", () => {
+  it("converts multiple nodes", () => {
+    assert.strictEqual(
+      convertNodes([
+        {mode: "md", value: "Hello, world!"},
+        {mode: "js", value: "1 + 2"}
+      ]),
+      "Hello, world!\n\n```js\n1 + 2\n```\n"
+    );
+  });
+});
+
+describe("convertNode", () => {
+  it("passes through Markdown, adding a newline", () => {
+    assert.strictEqual(convertNode({mode: "md", value: "Hello, world!"}), "Hello, world!\n");
+    assert.strictEqual(convertNode({mode: "md", value: "# Hello, world!"}), "# Hello, world!\n");
+    assert.strictEqual(convertNode({mode: "md", value: "# Hello, ${'world'}!"}), "# Hello, ${'world'}!\n");
+  });
+  it("wraps JavaScript in a fenced code block", () => {
+    assert.strictEqual(convertNode({mode: "js", value: "1 + 2"}), "```js\n1 + 2\n```\n");
+  });
+  it("converts pinned to echo", () => {
+    assert.strictEqual(convertNode({mode: "js", pinned: true, value: "1 + 2"}), "```js echo\n1 + 2\n```\n");
+  });
+});
+
+describe("inferFileName", () => {
+  it("infers a suitable file name based on identifier", () => {
+    assert.strictEqual(inferFileName("https://api.observablehq.com/document/1111111111111111"), "1111111111111111.md");
+  });
+  it("infers a suitable file name based on slug", () => {
+    assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart"), "bar-chart.md");
+  });
+  it("handles a slug with a suffix", () => {
+    assert.strictEqual(inferFileName("https://api.observablehq.com/document/@d3/bar-chart/2"), "bar-chart,2.md");
+  });
+  it("handles a different origin", () => {
+    assert.strictEqual(inferFileName("https://api.example.com/document/@d3/bar-chart"), "bar-chart.md");
+  });
+});
+
+/* prettier-ignore */
+describe("resolveInput", () => {
+  it("resolves document identifiers", () => {
+    assert.strictEqual(resolveInput("1111111111111111"), "https://api.observablehq.com/document/1111111111111111");
+    assert.strictEqual(resolveInput("1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef");
+  });
+  it("resolves document slugs", () => {
+    assert.strictEqual(resolveInput("@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart");
+    assert.strictEqual(resolveInput("@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2");
+  });
+  it("resolves document versions", () => {
+    assert.strictEqual(resolveInput("1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123");
+    assert.strictEqual(resolveInput("1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest");
+    assert.strictEqual(resolveInput("1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0");
+    assert.strictEqual(resolveInput("@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123");
+    assert.strictEqual(resolveInput("@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest");
+    assert.strictEqual(resolveInput("@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0");
+    assert.strictEqual(resolveInput("@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123");
+    assert.strictEqual(resolveInput("@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest");
+    assert.strictEqual(resolveInput("@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0");
+  });
+  it("resolves urls", () => {
+    assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef"), "https://api.observablehq.com/document/1234567890abcdef");
+    assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@123"), "https://api.observablehq.com/document/1234567890abcdef@123");
+    assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef@latest"), "https://api.observablehq.com/document/1234567890abcdef@latest");
+    assert.strictEqual(resolveInput("https://observablehq.com/1234567890abcdef~0"), "https://api.observablehq.com/document/1234567890abcdef~0");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart"), "https://api.observablehq.com/document/@d3/bar-chart");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@123"), "https://api.observablehq.com/document/@d3/bar-chart@123");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart@latest"), "https://api.observablehq.com/document/@d3/bar-chart@latest");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart~0"), "https://api.observablehq.com/document/@d3/bar-chart~0");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2"), "https://api.observablehq.com/document/@d3/bar-chart/2");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@123"), "https://api.observablehq.com/document/@d3/bar-chart/2@123");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2@latest"), "https://api.observablehq.com/document/@d3/bar-chart/2@latest");
+    assert.strictEqual(resolveInput("https://observablehq.com/@d3/bar-chart/2~0"), "https://api.observablehq.com/document/@d3/bar-chart/2~0");
+  });
+  it("preserves the specified host", () => {
+    assert.strictEqual(resolveInput("https://example.com/1234567890abcdef"), "https://api.example.com/document/1234567890abcdef");
+    assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@123"), "https://api.example.com/document/1234567890abcdef@123");
+    assert.strictEqual(resolveInput("https://example.com/1234567890abcdef@latest"), "https://api.example.com/document/1234567890abcdef@latest");
+    assert.strictEqual(resolveInput("https://example.com/1234567890abcdef~0"), "https://api.example.com/document/1234567890abcdef~0");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart"), "https://api.example.com/document/@d3/bar-chart");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@123"), "https://api.example.com/document/@d3/bar-chart@123");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart@latest"), "https://api.example.com/document/@d3/bar-chart@latest");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart~0"), "https://api.example.com/document/@d3/bar-chart~0");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2"), "https://api.example.com/document/@d3/bar-chart/2");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@123"), "https://api.example.com/document/@d3/bar-chart/2@123");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2@latest"), "https://api.example.com/document/@d3/bar-chart/2@latest");
+    assert.strictEqual(resolveInput("https://example.com/@d3/bar-chart/2~0"), "https://api.example.com/document/@d3/bar-chart/2~0");
+  });
+});