init

.vscode/settings.json

@@ -6,5 +6,5 @@
   },
   "editor.defaultFormatter": "biomejs.biome",
   "editor.gotoLocation.multipleDefinitions": "goto",
-  "cSpell.words": ["mergebot", "studiocms", "withstudiocms"]
+  "cSpell.words": ["mergebot", "Shiki", "studiocms", "withstudiocms"]
 }

biome.json

@@ -34,6 +34,9 @@
       "recommended": true,
       "suspicious": {
         "noExplicitAny": "warn"
+      },
+      "style": {
+        "noParameterAssign": "info"
       }
     }
   },

package.json

@@ -6,20 +6,26 @@
       "node": "20.14.0"
     },
     "scripts": {
+      "build": "pnpm --filter @studiocms/markdown-remark build",
       "lint": "biome check .",
       "lint:fix": "biome check --write .",
       "ci:lint": "biome ci --formatter-enabled=true --organize-imports-enabled=true  --reporter=github",
       "ci:install": "pnpm install --frozen-lockfile",
+      "ci:build": "pnpm --filter @studiocms/markdown-remark build:ci",
       "ci:version": "pnpm changeset version",
       "ci:publish": "pnpm changeset publish",
       "ci:snapshot": "pnpx pkg-pr-new publish --pnpm './packages/*'"
     },
     "dependencies": {
+        "@astrojs/check": "^0.9.4",
         "@biomejs/biome": "^1.9.4",
         "@changesets/cli": "^2.27.10",
         "@changesets/config": "^3.0.4",
         "@changesets/changelog-github": "^0.5.0",
+        "@types/node": "^18.17.8",
+        "esbuild": "^0.21.5",
         "pkg-pr-new": "^0.0.39",
-        "typescript": "^5.7.2"
+        "typescript": "^5.7.2",
+        "vitest": "^2.1.8"
     }
   }

packages/markdown-remark/package.json

@@ -0,0 +1,69 @@
+{
+    "name": "@studiocms/markdown-remark",
+    "version": "0.0.1",
+    "type": "module",
+    "author": "studiocms",
+    "license": "MIT",
+    "repository": {
+      "type": "git",
+      "url": "git+https://github.com/withstudiocms/markdown-remark.git",
+      "directory": "packages/markdown-remark"
+    },
+    "bugs": "https://github.com/withstudiocms/markdown-remark/issues",
+    "homepage": "https://studiocms.dev",
+    "main": "./dist/index.js",
+    "exports": {
+      ".": "./dist/index.js"
+    },
+    "imports": {
+      "#import-plugin": {
+        "browser": "./dist/import-plugin-browser.js",
+        "default": "./dist/import-plugin-default.js"
+      }
+    },
+    "files": [
+      "dist"
+    ],
+    "scripts": {
+      "prepublish": "pnpm build",
+      "build": "run-scripts build \"src/**/*.ts\" && tsc -p tsconfig.json",
+      "build:ci": "run-scripts build \"src/**/*.ts\"",
+      "dev": "run-scripts dev \"src/**/*.ts\""
+    },
+    "dependencies": {
+      "@astrojs/prism": "^3.2.0",
+      "github-slugger": "^2.0.0",
+      "hast-util-from-html": "^2.0.3",
+      "hast-util-to-text": "^4.0.2",
+      "import-meta-resolve": "^4.1.0",
+      "js-yaml": "^4.1.0",
+      "mdast-util-definitions": "^6.0.0",
+      "rehype-raw": "^7.0.0",
+      "rehype-stringify": "^10.0.1",
+      "remark-gfm": "^4.0.0",
+      "remark-parse": "^11.0.0",
+      "remark-rehype": "^11.1.1",
+      "remark-smartypants": "^3.0.2",
+      "shiki": "^1.23.1",
+      "unified": "^11.0.5",
+      "unist-util-remove-position": "^5.0.0",
+      "unist-util-visit": "^5.0.0",
+      "unist-util-visit-parents": "^6.0.1",
+      "vfile": "^6.0.3",
+      "vitest": "^2.1.8"
+    },
+    "devDependencies": {
+      "@types/estree": "^1.0.6",
+      "@types/hast": "^3.0.4",
+      "@types/js-yaml": "^4.0.9",
+      "@types/mdast": "^4.0.4",
+      "@types/unist": "^3.0.3",
+      "run-scripts": "workspace:*",
+      "esbuild": "^0.21.5",
+      "fast-glob": "^3.3.2",
+      "mdast-util-mdx-expression": "^2.0.1"
+    },
+    "publishConfig": {
+      "provenance": true
+    }
+  }

packages/markdown-remark/src/HTMLString.ts

@@ -0,0 +1,5 @@
+export class HTMLString extends String {
+	get [Symbol.toStringTag]() {
+		return 'HTMLString';
+	}
+}

packages/markdown-remark/src/frontmatter.ts

@@ -0,0 +1,81 @@
+import yaml from 'js-yaml';
+
+// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+export function isFrontmatterValid(frontmatter: Record<string, any>) {
+	try {
+		// ensure frontmatter is JSON-serializable
+		JSON.stringify(frontmatter);
+	} catch {
+		return false;
+	}
+	return typeof frontmatter === 'object' && frontmatter !== null;
+}
+
+// Capture frontmatter wrapped with `---`, including any characters and new lines within it.
+// Only capture if `---` exists near the top of the file, including:
+// 1. Start of file (including if has BOM encoding)
+// 2. Start of file with any whitespace (but `---` must still start on a new line)
+const frontmatterRE = /(?:^\uFEFF?|^\s*\n)---([\s\S]*?\n)---/;
+export function extractFrontmatter(code: string): string | undefined {
+	return frontmatterRE.exec(code)?.[1];
+}
+
+export interface ParseFrontmatterOptions {
+	/**
+	 * How the frontmatter should be handled in the returned `content` string.
+	 * - `preserve`: Keep the frontmatter.
+	 * - `remove`: Remove the frontmatter.
+	 * - `empty-with-spaces`: Replace the frontmatter with empty spaces. (preserves sourcemap line/col/offset)
+	 * - `empty-with-lines`: Replace the frontmatter with empty line breaks. (preserves sourcemap line/col)
+	 *
+	 * @default 'remove'
+	 */
+	frontmatter: 'preserve' | 'remove' | 'empty-with-spaces' | 'empty-with-lines';
+}
+
+export interface ParseFrontmatterResult {
+	// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+	frontmatter: Record<string, any>;
+	rawFrontmatter: string;
+	content: string;
+}
+
+export function parseFrontmatter(
+	code: string,
+	options?: ParseFrontmatterOptions
+): ParseFrontmatterResult {
+	const rawFrontmatter = extractFrontmatter(code);
+
+	if (rawFrontmatter == null) {
+		return { frontmatter: {}, rawFrontmatter: '', content: code };
+	}
+
+	const parsed = yaml.load(rawFrontmatter);
+	// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+	const frontmatter = (parsed && typeof parsed === 'object' ? parsed : {}) as Record<string, any>;
+
+	let content: string;
+	switch (options?.frontmatter ?? 'remove') {
+		case 'preserve':
+			content = code;
+			break;
+		case 'remove':
+			content = code.replace(`---${rawFrontmatter}---`, '');
+			break;
+		case 'empty-with-spaces':
+			content = code.replace(
+				`---${rawFrontmatter}---`,
+				`   ${rawFrontmatter.replace(/[^\r\n]/g, ' ')}   `
+			);
+			break;
+		case 'empty-with-lines':
+			content = code.replace(`---${rawFrontmatter}---`, rawFrontmatter.replace(/[^\r\n]/g, ''));
+			break;
+	}
+
+	return {
+		frontmatter,
+		rawFrontmatter,
+		content,
+	};
+}

packages/markdown-remark/src/highlight.ts

@@ -0,0 +1,98 @@
+import type { Element, Parent, Root } from 'hast';
+import { fromHtml } from 'hast-util-from-html';
+import { toText } from 'hast-util-to-text';
+import { removePosition } from 'unist-util-remove-position';
+import { visitParents } from 'unist-util-visit-parents';
+
+type Highlighter = (
+	code: string,
+	language: string,
+	options?: { meta?: string }
+) => Promise<Root | string>;
+
+const languagePattern = /\blanguage-(\S+)\b/;
+
+/**
+ * A hast utility to syntax highlight code blocks with a given syntax highlighter.
+ *
+ * @param tree
+ *   The hast tree in which to syntax highlight code blocks.
+ * @param highlighter
+ *   A function which receives the code and language, and returns the HTML of a syntax
+ *   highlighted `<pre>` element.
+ */
+export async function highlightCodeBlocks(tree: Root, highlighter: Highlighter) {
+	const nodes: Array<{
+		node: Element;
+		language: string;
+		parent: Element;
+		grandParent: Parent;
+	}> = [];
+
+	// We’re looking for `<code>` elements
+	visitParents(tree, { type: 'element', tagName: 'code' }, (node, ancestors) => {
+		const parent = ancestors.at(-1);
+
+		// Whose parent is a `<pre>`.
+		if (parent?.type !== 'element' || parent.tagName !== 'pre') {
+			return;
+		}
+
+		// Where the `<code>` is the only child.
+		if (parent.children.length !== 1) {
+			return;
+		}
+
+		// And the `<code>` has a class name that starts with `language-`.
+		let languageMatch: RegExpMatchArray | null | undefined;
+		const { className } = node.properties;
+		if (typeof className === 'string') {
+			languageMatch = languagePattern.exec(className);
+		} else if (Array.isArray(className)) {
+			for (const cls of className) {
+				if (typeof cls !== 'string') {
+					continue;
+				}
+
+				languageMatch = languagePattern.exec(cls);
+				if (languageMatch) {
+					break;
+				}
+			}
+		}
+
+		// Don’t Highlight math code blocks.
+		if (languageMatch?.[1] === 'math') {
+			return;
+		}
+
+		nodes.push({
+			node,
+			language: languageMatch?.[1] || 'plaintext',
+			parent,
+			// biome-ignore lint/style/noNonNullAssertion: <explanation>
+			grandParent: ancestors.at(-2)!,
+		});
+	});
+
+	for (const { node, language, grandParent, parent } of nodes) {
+		// biome-ignore lint/suspicious/noExplicitAny: <explanation>
+		const meta = (node.data as any)?.meta ?? node.properties.metastring ?? undefined;
+		const code = toText(node, { whitespace: 'pre' });
+		const result = await highlighter(code, language, { meta });
+
+		let replacement: Element;
+		if (typeof result === 'string') {
+			// The replacement returns a root node with 1 child, the `<pre>` element replacement.
+			replacement = fromHtml(result, { fragment: true }).children[0] as Element;
+			// We just generated this node, so any positional information is invalid.
+			removePosition(replacement);
+		} else {
+			replacement = result.children[0] as Element;
+		}
+
+		// We replace the parent in its parent with the new `<pre>` element.
+		const index = grandParent.children.indexOf(parent);
+		grandParent.children[index] = replacement;
+	}
+}

packages/markdown-remark/src/import-plugin-browser.ts

@@ -0,0 +1,8 @@
+// This file should be imported as `#import-plugin`
+import type * as unified from 'unified';
+
+// In the browser, we can try to do a plain import
+export async function importPlugin(p: string): Promise<unified.Plugin> {
+	const importResult = await import(p);
+	return importResult.default;
+}

packages/markdown-remark/src/import-plugin-default.ts

@@ -0,0 +1,22 @@
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+// This file should be imported as `#import-plugin`
+import { resolve as importMetaResolve } from 'import-meta-resolve';
+import type * as unified from 'unified';
+
+let cwdUrlStr: string | undefined;
+
+// In non-browser environments, we can try to resolve from the filesystem too
+export async function importPlugin(p: string): Promise<unified.Plugin> {
+	// Try import from this package first
+	try {
+		const importResult = await import(p);
+		return importResult.default;
+	} catch {}
+
+	// Try import from user project
+	cwdUrlStr ??= pathToFileURL(path.join(process.cwd(), 'package.json')).toString();
+	const resolved = importMetaResolve(p, cwdUrlStr);
+	const importResult = await import(resolved);
+	return importResult.default;
+}