diff --git a/example/content.md b/example/content.md index c2e28a0..9efc389 100644 --- a/example/content.md +++ b/example/content.md @@ -47,6 +47,19 @@ document.getElementsByTagName("head")[0].innerHTML +=

+| Type | Description | example | +| ---------------- | ------------------------------------------------ | --------------------------------- | +| `string` | A string of characters. | `'Hello world'` | +| `number` | A numeric value, either float or integer. | `42` | +| `boolean` | A boolean value. | `true` | +| `enum` | An enum value. | `'drama'` | +| `geopoint` | A geopoint value. | `{ lat: 40.7128, lon: 74.0060 }` | +| `string[]` | An array of strings. | `['red', 'green', 'blue']` | +| `number[]` | An array of numbers. | `[42, 91, 28.5]` | +| `boolean[]` | An array of booleans. | `[true, false, false]` | +| `enum[]` | An array of enums. | `['comedy', 'action', 'romance']` | +| `vector[]` | A vector of numbers to perform vector search on. | `[0.403, 0.192, 0.830]` | + ## Math rendering We support code blocks with the "math" type! diff --git a/mod.ts b/mod.ts index 046ddaa..2aea62f 100644 --- a/mod.ts +++ b/mod.ts @@ -344,35 +344,55 @@ function mergeAttributes( return merged; } -function stripTokens(tokens: Marked.Token[]): string { - let out = ""; +function stripTokens( + tokens: Marked.Token[], + sections: MarkdownSections[], + header: boolean, +) { + let index = sections.length - 1; + for (const token of tokens) { + if (token.type === "heading") { + sections[index].header = sections[index].header.trim().replace( + /\n{3,}/g, + "\n", + ); + sections[index].content = sections[index].content.trim().replace( + /\n{3,}/g, + "\n", + ); + + sections.push({ header: "", depth: token.depth, content: "" }); + index += 1; + } + if ("tokens" in token && token.tokens) { - out += stripTokens(token.tokens); + stripTokens(token.tokens, sections, token.type === "heading"); } switch (token.type) { case "space": - out += token.raw; + sections[index][header ? "header" : "content"] += token.raw; break; case "code": if (token.lang != "math") { - out += token.text; + sections[index][header ? "header" : "content"] += token.text; } break; case "heading": - out += "\n\n"; break; case "table": for (const cell of token.header) { - out += stripTokens(cell.tokens) + " "; + stripTokens(cell.tokens, sections, header); + sections[index][header ? "header" : "content"] += " "; } - out += "\n"; + sections[index][header ? "header" : "content"] += "\n"; for (const row of token.rows) { for (const cell of row) { - out += stripTokens(cell.tokens) + " "; + stripTokens(cell.tokens, sections, header); + sections[index][header ? "header" : "content"] += " "; } - out += "\n"; + sections[index][header ? "header" : "content"] += "\n"; } break; case "hr": @@ -380,24 +400,25 @@ function stripTokens(tokens: Marked.Token[]): string { case "blockquote": break; case "list": - out += stripTokens(token.items); + stripTokens(token.items, sections, header); break; case "list_item": - out += "\n"; + sections[index][header ? "header" : "content"] += "\n"; break; case "paragraph": break; case "html": { // TODO: extract alt from img - out += sanitizeHtml(token.text, { - allowedTags: [], - allowedAttributes: {}, - }).trim() + "\n\n"; + sections[index][header ? "header" : "content"] += + sanitizeHtml(token.text, { + allowedTags: [], + allowedAttributes: {}, + }).trim() + "\n\n"; break; } case "text": if (!("tokens" in token) || !token.tokens) { - out += token.raw; + sections[index][header ? "header" : "content"] += token.raw; } break; case "def": @@ -408,9 +429,9 @@ function stripTokens(tokens: Marked.Token[]): string { break; case "image": if (token.title) { - out += token.title; + sections[index][header ? "header" : "content"] += token.title; } else { - out += token.text; + sections[index][header ? "header" : "content"] += token.text; } break; case "strong": @@ -418,7 +439,7 @@ function stripTokens(tokens: Marked.Token[]): string { case "em": break; case "codespan": - out += token.text; + sections[index][header ? "header" : "content"] += token.text; break; case "br": break; @@ -426,8 +447,6 @@ function stripTokens(tokens: Marked.Token[]): string { break; } } - - return out; } class StripTokenizer extends Marked.Tokenizer { @@ -450,10 +469,22 @@ class StripTokenizer extends Marked.Tokenizer { } } +export interface MarkdownSections { + /** The header of the section */ + header: string; + /** The depth-level of the header. 0 if it is root level */ + depth: number; + content: string; +} + /** - * Strip all markdown syntax to get a plaintext output + * Strip all markdown syntax to get a plaintext output, divided up in sections + * based on headers */ -export function strip(markdown: string, opts: RenderOptions = {}): string { +export function stripSplitBySections( + markdown: string, + opts: RenderOptions = {}, +): MarkdownSections[] { markdown = emojify(markdown).replace(BLOCK_MATH_REGEXP, "").replace( INLINE_MATH_REGEXP, "", @@ -462,5 +493,22 @@ export function strip(markdown: string, opts: RenderOptions = {}): string { ...getOpts(opts), tokenizer: new StripTokenizer(), }); - return stripTokens(tokens).trim().replace(/\n{3,}/g, "\n") + "\n"; + + const sections: MarkdownSections[] = [{ + header: "", + depth: 0, + content: "", + }]; + stripTokens(tokens, sections, false); + + return sections; +} + +/** + * Strip all markdown syntax to get a plaintext output + */ +export function strip(markdown: string, opts: RenderOptions = {}): string { + return stripSplitBySections(markdown, opts).map((section) => + section.header + "\n\n" + section.content + ).join("\n\n").trim().replace(/\n{3,}/g, "\n") + "\n"; } diff --git a/test/fixtures/example.html b/test/fixtures/example.html new file mode 100644 index 0000000..a448ff4 --- /dev/null +++ b/test/fixtures/example.html @@ -0,0 +1,195 @@ +
{
+  "json": {
+    "name": "Deno"
+  }
+}
- hello
++ world
+
import { build } from "https://deno.land/x/esbuild/mod.ts";
+import sassPlugin from "https://deno.land/x/esbuild_plugin_sass_deno/mod.ts";
+
+build({
+  entryPoints: [
+    "example/in.ts",
+  ],
+  bundle: true,
+  outfile: "example/out.js",
+  plugins: [sassPlugin()],
+});
+
import styles from "./styles.scss";
+
+document.getElementsByTagName("head")[0].innerHTML +=
+  `<style>${styles}</style>`;

Some strikethrough text

+
+ Summary +

Some Details + +

even more details

+

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TypeDescriptionexample
stringA string of characters.'Hello world'
numberA numeric value, either float or integer.42
booleanA boolean value.true
enumAn enum value.'drama'
geopointA geopoint value.{ lat: 40.7128, lon: 74.0060 }
string[]An array of strings.['red', 'green', 'blue']
number[]An array of numbers.[42, 91, 28.5]
boolean[]An array of booleans.[true, false, false]
enum[]An array of enums.['comedy', 'action', 'romance']
vector[<size>]A vector of numbers to perform vector search on.[0.403, 0.192, 0.830]
+

Math rendering

+

We support code blocks with the "math" type!

+
G_{\mu v} = \frac{8 \pi G}{c^4} T_{\mu v}

We also support math blocks and inline math blocks as well!

+

When $a \ne 0$, there are two solutions to $(ax^2 + bx + c = 0)$ and they are

+

$$ x = {-b \pm \sqrt{b^2-4ac} \over 2a} $$

+

You can even typeset individual letters or whole sentences inline just like $x$ +or $Quadratic ; formula$. You can also use math blocks to typeset whole +equations with $\LaTeX$:

+

$$ \begin{aligned} \dot{x} & = \sigma(y-x) \ \dot{y} & = \rho x - y - xz \ +\dot{z} & = -\beta z + xy \end{aligned} $$

+

Deno

+

Build Status - Cirrus Twitter handle +Discord Chat

+the deno mascot dinosaur standing in the rain + +

Deno is a simple, modern and secure runtime for JavaScript and +TypeScript that uses V8 and is built in Rust.

+

Features

+ +

Install

+

Shell (Mac, Linux):

+
curl -fsSL https://deno.land/x/install/install.sh | sh

PowerShell (Windows):

+
iwr https://deno.land/x/install/install.ps1 -useb | iex

Homebrew (Mac):

+
brew install deno

Chocolatey (Windows):

+
choco install deno

Scoop (Windows):

+
scoop install deno

Build and install from source using Cargo:

+
cargo install deno --locked

See +deno_install +and releases for other options.

+

Getting Started

+

Try running a simple program:

+
deno run https://deno.land/std/examples/welcome.ts

Or a more complex one:

+
const listener = Deno.listen({ port: 8000 });
+console.log("http://localhost:8000/");
+
+for await (const conn of listener) {
+  serve(conn);
+}
+
+async function serve(conn: Deno.Conn) {
+  for await (const { respondWith } of Deno.serveHttp(conn)) {
+    respondWith(new Response("Hello world"));
+  }
+}

You can find a deeper introduction, examples, and environment setup guides in +the manual.

+ + +

The complete API reference is available at the runtime +documentation.

+

Contributing

+

We appreciate your help!

+

To contribute, please read our +contributing instructions.

+
/** @jsx h */
+import { h, IS_BROWSER, useState } from "../deps.ts";
+
+export default function Home() {
+  return (
+    <div>
+      <p>
+        Welcome to `fresh`. Try update this message in the ./pages/index.tsx
+        file, and refresh.
+      </p>
+      <Counter />
+      <p>{IS_BROWSER ? "Viewing browser render." : "Viewing JIT render."}</p>
+    </div>
+  );
+}
+
+function Counter() {
+  const [count, setCount] = useState(0);
+  return (
+    <div>
+      <p>{count}</p>
+      <button
+        onClick={() => setCount(count - 1)}
+        disabled={!IS_BROWSER}
+      >
+        -1
+      </button>
+      <button
+        onClick={() => setCount(count + 1)}
+        disabled={!IS_BROWSER}
+      >
+        +1
+      </button>
+    </div>
+  );
+}
+
+export const config: PageConfig = { runtimeJS: true };
+ +
Figure 1. The deno mascot dinosaur standing in the rain.
+
diff --git a/test/fixtures/example.json b/test/fixtures/example.json new file mode 100644 index 0000000..431d486 --- /dev/null +++ b/test/fixtures/example.json @@ -0,0 +1,37 @@ +[ + { + "header": "", + "depth": 0, + "content": "{\n \"json\": {\n \"name\": \"Deno\"\n }\n}\n\n- hello\n+ world\n\nBuildscript\nimport { build } from \"https://deno.land/x/esbuild/mod.ts\";\nimport sassPlugin from \"https://deno.land/x/esbuild_plugin_sass_deno/mod.ts\";\n\nbuild({\n entryPoints: [\n \"example/in.ts\",\n ],\n bundle: true,\n outfile: \"example/out.js\",\n plugins: [sassPlugin()],\n});\n\nMain Entrypoint File:\nimport styles from \"./styles.scss\";\n\ndocument.getElementsByTagName(\"head\")[0].innerHTML +=\n ``;\n\nSome strikethrough text\n\nSummary\n Some Details\n\neven more details\nType Description example \nstring A string of characters. 'Hello world' \nnumber A numeric value, either float or integer. 42 \nboolean A boolean value. true \nenum An enum value. 'drama' \ngeopoint A geopoint value. { lat: 40.7128, lon: 74.0060 } \nstring[] An array of strings. ['red', 'green', 'blue'] \nnumber[] An array of numbers. [42, 91, 28.5] \nboolean[] An array of booleans. [true, false, false] \nenum[] An array of enums. ['comedy', 'action', 'romance'] \nvector[] A vector of numbers to perform vector search on. [0.403, 0.192, 0.830]" + }, + { + "header": "Math rendering", + "depth": 2, + "content": "We support code blocks with the \"math\" type!\nWe also support math blocks and inline math blocks as well!\n\nWhen, there are two solutions to and they are\nYou can even typeset individual letters or whole sentences inline just like\nor. You can also use math blocks to typeset whole\nequations with:\n \\begin{aligned} \\dot{x} & = \\sigma(y-x) \\dot{y} & = \\rho x - y - xz \n\\dot{z} & = -\\beta z + xy \\end{aligned}" + }, + { + "header": "Deno", + "depth": 1, + "content": "Build Status - Cirrus Twitter handle\nDiscord Chat\nDeno is a simple, modern and secure runtime for JavaScript and\nTypeScript that uses V8 and is built in Rust." + }, + { + "header": "Features", + "depth": 3, + "content": "Secure by default. No file, network, or environment access, unless explicitly\nenabled.\nSupports TypeScript out of the box.\nShips only a single executable file.\nBuilt-in utilities like a dependency inspector (deno info) and a code\nformatter (deno fmt).\nSet of reviewed standard modules that are guaranteed to work with\nDeno." + }, + { + "header": "Install", + "depth": 3, + "content": "Shell (Mac, Linux):\n\ncurl -fsSL https://deno.land/x/install/install.sh | sh\n\nPowerShell (Windows):\n\niwr https://deno.land/x/install/install.ps1 -useb | iex\n\nHomebrew (Mac):\n\nbrew install deno\n\nChocolatey (Windows):\n\nchoco install deno\n\nScoop (Windows):\n\nscoop install deno\n\nBuild and install from source using Cargo:\n\ncargo install deno --locked\n\nSee\ndeno_install\nand releases for other options." + }, + { + "header": "Getting Started", + "depth": 3, + "content": "Try running a simple program:\n\ndeno run https://deno.land/std/examples/welcome.ts\n\nOr a more complex one:\n\nconst listener = Deno.listen({ port: 8000 });\nconsole.log(\"http://localhost:8000/\");\n\nfor await (const conn of listener) {\n serve(conn);\n}\n\nasync function serve(conn: Deno.Conn) {\n for await (const { respondWith } of Deno.serveHttp(conn)) {\n respondWith(new Response(\"Hello world\"));\n }\n}\n\nYou can find a deeper introduction, examples, and environment setup guides in\nthe manual.\nThe complete API reference is available at the runtime\ndocumentation." + }, + { + "header": "Contributing", + "depth": 3, + "content": "We appreciate your help!\n\nTo contribute, please read our\ncontributing instructions.\n\n/** @jsx h */\nimport { h, IS_BROWSER, useState } from \"../deps.ts\";\n\nexport default function Home() {\n return (\n
\n

\n Welcome to `fresh`. Try update this message in the ./pages/index.tsx\n file, and refresh.\n

\n \n

{IS_BROWSER ? \"Viewing browser render.\" : \"Viewing JIT render.\"}

\n
\n );\n}\n\nfunction Counter() {\n const [count, setCount] = useState(0);\n return (\n
\n

{count}

\n setCount(count - 1)}\n disabled={!IS_BROWSER}\n >\n -1\n \n setCount(count + 1)}\n disabled={!IS_BROWSER}\n >\n +1\n \n
\n );\n}\n\nexport const config: PageConfig = { runtimeJS: true };\n\nFigure 1. The deno mascot dinosaur standing in the rain.\n\n" + } +] diff --git a/test/fixtures/example.strip b/test/fixtures/example.strip new file mode 100644 index 0000000..256127d --- /dev/null +++ b/test/fixtures/example.strip @@ -0,0 +1,179 @@ +{ + "json": { + "name": "Deno" + } +} + +- hello ++ world + +Buildscript +import { build } from "https://deno.land/x/esbuild/mod.ts"; +import sassPlugin from "https://deno.land/x/esbuild_plugin_sass_deno/mod.ts"; + +build({ + entryPoints: [ + "example/in.ts", + ], + bundle: true, + outfile: "example/out.js", + plugins: [sassPlugin()], +}); + +Main Entrypoint File: +import styles from "./styles.scss"; + +document.getElementsByTagName("head")[0].innerHTML += + ``; + +Some strikethrough text + +Summary + Some Details + +even more details +Type Description example +string A string of characters. 'Hello world' +number A numeric value, either float or integer. 42 +boolean A boolean value. true +enum An enum value. 'drama' +geopoint A geopoint value. { lat: 40.7128, lon: 74.0060 } +string[] An array of strings. ['red', 'green', 'blue'] +number[] An array of numbers. [42, 91, 28.5] +boolean[] An array of booleans. [true, false, false] +enum[] An array of enums. ['comedy', 'action', 'romance'] +vector[] A vector of numbers to perform vector search on. [0.403, 0.192, 0.830] + +Math rendering + +We support code blocks with the "math" type! +We also support math blocks and inline math blocks as well! + +When, there are two solutions to and they are +You can even typeset individual letters or whole sentences inline just like +or. You can also use math blocks to typeset whole +equations with: + \begin{aligned} \dot{x} & = \sigma(y-x) \dot{y} & = \rho x - y - xz +\dot{z} & = -\beta z + xy \end{aligned} + +Deno + +Build Status - Cirrus Twitter handle +Discord Chat +Deno is a simple, modern and secure runtime for JavaScript and +TypeScript that uses V8 and is built in Rust. + +Features + +Secure by default. No file, network, or environment access, unless explicitly +enabled. +Supports TypeScript out of the box. +Ships only a single executable file. +Built-in utilities like a dependency inspector (deno info) and a code +formatter (deno fmt). +Set of reviewed standard modules that are guaranteed to work with +Deno. + +Install + +Shell (Mac, Linux): + +curl -fsSL https://deno.land/x/install/install.sh | sh + +PowerShell (Windows): + +iwr https://deno.land/x/install/install.ps1 -useb | iex + +Homebrew (Mac): + +brew install deno + +Chocolatey (Windows): + +choco install deno + +Scoop (Windows): + +scoop install deno + +Build and install from source using Cargo: + +cargo install deno --locked + +See +deno_install +and releases for other options. + +Getting Started + +Try running a simple program: + +deno run https://deno.land/std/examples/welcome.ts + +Or a more complex one: + +const listener = Deno.listen({ port: 8000 }); +console.log("http://localhost:8000/"); + +for await (const conn of listener) { + serve(conn); +} + +async function serve(conn: Deno.Conn) { + for await (const { respondWith } of Deno.serveHttp(conn)) { + respondWith(new Response("Hello world")); + } +} + +You can find a deeper introduction, examples, and environment setup guides in +the manual. +The complete API reference is available at the runtime +documentation. + +Contributing + +We appreciate your help! + +To contribute, please read our +contributing instructions. + +/** @jsx h */ +import { h, IS_BROWSER, useState } from "../deps.ts"; + +export default function Home() { + return ( +
+

+ Welcome to `fresh`. Try update this message in the ./pages/index.tsx + file, and refresh. +

+ +

{IS_BROWSER ? "Viewing browser render." : "Viewing JIT render."}

+
+ ); +} + +function Counter() { + const [count, setCount] = useState(0); + return ( +
+

{count}

+ + +
+ ); +} + +export const config: PageConfig = { runtimeJS: true }; + +Figure 1. The deno mascot dinosaur standing in the rain. diff --git a/test/test.ts b/test/test.ts index ebd24df..ca35f16 100644 --- a/test/test.ts +++ b/test/test.ts @@ -1,6 +1,6 @@ import { assertEquals, assertStringIncludes } from "@std/assert"; import { DOMParser } from "https://deno.land/x/deno_dom@v0.1.43/deno-dom-wasm.ts"; -import { render, Renderer, strip } from "../mod.ts"; +import { render, Renderer, strip, stripSplitBySections } from "../mod.ts"; Deno.test("Basic markdown", async () => { const markdown = await Deno.readTextFile("./test/fixtures/basic.md"); @@ -427,3 +427,16 @@ Deno.test("anchor test", () => { const html = render(markdown); assertEquals(html, result); }); + +Deno.test("example file", () => { + const markdown = Deno.readTextFileSync("./example/content.md"); + const expectedHTML = Deno.readTextFileSync("./test/fixtures/example.html"); + const expectedStrip = Deno.readTextFileSync("./test/fixtures/example.strip"); + const expectedJSON = JSON.parse( + Deno.readTextFileSync("./test/fixtures/example.json"), + ); + + assertEquals(render(markdown), expectedHTML); + assertEquals(strip(markdown), expectedStrip); + assertEquals(stripSplitBySections(markdown), expectedJSON); +});