From 4f818a4b5dc482eeb78ce7a1c343a4234ddb917a Mon Sep 17 00:00:00 2001 From: Eugene Date: Fri, 15 Nov 2024 01:50:30 +0000 Subject: [PATCH] feat: add support for math blocks using $$ --- __tests__/html2/markdown/math5.html | 125 ++++++++++++++++++ .../src/markdown/mathExtension/constants.ts | 1 + .../src/markdown/mathExtension/index.ts | 19 +-- .../bundle/src/markdown/mathExtension/math.ts | 21 +++ .../{htmlRenderer.ts => mathHtml.ts} | 22 ++- .../src/markdown/mathExtension/tokenizer.ts | 58 +++++--- 6 files changed, 206 insertions(+), 40 deletions(-) create mode 100644 __tests__/html2/markdown/math5.html create mode 100644 packages/bundle/src/markdown/mathExtension/math.ts rename packages/bundle/src/markdown/mathExtension/{htmlRenderer.ts => mathHtml.ts} (65%) diff --git a/__tests__/html2/markdown/math5.html b/__tests__/html2/markdown/math5.html new file mode 100644 index 0000000000..db29ef2a56 --- /dev/null +++ b/__tests__/html2/markdown/math5.html @@ -0,0 +1,125 @@ + + + + + + + + + + + + + +
+ + + + diff --git a/packages/bundle/src/markdown/mathExtension/constants.ts b/packages/bundle/src/markdown/mathExtension/constants.ts index 67481ccdc1..cdfb3740cb 100644 --- a/packages/bundle/src/markdown/mathExtension/constants.ts +++ b/packages/bundle/src/markdown/mathExtension/constants.ts @@ -3,3 +3,4 @@ export const OPEN_PAREN = 40; // '(' export const CLOSE_PAREN = 41; // ')' export const OPEN_BRACKET = 91; // '[' export const CLOSE_BRACKET = 93; // ']' +export const DOLLAR = 36; // '$' diff --git a/packages/bundle/src/markdown/mathExtension/index.ts b/packages/bundle/src/markdown/mathExtension/index.ts index 0fe2b747de..c0c12bcc81 100644 --- a/packages/bundle/src/markdown/mathExtension/index.ts +++ b/packages/bundle/src/markdown/mathExtension/index.ts @@ -1,17 +1,2 @@ -import { BACKSLASH } from './constants'; -import { createTokenizer } from './tokenizer'; -import { type Extension } from 'micromark-util-types'; - -export function math(): Extension { - const construct = { - name: 'math', - tokenize: createTokenizer - }; - - return { - text: { [BACKSLASH]: construct }, - flow: { [BACKSLASH]: construct } - } as any; -} - -export { type CreateHtmlRendererOptions as mathHtmlOptions, default as mathHtml } from './htmlRenderer'; +export { default as math } from './math'; +export { default as mathHtml, type CreateHtmlRendererOptions as mathHtmlOptions } from './mathHtml'; diff --git a/packages/bundle/src/markdown/mathExtension/math.ts b/packages/bundle/src/markdown/mathExtension/math.ts new file mode 100644 index 0000000000..ee3ceda696 --- /dev/null +++ b/packages/bundle/src/markdown/mathExtension/math.ts @@ -0,0 +1,21 @@ +import type { Extension } from 'micromark-util-types'; +import { BACKSLASH, DOLLAR } from './constants'; +import { createTokenizer } from './tokenizer'; + +export default function math(): Extension { + const construct = { + name: 'math', + tokenize: createTokenizer + }; + + return { + text: { + [BACKSLASH]: construct, + [DOLLAR]: construct + }, + flow: { + [BACKSLASH]: construct, + [DOLLAR]: construct + } + } as any; +} diff --git a/packages/bundle/src/markdown/mathExtension/htmlRenderer.ts b/packages/bundle/src/markdown/mathExtension/mathHtml.ts similarity index 65% rename from packages/bundle/src/markdown/mathExtension/htmlRenderer.ts rename to packages/bundle/src/markdown/mathExtension/mathHtml.ts index 71d10a1f92..df7fdcc9e5 100644 --- a/packages/bundle/src/markdown/mathExtension/htmlRenderer.ts +++ b/packages/bundle/src/markdown/mathExtension/mathHtml.ts @@ -4,17 +4,25 @@ export type CreateHtmlRendererOptions = { renderMath?: ((content: string, isDisplay: boolean) => string) | undefined; }; -function extractMathContent(value) { - const isDisplay = value.startsWith('\\['); - const start = value.indexOf(isDisplay ? '[' : '(') + 1; - const end = value.lastIndexOf(isDisplay ? ']' : ')') - 1; +const delimeters = { + PAREN: ['\\(', '\\)'], + BRACKET: ['\\[', '\\]'], + DOLLAR: ['$$', '$$'] +} as const; + +function extractMathContent(value: string) { + const [mode, [startDelimiter, endDelimiter]] = Object.entries(delimeters).find(([, [start]]) => + value.startsWith(start) + ); + const start = value.indexOf(startDelimiter) + startDelimiter.length; + const end = value.lastIndexOf(endDelimiter); return { - content: value.slice(start, end).trim(), - isDisplay + content: value.substring(start, end).trim(), + isDisplay: mode === 'BRACKET' || mode === 'DOLLAR' }; } -export default function createHtmlRenderer(options: CreateHtmlRendererOptions = {}): HtmlExtension { +export default function mathHtml(options: CreateHtmlRendererOptions = {}): HtmlExtension { return { exit: { math(token: Token) { diff --git a/packages/bundle/src/markdown/mathExtension/tokenizer.ts b/packages/bundle/src/markdown/mathExtension/tokenizer.ts index 1eaa98ccd0..90c9441353 100644 --- a/packages/bundle/src/markdown/mathExtension/tokenizer.ts +++ b/packages/bundle/src/markdown/mathExtension/tokenizer.ts @@ -1,5 +1,6 @@ +/* eslint-disable no-magic-numbers */ /* eslint-disable @typescript-eslint/no-use-before-define */ -import { BACKSLASH, OPEN_PAREN, CLOSE_PAREN, OPEN_BRACKET, CLOSE_BRACKET } from './constants'; +import { BACKSLASH, OPEN_PAREN, CLOSE_PAREN, OPEN_BRACKET, CLOSE_BRACKET, DOLLAR } from './constants'; import { markdownLineEnding } from 'micromark-util-character'; import { type Code, type Effects, type State } from 'micromark-util-types'; @@ -11,27 +12,43 @@ type MathEffects = Omit & { }; export function createTokenizer(effects: MathEffects, ok: State, nok: State) { - let isDisplay = false; + let expectedCloseDelimiter: number; + let dollarDelimiterCount = 0; return start; function start(code: Code): State { - if (code !== BACKSLASH) { - return nok(code); + if (code === BACKSLASH || code === DOLLAR) { + effects.enter('math'); + effects.enter('mathChunk'); + effects.consume(code); + dollarDelimiterCount = code === DOLLAR ? 1 : 0; + return openDelimiter; } - effects.enter('math'); - effects.consume(code); - return openDelimiter; + + return nok(code); } function openDelimiter(code: Code): State { - if (code === OPEN_PAREN || code === OPEN_BRACKET) { - isDisplay = code === OPEN_BRACKET; - effects.consume(code); - effects.enter('mathChunk'); - return content; + switch (code) { + case OPEN_PAREN: + expectedCloseDelimiter = CLOSE_PAREN; + break; + case OPEN_BRACKET: + expectedCloseDelimiter = CLOSE_BRACKET; + break; + case DOLLAR: + expectedCloseDelimiter = DOLLAR; + dollarDelimiterCount++; + if (dollarDelimiterCount !== 2) { + return nok(code); + } + break; + default: + return nok(code); } - return nok(code); + effects.consume(code); + return content; } function content(code: Code): State { @@ -39,8 +56,9 @@ export function createTokenizer(effects: MathEffects, ok: State, nok: State) { return nok(code); } - if (code === BACKSLASH) { + if (code === BACKSLASH || (dollarDelimiterCount && code === DOLLAR)) { effects.consume(code); + code === DOLLAR && dollarDelimiterCount--; return maybeCloseDelimiter; } @@ -55,11 +73,19 @@ export function createTokenizer(effects: MathEffects, ok: State, nok: State) { } function maybeCloseDelimiter(code: Code): State { - if ((!isDisplay && code === CLOSE_PAREN) || (isDisplay && code === CLOSE_BRACKET)) { + if (code === expectedCloseDelimiter) { + code === DOLLAR && dollarDelimiterCount--; + if (dollarDelimiterCount !== 0) { + return nok(code); + } + effects.consume(code); effects.exit('mathChunk'); effects.exit('math'); - return ok(code); + + dollarDelimiterCount = 0; + expectedCloseDelimiter = undefined; + return ok; } return content(code);