From ea032b356e3ba7c24936e3586a3eb48ecbd685c7 Mon Sep 17 00:00:00 2001 From: Xiaoxing Hu Date: Sat, 11 Nov 2023 11:04:41 +1300 Subject: [PATCH] fix bugs in editor (#219) * [editor] fix tree length vs. doc length mismatch in parsing when there are ignored elements, the length didn't match. which cause extra parsing and the syntax highlight was failing * [editor] create "generate link button" * add changeset --- .changeset/proud-pets-retire.md | 7 ++++++ packages/lezer/lib/context.js | 11 ++++++++-- packages/lezer/lib/fragments.js | 24 ++++++++++++++++---- packages/lezer/lib/index.js | 8 ++++--- packages/lezer/lib/nodes.js | 2 +- packages/lezer/lib/oast-to-lezer.js | 2 +- packages/lezer/lib/tree.js | 19 +++++++++++----- packages/lezer/test/incremental.test.js | 4 ++++ website/src/components/playground.tsx | 29 +++++++++++++++++++------ website/src/utils/org.ts | 9 ++++++-- 10 files changed, 90 insertions(+), 25 deletions(-) create mode 100644 .changeset/proud-pets-retire.md diff --git a/.changeset/proud-pets-retire.md b/.changeset/proud-pets-retire.md new file mode 100644 index 00000000..251243c9 --- /dev/null +++ b/.changeset/proud-pets-retire.md @@ -0,0 +1,7 @@ +--- +'@orgajs/editor': patch +'@orgajs/lezer': patch +'website': patch +--- + +bug fix, link generating button diff --git a/packages/lezer/lib/context.js b/packages/lezer/lib/context.js index 5b298a9b..6c400c96 100644 --- a/packages/lezer/lib/context.js +++ b/packages/lezer/lib/context.js @@ -44,10 +44,16 @@ export function parseContext(config, input, _fragments, ranges) { */ function advance() { log('advance called', cursor, end) + // TODO: should we set an end to the tree? + // will take a look when this actually happens if (stoppedAt != null && cursor > stoppedAt) return builder.build() if (cursor >= end) { - return builder.build() + // end of the tree has to match the end of the input + // sometimes the the chidren does not fill the whole tree + // which causes the end of the tree to be smaller than the end of the input + // set it to the end of the input will prevent unnecessary parsing + return builder.build(end) } if (fragments !== null && couldReuse(fragments)) { @@ -86,8 +92,9 @@ export function parseContext(config, input, _fragments, ranges) { if (!parser) return const document = parser.finish() log( - `wrap up parser: ${document.children.length}, ${document.position?.start.offset}, ${document.position?.end.offset}` + `wrap up parser: ${document.children.length}, ${document.position?.start.offset}, ${document.position?.end.offset}`, ) + log(document) const tree = toLezer(document, nodeSet) builder.takeChildren(tree, document.position?.start.offset) if (document.position?.end.offset) cursor = document.position.end.offset diff --git a/packages/lezer/lib/fragments.js b/packages/lezer/lib/fragments.js index cabdf5d2..774aec66 100644 --- a/packages/lezer/lib/fragments.js +++ b/packages/lezer/lib/fragments.js @@ -97,16 +97,21 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) { taken: 0, } + log('takeNodes', start, ranges, rangeI) + if (!cursor || !fragment) return result let cur = cursor let off = fragment.offset let end = start + // let prevEnd = end + let blockI = 0 + // let prevI = 0 const fragEnd = fragmentEnd - (fragment.openEnd ? 1 : 0) for (;;) { if (cur.to - off > fragEnd) { if (cur.type.isAnonymous && cur.firstChild()) continue log( - `stop takeNodes from ${cur.name}: fe: ${fragEnd} | ${fragmentEnd}, cur: ${cur.name}, cur.from: ${cur.from}, cur.to: ${cur.to}, off: ${off}` + `stop takeNodes from ${cur.name}: fe: ${fragEnd} | ${fragmentEnd}, cur: ${cur.name}, cur.from: ${cur.from}, cur.to: ${cur.to}, off: ${off}`, ) break } @@ -115,7 +120,7 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) { if (cur.to - off <= ranges[rangeI].to) { // Fits in current range log( - `Fits in current range, name: ${cur.name}, pos: ${pos}, cur.from: ${cur.from}, cur.to: ${cur.to}` + `Fits in current range, name: ${cur.name}, pos: ${pos}, cur.from: ${cur.from}, cur.to: ${cur.to}`, ) const tree = cur.tree @@ -129,7 +134,7 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) { nodeSet.types[nodes.paragraph], [], [], - 0 + 0, // cx.block.hashProp ) result.nodes.push(dummy) @@ -137,9 +142,20 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) { // reuse dummy? } - end = cur.to - off + if (cur.type.is('Block')) { + log('got block:', cur.type.name) + end = cur.to - off + blockI = result.nodes.length + } if (!cur.nextSibling()) break } + log(`>> popping: ${blockI} / ${result.nodes.length}`) + log('node:', result.nodes.map((n) => n.type.name).join(',')) + while (result.nodes.length > blockI) { + const n = result.nodes.pop() + log('.. pop non blocks', n?.type.name) + result.positions.pop() + } result.taken = end - start return result diff --git a/packages/lezer/lib/index.js b/packages/lezer/lib/index.js index 12645175..dc711a0b 100644 --- a/packages/lezer/lib/index.js +++ b/packages/lezer/lib/index.js @@ -35,9 +35,11 @@ export class OrgParser extends Parser { const r = ranges.map((r) => `${r.from}-${r.to}`).join(', ') const frags = fragments .map( - (f) => `(${f.from}-${f.to}, offset: ${f.offset}, openEnd: ${f.openEnd})` + (f) => + `(${f.from}-${f.to}, offset: ${f.offset}, openEnd: ${f.openEnd})`, ) .join(' ') + // TODO: add info more about fragments, how do I use ranges vs fragments? this.log('createParse', `ranges: (${r}), frags: [${frags}]`) const parse = parseContext(this, input, fragments, ranges) return parse @@ -59,6 +61,6 @@ export class OrgParser extends Parser { } export const parser = new OrgParser( - nodeSet - // console.log.bind(console, 'orga-parser') + nodeSet, + // console.log.bind(console, 'orga-parser'), ) diff --git a/packages/lezer/lib/nodes.js b/packages/lezer/lib/nodes.js index 1f298ba0..6e128ebb 100644 --- a/packages/lezer/lib/nodes.js +++ b/packages/lezer/lib/nodes.js @@ -43,7 +43,7 @@ export const nodeTypes = Object.entries(nodes).map(([name, id]) => name, props: id >= nodes.stars ? [] : [[NodeProp.group, ['Block']]], top: name === 'document', - }) + }), ) // extra tags diff --git a/packages/lezer/lib/oast-to-lezer.js b/packages/lezer/lib/oast-to-lezer.js index 5407fa58..9a216f1f 100644 --- a/packages/lezer/lib/oast-to-lezer.js +++ b/packages/lezer/lib/oast-to-lezer.js @@ -27,7 +27,7 @@ function createParseState(file, nodeSet) { /** @type {ParseState} */ const state = { file, - ignore: ['newline', 'emptyLine', 'section'], + ignore: ['section', 'newline', 'emptyLine'], nodeSet: nodeSet, handlers, one(node, parent, base = 0) { diff --git a/packages/lezer/lib/tree.js b/packages/lezer/lib/tree.js index 5f7f3a66..a676cb7d 100644 --- a/packages/lezer/lib/tree.js +++ b/packages/lezer/lib/tree.js @@ -1,4 +1,5 @@ import { NodeProp, NodeType, Tree } from '@lezer/common' +// import { nodes } from './nodes.js' /** * @param {number} type @@ -15,9 +16,16 @@ function hash(type, value, parentHash = 0) { * @param {number} value * @param {number} from * @param {number} parentHash - * @param {number} end + * @param {number} originalEnd */ -export function treeBuilder(nodeSet, type, value, from, parentHash, end) { +export function treeBuilder( + nodeSet, + type, + value, + from, + parentHash, + originalEnd, +) { /** @type {Tree[]} */ const children = [] /** @type {number[]} */ @@ -39,7 +47,7 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) { child.children, child.positions, child.length, - props + props, ) children.push(child) positions.push(pos) @@ -71,9 +79,10 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) { } /** + * @param {number} [end = originalEnd] - end of the tree * @returns {Tree} */ - function build() { + function build(end = originalEnd) { let last = children.length - 1 if (last >= 0) end = Math.max(end, positions[last] + children[last].length + from) @@ -82,7 +91,7 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) { nodeSet.types[type], children, positions, - end - from + end - from, ).balance({ makeTree: (children, positions, length) => new Tree(NodeType.none, children, positions, length, props), diff --git a/packages/lezer/test/incremental.test.js b/packages/lezer/test/incremental.test.js index 136f4257..3d0db6bb 100644 --- a/packages/lezer/test/incremental.test.js +++ b/packages/lezer/test/incremental.test.js @@ -17,6 +17,7 @@ console.log('hello') here is a link: [[https://example.com][link text]] ` +const docLength = doc.length class State { constructor(doc, tree, fragments) { @@ -128,4 +129,7 @@ describe('incremential parsing', () => { // it('can handle adding to a quoted block', () => {}) // it('can handle a change in a post-linkref paragraph', () => {}) // it('can handle a change in a paragraph-adjacent linkrefs', () => {}) + + it('can handle insertion at the eof', () => + testChange([{ from: docLength, to: docLength, insert: '* h' }])) }) diff --git a/website/src/components/playground.tsx b/website/src/components/playground.tsx index 7349a28b..f2682229 100644 --- a/website/src/components/playground.tsx +++ b/website/src/components/playground.tsx @@ -35,6 +35,13 @@ export default function Playground({ content }: { content: string }) { [setValue] ) + function copyToClipboard() { + const content = encodeURIComponent(state.original) + const host = window.location.host + const link = `${host}/playground?text=${content}` + navigator.clipboard.writeText(link) + } + const tabs = [ { tab: 'render', @@ -77,13 +84,21 @@ export default function Playground({ content }: { content: string }) { ] return ( - - - +
+ + + + +
) } diff --git a/website/src/utils/org.ts b/website/src/utils/org.ts index d48962dd..41d4b619 100644 --- a/website/src/utils/org.ts +++ b/website/src/utils/org.ts @@ -11,12 +11,17 @@ import { debounce } from './debounce' import { removePosition } from 'unist-util-remove-position' interface OrgState { + original: string file: VFile | null Content: OrgContent | null } export function useOrg({ value }: { value: string }) { - const [state, setState] = useState({ file: null }) + const [state, setState] = useState({ + original: value, + file: null, + Content: null, + }) const filename = 'example.org' useEffect(() => { @@ -46,7 +51,7 @@ export function useOrg({ value }: { value: string }) { ).default removePosition(file.data.oast, { force: true }) removePosition(file.data.hast, { force: true }) - setState({ file, Content }) + setState({ original: content, file, Content }) } return { state, setValue: debounce(setValue, 300) } }