fix bugs in editor (#219)

* [editor] fix tree length vs. doc length mismatch in parsing

when there are ignored elements, the length didn't match. which cause
extra parsing and the syntax highlight was failing

* [editor] create "generate link button"

* add changeset

.changeset/proud-pets-retire.md

@@ -0,0 +1,7 @@
+---
+'@orgajs/editor': patch
+'@orgajs/lezer': patch
+'website': patch
+---
+
+bug fix, link generating button

packages/lezer/lib/context.js

@@ -44,10 +44,16 @@ export function parseContext(config, input, _fragments, ranges) {
    */
   function advance() {
     log('advance called', cursor, end)
+    // TODO: should we set an end to the tree?
+    // will take a look when this actually happens
     if (stoppedAt != null && cursor > stoppedAt) return builder.build()
 
     if (cursor >= end) {
-      return builder.build()
+      // end of the tree has to match the end of the input
+      // sometimes the the chidren does not fill the whole tree
+      // which causes the end of the tree to be smaller than the end of the input
+      // set it to the end of the input will prevent unnecessary parsing
+      return builder.build(end)
     }
 
     if (fragments !== null && couldReuse(fragments)) {
@@ -86,8 +92,9 @@ export function parseContext(config, input, _fragments, ranges) {
     if (!parser) return
     const document = parser.finish()
     log(
-      `wrap up parser: ${document.children.length}, ${document.position?.start.offset}, ${document.position?.end.offset}`
+      `wrap up parser: ${document.children.length}, ${document.position?.start.offset}, ${document.position?.end.offset}`,
     )
+    log(document)
     const tree = toLezer(document, nodeSet)
     builder.takeChildren(tree, document.position?.start.offset)
     if (document.position?.end.offset) cursor = document.position.end.offset

packages/lezer/lib/fragments.js

@@ -97,16 +97,21 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) {
       taken: 0,
     }
 
+    log('takeNodes', start, ranges, rangeI)
+
     if (!cursor || !fragment) return result
     let cur = cursor
     let off = fragment.offset
     let end = start
+    // let prevEnd = end
+    let blockI = 0
+    // let prevI = 0
     const fragEnd = fragmentEnd - (fragment.openEnd ? 1 : 0)
     for (;;) {
       if (cur.to - off > fragEnd) {
         if (cur.type.isAnonymous && cur.firstChild()) continue
         log(
-          `stop takeNodes from ${cur.name}: fe: ${fragEnd} | ${fragmentEnd}, cur: ${cur.name}, cur.from: ${cur.from}, cur.to: ${cur.to}, off: ${off}`
+          `stop takeNodes from ${cur.name}: fe: ${fragEnd} | ${fragmentEnd}, cur: ${cur.name}, cur.from: ${cur.from}, cur.to: ${cur.to}, off: ${off}`,
         )
         break
       }
@@ -115,7 +120,7 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) {
       if (cur.to - off <= ranges[rangeI].to) {
         // Fits in current range
         log(
-          `Fits in current range, name: ${cur.name}, pos: ${pos}, cur.from: ${cur.from}, cur.to: ${cur.to}`
+          `Fits in current range, name: ${cur.name}, pos: ${pos}, cur.from: ${cur.from}, cur.to: ${cur.to}`,
         )
         const tree = cur.tree
 
@@ -129,17 +134,28 @@ export function fragmentCursor({ log, nodeSet }, fragments, input) {
           nodeSet.types[nodes.paragraph],
           [],
           [],
-          0
+          0,
           // cx.block.hashProp
         )
         result.nodes.push(dummy)
         result.positions.push(pos)
         // reuse dummy?
       }
 
-      end = cur.to - off
+      if (cur.type.is('Block')) {
+        log('got block:', cur.type.name)
+        end = cur.to - off
+        blockI = result.nodes.length
+      }
       if (!cur.nextSibling()) break
     }
+    log(`>> popping: ${blockI} / ${result.nodes.length}`)
+    log('node:', result.nodes.map((n) => n.type.name).join(','))
+    while (result.nodes.length > blockI) {
+      const n = result.nodes.pop()
+      log('.. pop non blocks', n?.type.name)
+      result.positions.pop()
+    }
 
     result.taken = end - start
     return result

packages/lezer/lib/index.js

@@ -35,9 +35,11 @@ export class OrgParser extends Parser {
     const r = ranges.map((r) => `${r.from}-${r.to}`).join(', ')
     const frags = fragments
       .map(
-        (f) => `(${f.from}-${f.to}, offset: ${f.offset}, openEnd: ${f.openEnd})`
+        (f) =>
+          `(${f.from}-${f.to}, offset: ${f.offset}, openEnd: ${f.openEnd})`,
       )
       .join(' ')
+    // TODO: add info more about fragments, how do I use ranges vs fragments?
     this.log('createParse', `ranges: (${r}), frags: [${frags}]`)
     const parse = parseContext(this, input, fragments, ranges)
     return parse
@@ -59,6 +61,6 @@ export class OrgParser extends Parser {
 }
 
 export const parser = new OrgParser(
-  nodeSet
-  // console.log.bind(console, 'orga-parser')
+  nodeSet,
+  // console.log.bind(console, 'orga-parser'),
 )

packages/lezer/lib/nodes.js

@@ -43,7 +43,7 @@ export const nodeTypes = Object.entries(nodes).map(([name, id]) =>
     name,
     props: id >= nodes.stars ? [] : [[NodeProp.group, ['Block']]],
     top: name === 'document',
-  })
+  }),
 )
 
 // extra tags

packages/lezer/lib/oast-to-lezer.js

@@ -27,7 +27,7 @@ function createParseState(file, nodeSet) {
   /** @type {ParseState} */
   const state = {
     file,
-    ignore: ['newline', 'emptyLine', 'section'],
+    ignore: ['section', 'newline', 'emptyLine'],
     nodeSet: nodeSet,
     handlers,
     one(node, parent, base = 0) {

packages/lezer/lib/tree.js

@@ -1,4 +1,5 @@
 import { NodeProp, NodeType, Tree } from '@lezer/common'
+// import { nodes } from './nodes.js'
 
 /**
  * @param {number} type
@@ -15,9 +16,16 @@ function hash(type, value, parentHash = 0) {
  * @param {number} value
  * @param {number} from
  * @param {number} parentHash
- * @param {number} end
+ * @param {number} originalEnd
  */
-export function treeBuilder(nodeSet, type, value, from, parentHash, end) {
+export function treeBuilder(
+  nodeSet,
+  type,
+  value,
+  from,
+  parentHash,
+  originalEnd,
+) {
   /** @type {Tree[]} */
   const children = []
   /** @type {number[]} */
@@ -39,7 +47,7 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) {
         child.children,
         child.positions,
         child.length,
-        props
+        props,
       )
     children.push(child)
     positions.push(pos)
@@ -71,9 +79,10 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) {
   }
 
   /**
+   * @param {number} [end = originalEnd] - end of the tree
    * @returns {Tree}
    */
-  function build() {
+  function build(end = originalEnd) {
     let last = children.length - 1
     if (last >= 0)
       end = Math.max(end, positions[last] + children[last].length + from)
@@ -82,7 +91,7 @@ export function treeBuilder(nodeSet, type, value, from, parentHash, end) {
       nodeSet.types[type],
       children,
       positions,
-      end - from
+      end - from,
     ).balance({
       makeTree: (children, positions, length) =>
         new Tree(NodeType.none, children, positions, length, props),

packages/lezer/test/incremental.test.js

@@ -17,6 +17,7 @@ console.log('hello')
 
 here is a link: [[https://example.com][link text]]
 `
+const docLength = doc.length
 
 class State {
   constructor(doc, tree, fragments) {
@@ -128,4 +129,7 @@ describe('incremential parsing', () => {
   // it('can handle adding to a quoted block', () => {})
   // it('can handle a change in a post-linkref paragraph', () => {})
   // it('can handle a change in a paragraph-adjacent linkrefs', () => {})
+
+  it('can handle insertion at the eof', () =>
+    testChange([{ from: docLength, to: docLength, insert: '* h' }]))
 })

website/src/components/playground.tsx

@@ -35,6 +35,13 @@ export default function Playground({ content }: { content: string }) {
     [setValue]
   )
 
+  function copyToClipboard() {
+    const content = encodeURIComponent(state.original)
+    const host = window.location.host
+    const link = `${host}/playground?text=${content}`
+    navigator.clipboard.writeText(link)
+  }
+
   const tabs = [
     {
       tab: 'render',
@@ -77,13 +84,21 @@ export default function Playground({ content }: { content: string }) {
   ]
 
   return (
-    <Magic tabs={tabs}>
-      <OrgEditor
-        className="h-full w-full not-prose"
-        onChange={onChange}
-        content={content}
-      />
-    </Magic>
+    <div className="h-full w-full">
+      <Magic tabs={tabs}>
+        <OrgEditor
+          className="h-full w-full not-prose"
+          onChange={onChange}
+          content={content}
+        />
+      </Magic>
+      <button
+        className="fixed right-2 bottom-2 text-white bg-blue-600 p-2 px-4 rounded hover:bg-blue-500 border border-blue-600"
+        onClick={copyToClipboard}
+      >
+        generate link
+      </button>
+    </div>
   )
 }
 

website/src/utils/org.ts

@@ -11,12 +11,17 @@ import { debounce } from './debounce'
 import { removePosition } from 'unist-util-remove-position'
 
 interface OrgState {
+  original: string
   file: VFile | null
   Content: OrgContent | null
 }
 
 export function useOrg({ value }: { value: string }) {
-  const [state, setState] = useState<OrgState>({ file: null })
+  const [state, setState] = useState<OrgState>({
+    original: value,
+    file: null,
+    Content: null,
+  })
   const filename = 'example.org'
 
   useEffect(() => {
@@ -46,7 +51,7 @@ export function useOrg({ value }: { value: string }) {
     ).default
     removePosition(file.data.oast, { force: true })
     removePosition(file.data.hast, { force: true })
-    setState({ file, Content })
+    setState({ original: content, file, Content })
   }
   return { state, setValue: debounce(setValue, 300) }
 }