Add support for more languages (#69)

* Contain language selection dialog in an element that can be scrolled, and automatically scroll it if needed when navigating the list with arrow keys * Add support for more languages: Clojure, Erlang, Golang, Lezer, Ruby, Shell, YAML * Move prettier auto format settings for languages into Language() class * Remove invalid import * Fix bug that could cause auto formatting to fail for the last block. Add tests for language auto detection and formatting. * Fix broken tests * Fix language auto detection on Safari Webkit which was broken * Remove unnecessary wait time
heyman · Dec 25, 2023 · bb511b8 · bb511b8
1 parent 6eda3ef
commit bb511b8
Show file tree

Hide file tree

Showing 15 changed files with 361 additions and 70 deletions.
diff --git a/README.md b/README.md
@@ -17,17 +17,24 @@ Available for Mac, Windows, and Linux.
 -   Syntax highlighting
     -   C++
     -   C#
+    -   Clojure
     -   CSS
+    -   Erlang
+    -   Go
     -   HTML
     -   Java
     -   JavaScript
     -   JSON
+    -   Lezer
     -   Markdown
     -   PHP
     -   Python
+    -   Ruby
     -   Rust
+    -   Shell
     -   SQL
     -   XML
+    -   YAML
 -   Language auto-detection
 -   Auto-formatting
 -   Math/Calculator mode

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -46,6 +46,7 @@
     "@codemirror/lang-sql": "^6.5.4",
     "@codemirror/lang-xml": "^6.0.2",
     "@codemirror/language": "^6.9.3",
+    "@codemirror/legacy-modes": "^6.3.3",
     "@codemirror/lint": "^6.4.2",
     "@codemirror/search": "^6.5.5",
     "@codemirror/state": "^6.3.3",

diff --git a/playwright.config.ts b/playwright.config.ts
@@ -41,7 +41,7 @@ export default defineConfig({
       name: 'firefox',
       use: { ...devices['Desktop Firefox'] },
     },
-
+    
     {
       name: 'webkit',
       use: { ...devices['Desktop Safari'] },

diff --git a/public/langdetect-worker.js b/public/langdetect-worker.js
@@ -14,6 +14,12 @@ GUESSLANG_LANGUAGES = [
     "rs",
     "md",
     "cs",
+    "rb",
+    "sh",
+    "yaml",
+    "go",
+    "clj",
+    "erl",
 ]
 
 const guessLang = new self.GuessLang()

diff --git a/src/components/LanguageSelector.vue b/src/components/LanguageSelector.vue
@@ -37,9 +37,20 @@
                 if (event.key === "ArrowDown") {
                     this.selected = Math.min(this.selected + 1, this.filteredItems.length - 1)
                     event.preventDefault()
+                    if (this.selected === this.filteredItems.length - 1) {
+                        this.$refs.container.scrollIntoView({block: "end"})
+                    } else {
+                        this.$refs.item[this.selected].scrollIntoView({block: "nearest"})
+                    }
+                    
                 } else if (event.key === "ArrowUp") {
                     this.selected = Math.max(this.selected - 1, 0)
                     event.preventDefault()
+                    if (this.selected === 0) {
+                        this.$refs.container.scrollIntoView({block: "start"})
+                    } else {
+                        this.$refs.item[this.selected].scrollIntoView({block: "nearest"})
+                    }
                 } else if (event.key === "Enter") {
                     this.selectItem(this.filteredItems[this.selected].token)
                     event.preventDefault()
@@ -69,28 +80,38 @@
 </script>
 
 <template>
-    <form class="language-selector" tabindex="-1" @focusout="onFocusOut" ref="container">
-        <input 
-            type="text" 
-            ref="input"
-            @keydown="onKeydown"
-            @input="onInput"
-            v-model="filter"
-        />
-        <ul class="items">
-            <li
-                v-for="item, idx in filteredItems"
-                :key="item.token"
-                :class="idx === selected ? 'selected' : ''"
-                @click="selectItem(item.token)"
-            >
-                {{ item.name }}
-            </li>
-        </ul>
-    </form>
+    <div class="scroller">
+        <form class="language-selector" tabindex="-1" @focusout="onFocusOut" ref="container">
+            <input 
+                type="text" 
+                ref="input"
+                @keydown="onKeydown"
+                @input="onInput"
+                v-model="filter"
+            />
+            <ul class="items">
+                <li
+                    v-for="item, idx in filteredItems"
+                    :key="item.token"
+                    :class="idx === selected ? 'selected' : ''"
+                    @click="selectItem(item.token)"
+                    ref="item"
+                >
+                    {{ item.name }}
+                </li>
+            </ul>
+        </form>
+    </div>
 </template>
 
 <style scoped lang="sass">    
+    .scroller
+        overflow: auto
+        position: fixed
+        top: 0
+        left: 0
+        bottom: 0
+        right: 0
     .language-selector
         font-size: 13px
         padding: 10px

diff --git a/src/editor/block/format-code.js b/src/editor/block/format-code.js
@@ -1,32 +1,19 @@
 import { EditorSelection } from "@codemirror/state"
 
 import * as prettier from "prettier/standalone"
-import babelParser from "prettier/plugins/babel.mjs"
-import htmlParser from "prettier/esm/parser-html.mjs"
-import cssParser from "prettier/esm/parser-postcss.mjs"
-import markdownParser from "prettier/esm/parser-markdown.mjs"
-import * as prettierPluginEstree from "prettier/plugins/estree.mjs";
-
 import { getActiveNoteBlock } from "./block.js"
-
-
-const PARSER_MAP = {
-    "json": {parser:"json-stringify", plugins: [babelParser, prettierPluginEstree]},
-    "javascript": {parser:"babel", plugins: [babelParser, prettierPluginEstree]},
-    "html": {parser:"html", plugins: [htmlParser]},
-    "css": {parser:"css", plugins: [cssParser]},
-    "markdown": {parser:"markdown", plugins: [markdownParser]},
-}
+import { getLanguage } from "../languages.js"
 
 
 export const formatBlockContent = async ({ state, dispatch }) => {
     if (state.readOnly)
         return false
     const block = getActiveNoteBlock(state)
 
-    const langName = block.language.name
-    if (!(langName in PARSER_MAP))
+    const language = getLanguage(block.language.name)
+    if (!language.prettier) {
         return false
+    }
 
     // get current cursor position
     const cursorPos = state.selection.asSingle().ranges[0].head
@@ -49,17 +36,17 @@ export const formatBlockContent = async ({ state, dispatch }) => {
         if (useFormat) {
             formattedContent = {
                 formatted: await prettier.format(content, {
-                    parser: PARSER_MAP[langName].parser,
-                    plugins: PARSER_MAP[langName].plugins,
+                    parser: language.prettier.parser,
+                    plugins: language.prettier.plugins,
                     tabWidth: state.tabSize,
                 }),
-                cursorOffset: cursorPos == block.content.from ? 0 : content.length,
             }
+            formattedContent.cursorOffset = cursorPos == block.content.from ? 0 : formattedContent.formatted.length
         } else {
             formattedContent = await prettier.formatWithCursor(content, {
                 cursorOffset: cursorPos - block.content.from,
-                parser: PARSER_MAP[langName].parser,
-                plugins: PARSER_MAP[langName].plugins,
+                parser: language.prettier.parser,
+                plugins: language.prettier.plugins,
                 tabWidth: state.tabSize,
             })
         }
@@ -75,7 +62,7 @@ export const formatBlockContent = async ({ state, dispatch }) => {
             to: block.content.to,
             insert: formattedContent.formatted,
         },
-        selection: EditorSelection.cursor(block.content.from + formattedContent.cursorOffset),
+        selection: EditorSelection.cursor(block.content.from + Math.min(formattedContent.cursorOffset, formattedContent.formatted.length)),
     }, {
         userEvent: "input",
         scrollIntoView: true,

diff --git a/src/editor/lang-heynote/heynote.grammar b/src/editor/lang-heynote/heynote.grammar
@@ -11,7 +11,7 @@ NoteDelimiter {
 
 @tokens {
     noteDelimiterMark { "∞∞∞" }
-    NoteLanguage { "text" | "math" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "php" | "css" | "xml" | "cpp" | "rust" | "csharp" }
+    NoteLanguage { "text" | "math" | "javascript" | "json" | "python" | "html" | "sql" | "markdown" | "java" | "php" | "css" | "xml" | "cpp" | "rust" | "csharp" | "ruby" | "shell" | "yaml" | "golang" | "clojure" | "erlang" | "lezer" }
     Auto { "-a" }
     noteDelimiterEnter { "\n" }
     //NoteContent { String }

diff --git a/src/editor/lang-heynote/parser.js b/src/editor/lang-heynote/parser.js
@@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
   maxTerm: 10,
   skippedNodes: [0],
   repeatNodeCount: 1,
-  tokenData: "'f~R[YZw}!O|#V#W!X#[#]#S#^#_#f#a#b%P#d#e&O#f#g&e#g#h&q#h#i&w#l#m#Y%&x%&y'T~|OX~~!PP#T#U!S~!XOU~~![Q#d#e!b#g#h!m~!eP#d#e!h~!mOT~~!pQ#[#]!v#g#h!h~!yP#T#U!|~#PP#f#g!b~#VP#h#i#Y~#]P#a#b#`~#cP#`#a!h~#iQ#T#U#o#g#h$s~#rP#j#k#u~#xP#T#U#{~$QPT~#g#h$T~$WP#V#W$Z~$^P#f#g$a~$dP#]#^$g~$jP#d#e$m~$pP#h#i!h~$vP#c#d$y~$|P#b#c!h~%SP#T#U%V~%YQ#f#g%`#h#i%x~%cP#_#`%f~%iP#W#X%l~%oP#c#d%r~%uP#k#l$y~%{P#[#]!h~&RQ#[#]!b#m#n&X~&[P#h#i&_~&bP#[#]$s~&hP#i#j&k~&nP#g#h$m~&tP#e#f#`~&zP#X#Y&}~'QP#l#m$m~'WP%&x%&y'Z~'^P%&x%&y'a~'fOY~",
+  tokenData: "*c~R`YZ!T}!O!Y#V#W!e#X#Y$R#Z#[$q#[#]$w#^#_%Z#`#a&t#a#b'^#d#e(]#f#g(r#g#h)X#h#i)n#l#m$}#m#n)z%&x%&y*Q~!YOX~~!]P#T#U!`~!eOU~~!hR#`#a!q#d#e#f#g#h#l~!tP#c#d!w~!zP#^#_!}~#QP#i#j#T~#WP#f#g#Z~#^P#X#Y#a~#fOT~~#iP#d#e#a~#oQ#[#]#u#g#h#a~#xP#T#U#{~$OP#f#g#f~$UP#f#g$X~$[P#`#a$_~$bP#T#U$e~$hP#b#c$k~$nP#Z#[#a~$tP#c#d$X~$zP#h#i$}~%QP#a#b%T~%WP#`#a#a~%^Q#T#U%d#g#h&h~%gP#j#k%j~%mP#T#U%p~%uPT~#g#h%x~%{P#V#W&O~&RP#f#g&U~&XP#]#^&[~&_P#d#e&b~&eP#h#i#a~&kP#c#d&n~&qP#b#c#a~&wP#X#Y&z~&}P#n#o'Q~'TP#X#Y'W~'ZP#f#g#a~'aP#T#U'd~'gQ#f#g'm#h#i(V~'pP#_#`'s~'vP#W#X'y~'|P#c#d(P~(SP#k#l&n~(YP#[#]#a~(`Q#[#]#f#m#n(f~(iP#h#i(l~(oP#[#]&h~(uP#i#j(x~({Q#U#V)R#g#h&b~)UP#m#n#a~)[Q#[#])b#e#f%T~)eP#X#Y)h~)kP#`#a%T~)qP#X#Y)t~)wP#l#m&b~)}P#T#U$}~*TP%&x%&y*W~*ZP%&x%&y*^~*cOY~",
   tokenizers: [0, noteContent],
   topRules: {"Document":[0,2]},
   tokenPrec: 0

diff --git a/src/editor/language-detection/autodetect.js b/src/editor/language-detection/autodetect.js
@@ -9,6 +9,21 @@ import { LANGUAGE_CHANGE } from "../annotation";
 
 const GUESSLANG_TO_TOKEN = Object.fromEntries(LANGUAGES.map(l => [l.guesslang,l.token]))
 
+function requestIdleCallbackCompat(cb) {
+    if (window.requestIdleCallback) {
+        return window.requestIdleCallback(cb)
+    } else {
+        return setTimeout(cb, 0)
+    }
+}
+
+function cancelIdleCallbackCompat(id) {
+    if (window.cancelIdleCallback) {
+        window.cancelIdleCallback(id)
+    } else {
+        clearTimeout(id)
+    }
+}
 
 export function languageDetection(getView) {
     const previousBlockContent = {}
@@ -45,11 +60,11 @@ export function languageDetection(getView) {
     const plugin = EditorView.updateListener.of(update => {
         if (update.docChanged) {
             if (idleCallbackId !== null) {
-                cancelIdleCallback(idleCallbackId)
+                cancelIdleCallbackCompat(idleCallbackId)
                 idleCallbackId = null
             }
 
-            idleCallbackId = requestIdleCallback(() => {
+            idleCallbackId = requestIdleCallbackCompat(() => {
                 idleCallbackId = null
 
                 const range = update.state.selection.asSingle().ranges[0]
-Original file line number
+Diff line change
@@ Expand Up / @@ -14,6 +14,12 @@ GUESSLANG_LANGUAGES = [ @@
         "rs",
         "md",
         "cs",
+        "rb",
+        "sh",
+        "yaml",
+        "go",
+        "clj",
+        "erl",
     ]
     const guessLang = new self.GuessLang()
@@ Expand Down @@