Add Typst parser

build.rs

@@ -358,6 +358,11 @@ fn main() {
             src_dir: "vendored_parsers/tree-sitter-typescript-src/typescript/src",
             extra_files: vec!["scanner.c"],
         },
+        TreeSitterParser {
+            name: "tree-sitter-typst",
+            src_dir: "vendored_parsers/tree-sitter-typst-src",
+            extra_files: vec!["scanner.c"],
+        },
         TreeSitterParser {
             name: "tree-sitter-vhdl",
             src_dir: "vendored_parsers/tree-sitter-vhdl-src",

sample_files/compare.expected

@@ -277,6 +277,9 @@ fee7ee33d2037ad1941ba6bb5532a1db  -
 sample_files/typing_1.ml sample_files/typing_2.ml
 36161bd77a8c86643bc90656ec41c92c  -
 
+sample_files/typst_1.typ sample_files/typst_2.typ
+1eb9abd1d35daaaaf12f4cc964b831ff  -
+
 sample_files/utf16_1.py sample_files/utf16_2.py
 39014a682ed2318f980c7ea4177cf659  -
 

sample_files/typst_1.typ

@@ -0,0 +1,25 @@
+#set page(width: 10cm, height: auto)
+#set heading(numbering: "1.")
+
+= Fibonacci sequence
+The Fibonacci sequence is defined through the
+recurrence relation $F_n = F_(n-1) + F_(n-2)$.
+It can also be expressed in _closed form:_
+
+$ F_n = round(1 / sqrt(5) phi.alt^n), quad
+  phi.alt = (1 + sqrt(5)) / 2 $
+
+#let count = 8
+#let nums = range(1, count + 1)
+#let fib(n) = (
+  if n <= 2 { 1 }
+  else { fib(n - 1) + fib(n - 2) }
+)
+
+The first #count numbers of the sequence are:
+
+#align(center, table(
+  columns: count,
+  ..nums.map(n => $F_#n$),
+  ..nums.map(n => str(fib(n))),
+))

sample_files/typst_2.typ

@@ -0,0 +1,25 @@
+#set page(width: 10cm, height: auto)
+#set heading(numbering: "1.")
+
+= Fibonacci sequence
+The Fibonacci sequence is defined through the
+recurrence relation $F_n = F_(n-1) + F_(n-2)$.
+It can also be expressed in _closed form:_
+
+$ F_n = round(1 / sqrt(5) phi.alt^n), quad
+  phi.alt = (1 + sqrt(5)) / 2 $
+
+#let count = 8
+#let fib(n) = (
+  if n <= 2 { 1 }
+  else { fib(n - 1) + fib(n - 2) }
+)
+
+The first #count numbers of the sequence are:
+
+#let function_table(f, nums) = align(center, table(
+  columns: nums.len(),
+  ..nums.map(n => $F_#n$),
+  ..nums.map(n => str(f(n))),
+))
+#function_table(fib, range(1, count+1))

src/parse/guess_language.rs

@@ -77,6 +77,7 @@ pub(crate) enum Language {
     Toml,
     TypeScript,
     TypeScriptTsx,
+    Typst,
     Vhdl,
     Xml,
     Yaml,
@@ -170,6 +171,7 @@ pub(crate) fn language_name(language: Language) -> &'static str {
         Toml => "TOML",
         TypeScript => "TypeScript",
         TypeScriptTsx => "TypeScript TSX",
+        Typst => "Typst",
         Vhdl => "VHDL",
         Xml => "XML",
         Yaml => "YAML",
@@ -368,6 +370,7 @@ pub(crate) fn language_globs(language: Language) -> Vec<glob::Pattern> {
         ],
         TypeScript => &["*.ts"],
         TypeScriptTsx => &["*.tsx"],
+        Typst => &["*.typ"],
         Vhdl => &["*.vhdl", "*.vhd"],
         Xml => &[
             "*.ant",
@@ -540,6 +543,7 @@ fn from_emacs_mode_header(src: &str) -> Option<Language> {
             "toml" => Some(Toml),
             "tuareg" => Some(OCaml),
             "typescript" => Some(TypeScript),
+            "typst" => Some(Typst),
             "vhdl" => Some(Vhdl),
             "yaml" => Some(Yaml),
             "zig" => Some(Zig),

src/parse/tree_sitter_parser.rs

@@ -118,6 +118,7 @@ extern "C" {
     fn tree_sitter_toml() -> ts::Language;
     fn tree_sitter_tsx() -> ts::Language;
     fn tree_sitter_typescript() -> ts::Language;
+    fn tree_sitter_typst() -> ts::Language;
     fn tree_sitter_vhdl() -> ts::Language;
     fn tree_sitter_xml() -> ts::Language;
     fn tree_sitter_yaml() -> ts::Language;
@@ -1133,6 +1134,20 @@ pub(crate) fn from_language(language: guess::Language) -> TreeSitterConfig {
                 sub_languages: vec![],
             }
         }
+        Typst => {
+            let language = unsafe { tree_sitter_typst() };
+            TreeSitterConfig {
+                language,
+                atom_nodes: vec!["string"].into_iter().collect(),
+                delimiter_tokens: vec![("{", "}"), ("(", ")"), ("[", "]")],
+                highlight_query: ts::Query::new(
+                    language,
+                    include_str!("../../vendored_parsers/highlights/typst.scm"),
+                )
+                .unwrap(),
+                sub_languages: vec![],
+            }
+        }
         Xml => {
             let language = unsafe { tree_sitter_xml() };
             TreeSitterConfig {

vendored_parsers/highlights/typst.scm

@@ -0,0 +1 @@
+../tree-sitter-typst/queries/typst/highlights.scm

vendored_parsers/tree-sitter-typst-src

@@ -0,0 +1 @@
+tree-sitter-typst/src/

vendored_parsers/tree-sitter-typst/.gitignore

@@ -0,0 +1,8 @@
+log.html
+node_modules
+package-lock.json
+target
+# src/grammar.json
+# src/node-types.json
+# src/tree_sitter/parser.h
+# src/parser.c

vendored_parsers/tree-sitter-typst/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tree-sitter-typst"
+description = "typst grammar for the tree-sitter parsing library"
+version = "0.0.1"
+keywords = ["incremental", "parsing", "typst"]
+categories = ["parsing", "text-editors"]
+repository = "https://github.com/uben0/tree-sitter-typst"
+edition = "2018"
+license = "MIT"
+
+build = "bindings/rust/build.rs"
+include = [
+  "bindings/rust/*",
+  "grammar.js",
+  "queries/*",
+  "src/*",
+]
+
+[lib]
+path = "bindings/rust/lib.rs"
+
+[dependencies]
+tree-sitter = "~0.20.10"
+
+[build-dependencies]
+cc = "1.0"

vendored_parsers/tree-sitter-typst/DOC.md

@@ -0,0 +1,93 @@
+# Implementation documentation
+
+## Tricky Typst
+
+List of tests exposing particular behaviors from Typst
+
+- Test `positive/320`: indent takes in account comments, but redent is done at item, term or heading token.
+- Tests `positive/328` and `positive/329`: math function don't work with symbol.
+- Test `positive/330`: Comment block don't need closing end.
+
+## FIXME
+
+- [ ] Test `fixme/011`: Embedded code causes a segfault
+  - With new version of Tree Sitter or Helix, this bug seems to be fixed
+- [X] ~Test `fixme/014`: Right square bracket exits from all containers~
+- [X] ~Test `fixme/012`: Matching square brackets in text are paired~
+- [X] ~Test `fixme/013`: Point ending a ref is not part of the ref~
+- [X] ~Test `fixme/010`: Math shorthand and letter can applied~
+- [X] ~Test `fixme/009`: Indentation and comments~
+- [X] ~Test `fixme/001`: Group termination in math~
+- [X] ~Test `fixme/002`: Import precedence over list~
+- [X] ~Test `fixme/008`: Condition if set statement~
+- [X] ~Test `fixme/007`: Trailing comments before `else`~
+- [X] ~Test `fixme/003`: Spaces in method notation~
+- [X] ~Test `fixme/004`: Leading space not recognized~
+- [X] ~Test `fixme/005`: Inlined code absorbs new line~
+
+Failing tests are found in [`corpus/fixme.scm`](https://github.com/uben0/tree-sitter-typst/blob/master/corpus/fixme.scm).
+
+## Optimization with extras
+
+When searching ways to optimize the parser and simplify the grammar, I thought about using the *extras* feature for spaces and comments (and line breaks as well). At the end, it significantly reduced parser size. The only problem arises with function calls and, in inline code, field access. They must be directly joined (no space nor comment in between). The use of the *immediate* feature won't solve the problem as it only takes in acount inline regex (which would be ok with spaces but not comments, as they have to appear in output tree).
+
+The solution is to rely on external scanner when parsing spaces or comments. Lets call a "pre-immediate" token, a token susceptible to be followed by immediate token. When a pre-immediate token is parsed, it sets a flag to `true`, and when a space or comment is parsed, it resets the flag to `false` (this flag is stored in scanner's state as a boolean).
+
+This way when a token has to be immediate, an external token can be required and will only match if flag is `true`. It means, any pre-immediate token have to be preceded by a token that will set to `true` the flag.
+
+# Scanner
+
+## Containers
+
+The scanner uses a stack of "containers" as internal states in order to simplify the parsing of some nodes like `emph`, `strong` and `content`. At any moment, the scanner knows what are the containing nodes. For instance:
+
+```typst
+* _ #[Hello *World*] _ *
+```
+
+Here, when the scanner will be called at the beginning of "World", the container stack will be `stron/emph/content/strong`. When a '*' is encountered while the top of the stack is `strong`, it will be interpreted as the end of this container. If the top of the stack was `emph`, it would be the start of a `strong` container, and it would be pushed on the stack.
+
+It is mandatory to use the external scanner for the containers because of indentation.
+
+## Indentation
+
+Three distinct token are dedicated to indentation:
+
+- `indent`: When the following line start further than the current one
+- `dedent`: When the following line start at a column previously set as indentation column
+- `redent`: When the following line start at the same column as current one, or before, but further than the previous indentation.
+
+The concept of redent can be seen here:
+```typst
+- 1
+  - 2
+ - 3
+  - 4
+```
+Which have the same hierarchy as:
+```typst
+- 1
+  - 2
+  - 3
+    - 4
+```
+
+## Character class
+
+Five character classes are defined in the external scanner:
+
+- space (`is_sp`)
+- line break (`is_lb`)
+- xid start (`is_id_start`)
+- xid continue (`is_id_continue`)
+- word part (`is_word_part`)
+
+The three functions `is_id_start`, `is_id_continue` and `is_word_part` are implemented as binary search.
+
+The character list is based on the Unicode database which can be found here: https://www.unicode.org/Public/UCD/latest/ucd/
+
+A utility is used to produce those tables: https://github.com/uben0/unicode-table
+
+## Barrier
+
+The `heading`, `item` and `term` nodes are technically delimited contexts, but because thay behave similarly, they have the same external token as container, which is `barrier`.

vendored_parsers/tree-sitter-typst/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023 Gerbais-Nief Eddie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendored_parsers/tree-sitter-typst/Makefile

@@ -0,0 +1,19 @@
+test: src/parser.c
+	tree-sitter test -f Test
+
+all: src/parser.c
+	tree-sitter test
+
+fixme: src/parser.c
+	tree-sitter test -f Fixme
+
+src/parser.c: grammar.js
+	tree-sitter generate
+
+build: src/parser.c size
+
+size: src/parser.c
+	du -b src/parser.c
+	du -b src/parser.c > size.txt
+
+.PHONY: fixme test build all size

vendored_parsers/tree-sitter-typst/Package.swift

@@ -0,0 +1,48 @@
+// swift-tools-version:5.3
+import PackageDescription
+
+let package = Package(
+    name: "TreeSitterTypst",
+    platforms: [.macOS(.v10_13), .iOS(.v11)],
+    products: [
+        .library(name: "TreeSitterTypst", targets: ["TreeSitterTypst"]),
+    ],
+    dependencies: [],
+    targets: [
+        .target(name: "TreeSitterTypst",
+                path: ".",
+                exclude: [
+                    "Cargo.toml",
+                    "Makefile",
+                    "binding.gyp",
+                    "bindings/c",
+                    "bindings/go",
+                    "bindings/node",
+                    "bindings/python",
+                    "bindings/rust",
+                    "prebuilds",
+                    "grammar.js",
+                    "package.json",
+                    "package-lock.json",
+                    "pyproject.toml",
+                    "setup.py",
+                    "test",
+                    "examples",
+                    ".editorconfig",
+                    ".github",
+                    ".gitignore",
+                    ".gitattributes",
+                    ".gitmodules",
+                ],
+                sources: [
+                    "src/parser.c",
+                    // NOTE: if your language has an external scanner, add it here.
+                ],
+                resources: [
+                    .copy("queries")
+                ],
+                publicHeadersPath: "bindings/swift",
+                cSettings: [.headerSearchPath("src")])
+    ],
+    cLanguageStandard: .c11
+)