Implement extends

src/parsing/metadata.rs

@@ -493,13 +493,13 @@ mod tests {
 
     #[test]
     fn load_raw() {
-        let comments_file: &str = "testdata/Packages/Go/Comments.tmPreferences";
+        let comments_file: &str = "testdata/Packages/Go/GoCommentRules.tmPreferences";
         assert!(Path::new(comments_file).exists());
 
         let r = RawMetadataEntry::load(comments_file);
         assert!(r.is_ok());
 
-        let indent_file: &str = "testdata/Packages/Go/Indentation Rules.tmPreferences";
+        let indent_file: &str = "testdata/Packages/Go/Indents/GoIndent.tmPreferences";
         assert!(Path::new(indent_file).exists());
 
         let r = RawMetadataEntry::load(indent_file).unwrap();

src/parsing/parser.rs

@@ -757,24 +757,35 @@ mod tests {
         let ops1 = ops(&mut state, "module Bob::Wow::Troll::Five; 5; end", &ss);
         let test_ops1 = vec![
             (0, Push(Scope::new("source.ruby.rails").unwrap())),
-            (0, Push(Scope::new("meta.module.ruby").unwrap())),
-            (0, Push(Scope::new("keyword.control.module.ruby").unwrap())),
+            (0, Push(Scope::new("meta.namespace.ruby").unwrap())),
+            (0, Push(Scope::new("storage.type.namespace.ruby").unwrap())),
+            (
+                0,
+                Push(Scope::new("keyword.declaration.namespace.ruby").unwrap()),
+            ),
             (6, Pop(2)),
-            (6, Push(Scope::new("meta.module.ruby").unwrap())),
             (7, Pop(1)),
-            (7, Push(Scope::new("meta.module.ruby").unwrap())),
-            (7, Push(Scope::new("entity.name.module.ruby").unwrap())),
+            (7, Push(Scope::new("meta.namespace.ruby").unwrap())),
+            (7, Push(Scope::new("entity.name.namespace.ruby").unwrap())),
             (7, Push(Scope::new("support.other.namespace.ruby").unwrap())),
             (10, Pop(1)),
-            (10, Push(Scope::new("punctuation.accessor.ruby").unwrap())),
+            (
+                10,
+                Push(Scope::new("punctuation.accessor.double-colon.ruby").unwrap()),
+            ),
+            (12, Pop(1)),
         ];
         assert_eq!(&ops1[0..test_ops1.len()], &test_ops1[..]);
 
         let ops2 = ops(&mut state, "def lol(wow = 5)", &ss);
-        let test_ops2 = vec![
+        let test_ops2 = [
             (0, Push(Scope::new("meta.function.ruby").unwrap())),
-            (0, Push(Scope::new("keyword.control.def.ruby").unwrap())),
-            (3, Pop(2)),
+            (0, Push(Scope::new("storage.type.function.ruby").unwrap())),
+            (
+                0,
+                Push(Scope::new("keyword.declaration.function.ruby").unwrap()),
+            ),
+            (3, Pop(3)),
             (3, Push(Scope::new("meta.function.ruby").unwrap())),
             (4, Push(Scope::new("entity.name.function.ruby").unwrap())),
             (7, Pop(1)),
@@ -829,6 +840,7 @@ mod tests {
         test_stack.push(Scope::new("text.html.basic").unwrap());
         test_stack.push(Scope::new("source.js.embedded.html").unwrap());
         test_stack.push(Scope::new("source.js").unwrap());
+        test_stack.push(Scope::new("meta.string.js").unwrap());
         test_stack.push(Scope::new("string.quoted.single.js").unwrap());
         test_stack.push(Scope::new("source.ruby.rails.embedded.html").unwrap());
         test_stack.push(Scope::new("meta.function.parameters.ruby").unwrap());
@@ -860,40 +872,39 @@ mod tests {
                     Push(Scope::new("keyword.operator.assignment.ruby").unwrap())
                 ),
                 (5, Pop(1)),
-                (
-                    6,
-                    Push(Scope::new("string.unquoted.embedded.sql.ruby").unwrap())
-                ),
+                (6, Push(Scope::new("meta.string.heredoc.ruby").unwrap())),
+                (6, Push(Scope::new("string.unquoted.heredoc.ruby").unwrap())),
                 (
                     6,
                     Push(Scope::new("punctuation.definition.string.begin.ruby").unwrap())
                 ),
+                (12, Pop(2)),
                 (12, Pop(1)),
-                (12, Pop(1)),
+                (12, Push(Scope::new("meta.string.heredoc.ruby").unwrap())),
+                (12, Push(Scope::new("source.sql.embedded.ruby").unwrap())),
+                (12, Clear(ClearAmount::TopN(2))),
                 (
                     12,
-                    Push(Scope::new("string.unquoted.embedded.sql.ruby").unwrap())
+                    Push(Scope::new("punctuation.accessor.dot.ruby").unwrap())
                 ),
-                (12, Push(Scope::new("text.sql.embedded.ruby").unwrap())),
-                (12, Clear(ClearAmount::TopN(2))),
-                (12, Push(Scope::new("punctuation.accessor.ruby").unwrap())),
                 (13, Pop(1)),
-                (18, Restore),
             ]
         );
 
-        assert_eq!(ops(&mut state, "wow", &ss), vec![]);
+        assert_eq!(ops(&mut state, "wow", &ss), vec![(0, Restore)]);
 
         assert_eq!(
             ops(&mut state, "SQL", &ss),
             vec![
                 (0, Pop(1)),
+                (0, Push(Scope::new("string.unquoted.heredoc.ruby").unwrap())),
                 (
                     0,
                     Push(Scope::new("punctuation.definition.string.end.ruby").unwrap())
                 ),
                 (3, Pop(1)),
                 (3, Pop(1)),
+                (3, Pop(1)),
             ]
         );
     }
@@ -998,8 +1009,8 @@ mod tests {
         let mut state1 = ParseState::new(syntax);
         let mut state2 = ParseState::new(syntax);
 
-        assert_eq!(ops(&mut state1, "class Foo {", &ss).len(), 11);
-        assert_eq!(ops(&mut state2, "class Fooo {", &ss).len(), 11);
+        assert_eq!(ops(&mut state1, "class Foo {", &ss).len(), 12);
+        assert_eq!(ops(&mut state2, "class Fooo {", &ss).len(), 12);
 
         assert_eq!(state1, state2);
         ops(&mut state1, "}", &ss);

src/parsing/syntax_definition.rs

@@ -51,7 +51,7 @@ pub struct Context {
     pub meta_scope: Vec<Scope>,
     pub meta_content_scope: Vec<Scope>,
     /// This being set false in the syntax file implies this field being set false,
-    /// but it can also be set falso for contexts that don't include the prototype for other reasons
+    /// but it can also be set false for contexts that don't include the prototype for other reasons
     pub meta_include_prototype: bool,
     pub clear_scopes: Option<ClearAmount>,
     /// This is filled in by the linker at link time
@@ -75,6 +75,27 @@ impl Context {
             prototype: None,
         }
     }
+
+    pub(crate) fn extend(&mut self, other: Context) {
+        let Context {
+            meta_scope,
+            meta_content_scope,
+            meta_include_prototype,
+            clear_scopes,
+            prototype,
+            uses_backrefs,
+            patterns,
+        } = other;
+        self.meta_scope.extend(meta_scope);
+        self.meta_content_scope.extend(meta_content_scope);
+        self.meta_include_prototype = meta_include_prototype;
+        self.clear_scopes = clear_scopes;
+        if self.prototype.is_none() || prototype.is_some() {
+            self.prototype = prototype;
+        }
+        self.uses_backrefs |= uses_backrefs;
+        self.patterns.extend(patterns);
+    }
 }
 
 #[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)]

src/parsing/syntax_set.rs

@@ -13,6 +13,7 @@ use std::fs::File;
 use std::io::{self, BufRead, BufReader};
 use std::mem;
 use std::path::Path;
+use std::path::PathBuf;
 
 use super::regex::Regex;
 use crate::parsing::syntax_definition::ContextId;
@@ -83,6 +84,7 @@ pub(crate) struct LazyContexts {
 pub struct SyntaxSetBuilder {
     syntaxes: Vec<SyntaxDefinition>,
     path_syntaxes: Vec<(String, usize)>,
+    extends_syntaxes: Vec<(PathBuf, String)>,
     #[cfg(feature = "metadata")]
     raw_metadata: LoadMetadata,
 
@@ -108,6 +110,23 @@ fn load_syntax_file(
     .map_err(|e| LoadingError::ParseSyntax(e, format!("{}", p.display())))
 }
 
+#[cfg(feature = "yaml-load")]
+fn load_syntax_file_with_extends(
+    p: &Path,
+    base_syntax: &SyntaxDefinition,
+    lines_include_newline: bool,
+) -> Result<SyntaxDefinition, LoadingError> {
+    let s = std::fs::read_to_string(p)?;
+
+    SyntaxDefinition::load_from_str_extended(
+        &s,
+        Some(base_syntax),
+        lines_include_newline,
+        p.file_stem().and_then(|x| x.to_str()),
+    )
+    .map_err(|e| LoadingError::ParseSyntax(e, format!("{}", p.display())))
+}
+
 impl Clone for SyntaxSet {
     fn clone(&self) -> SyntaxSet {
         SyntaxSet {
@@ -375,6 +394,7 @@ impl SyntaxSet {
         SyntaxSetBuilder {
             syntaxes: builder_syntaxes,
             path_syntaxes,
+            extends_syntaxes: Vec::new(),
             #[cfg(feature = "metadata")]
             existing_metadata: Some(metadata),
             #[cfg(feature = "metadata")]
@@ -516,6 +536,8 @@ impl SyntaxSetBuilder {
         folder: P,
         lines_include_newline: bool,
     ) -> Result<(), LoadingError> {
+        use super::ParseSyntaxError;
+
         for entry in crate::utils::walk_dir(folder).sort_by(|a, b| a.file_name().cmp(b.file_name()))
         {
             let entry = entry.map_err(LoadingError::WalkDir)?;
@@ -524,7 +546,27 @@ impl SyntaxSetBuilder {
                 .extension()
                 .map_or(false, |e| e == "sublime-syntax")
             {
-                let syntax = load_syntax_file(entry.path(), lines_include_newline)?;
+                let syntax = match load_syntax_file(entry.path(), lines_include_newline) {
+                    Ok(syntax) => syntax,
+                    // We are extending another syntax, look it up in the set first
+                    Err(LoadingError::ParseSyntax(
+                        ParseSyntaxError::ExtendsNotFound { name, extends },
+                        _,
+                    )) => {
+                        if let Some(ix) = self
+                            .path_syntaxes
+                            .iter()
+                            .find(|(s, _)| s.ends_with(extends.as_str()))
+                            .map(|(_, ix)| *ix)
+                        {
+                            todo!("lookup {ix} and pass to {name}");
+                        }
+                        self.extends_syntaxes
+                            .push((entry.path().to_path_buf(), extends));
+                        continue;
+                    }
+                    Err(err) => return Err(err),
+                };
                 if let Some(path_str) = entry.path().to_str() {
                     // Split the path up and rejoin with slashes so that syntaxes loaded on Windows
                     // can still be loaded the same way.
@@ -550,6 +592,45 @@ impl SyntaxSetBuilder {
         Ok(())
     }
 
+    fn resolve_extends(&mut self) {
+        let mut prev_len = usize::MAX;
+        // Loop while syntaxes are being resolved
+        while !self.extends_syntaxes.is_empty() && prev_len > self.extends_syntaxes.len() {
+            prev_len = self.extends_syntaxes.len();
+            // Split borrows to make the borrow cheker happy
+            let syntaxes = &mut self.syntaxes;
+            let paths = &mut self.path_syntaxes;
+            // Resolve syntaxes
+            self.extends_syntaxes.retain(|(path, extends)| {
+                let Some(ix) = paths
+                    .iter()
+                    .find(|(s, _)| s.ends_with(extends.as_str()))
+                    .map(|(_, ix)| *ix)
+                else {
+                    return true;
+                };
+                let base_syntax = &syntaxes[ix];
+                // FIXME: don't unwrap
+                let syntax = load_syntax_file_with_extends(path, base_syntax, false).unwrap();
+                if let Some(path_str) = path.to_str() {
+                    // Split the path up and rejoin with slashes so that syntaxes loaded on Windows
+                    // can still be loaded the same way.
+                    let path = Path::new(path_str);
+                    let path_parts: Vec<_> = path.iter().map(|c| c.to_str().unwrap()).collect();
+                    paths.push((path_parts.join("/").to_string(), syntaxes.len()));
+                }
+                syntaxes.push(syntax);
+                false
+            });
+        }
+
+        if !self.extends_syntaxes.is_empty() {
+            dbg!(&self.path_syntaxes);
+            dbg!(&self.extends_syntaxes);
+            todo!("warn, unresolved syntaxes");
+        }
+    }
+
     /// Build a [`SyntaxSet`] from the syntaxes that have been added to this
     /// builder.
     ///
@@ -571,16 +652,20 @@ impl SyntaxSetBuilder {
     /// directly load the [`SyntaxSet`].
     ///
     /// [`SyntaxSet`]: struct.SyntaxSet.html
-    pub fn build(self) -> SyntaxSet {
+    pub fn build(mut self) -> SyntaxSet {
+        self.resolve_extends();
+
         #[cfg(not(feature = "metadata"))]
         let SyntaxSetBuilder {
             syntaxes: syntax_definitions,
             path_syntaxes,
+            extends_syntaxes: _,
         } = self;
         #[cfg(feature = "metadata")]
         let SyntaxSetBuilder {
             syntaxes: syntax_definitions,
             path_syntaxes,
+            extends_syntaxes: _,
             raw_metadata,
             existing_metadata,
         } = self;
@@ -1032,7 +1117,7 @@ mod tests {
             .get_context(&syntax.context_ids()["main"])
             .expect("#[cfg(test)]");
         let count = syntax_definition::context_iter(&ps, main_context).count();
-        assert_eq!(count, 109);
+        assert_eq!(count, 168);
     }
 
     #[test]