From 2c81736a644670e26f80b83d14b1730d3d141d3a Mon Sep 17 00:00:00 2001 From: Robin Stocker Date: Fri, 29 Jun 2018 17:44:41 +1000 Subject: [PATCH 01/58] Cache compiled regexes with lazycell With that, MatchPattern does not have to be mutably borrowed to do matching anymore. It removes one obstacle for allowing SyntaxSet to be used by multiple threads, while still keeping regex compilation lazy. --- Cargo.toml | 1 + src/lib.rs | 2 + src/parsing/parser.rs | 46 +++++++------- src/parsing/syntax_definition.rs | 100 ++++++++++++++++++++++--------- src/parsing/yaml_load.rs | 17 +++--- 5 files changed, 109 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6db8250c..a5e15471 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ onig = { version = "3.2.1", optional = true } walkdir = "2.0" regex-syntax = { version = "0.4", optional = true } lazy_static = "1.0" +lazycell = "1.0" bitflags = "1.0" plist = "0.3" bincode = { version = "1.0", optional = true } diff --git a/src/lib.rs b/src/lib.rs index c2e9860d..06b26d68 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,6 +26,7 @@ extern crate walkdir; extern crate regex_syntax; #[macro_use] extern crate lazy_static; +extern crate lazycell; extern crate plist; #[cfg(any(feature = "dump-load-rs", feature = "dump-load", feature = "dump-create"))] extern crate bincode; @@ -42,6 +43,7 @@ extern crate serde_json; #[cfg(test)] #[macro_use] extern crate pretty_assertions; + pub mod highlighting; pub mod parsing; pub mod util; diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs index 53eb8114..5a8c6d95 100644 --- a/src/parsing/parser.rs +++ b/src/parsing/parser.rs @@ -326,8 +326,8 @@ impl ParseState { for (from_with_proto, ctx, captures) in context_chain { for (pat_context_ptr, pat_index) in context_iter(ctx) { - let mut pat_context = pat_context_ptr.borrow_mut(); - let match_pat = pat_context.match_at_mut(pat_index); + let pat_context = pat_context_ptr.borrow(); + let match_pat = pat_context.match_at(pat_index); if let Some(match_region) = self.search( line, start, match_pat, captures, search_cache, regions @@ -374,7 +374,7 @@ impl ParseState { fn search(&self, line: &str, start: usize, - match_pat: &mut MatchPattern, + match_pat: &MatchPattern, captures: Option<&(Region, String)>, search_cache: &mut SearchCache, regions: &mut Region) @@ -396,24 +396,30 @@ impl ParseState { } } - match_pat.ensure_compiled_if_possible(); - let refs_regex = if match_pat.has_captures && captures.is_some() { + let (matched, can_cache) = if match_pat.has_captures && captures.is_some() { let &(ref region, ref s) = captures.unwrap(); - Some(match_pat.compile_with_refs(region, s)) + let regex = match_pat.regex_with_refs(region, s); + let matched = regex.search_with_param( + line, + start, + line.len(), + SearchOptions::SEARCH_OPTION_NONE, + Some(regions), + MatchParam::default(), + ); + (matched, false) } else { - None + let regex = match_pat.regex(); + let matched = regex.search_with_param( + line, + start, + line.len(), + SearchOptions::SEARCH_OPTION_NONE, + Some(regions), + MatchParam::default() + ); + (matched, true) }; - let regex = if let Some(ref rgx) = refs_regex { - rgx - } else { - match_pat.regex.as_ref().unwrap() - }; - let matched = regex.search_with_param(line, - start, - line.len(), - SearchOptions::SEARCH_OPTION_NONE, - Some(regions), - MatchParam::default()); // If there's an error during search, treat it as non-matching. // For example, in case of catastrophic backtracking, onig should @@ -425,14 +431,14 @@ impl ParseState { MatchOperation::None => match_start != match_end, _ => true, }; - if refs_regex.is_none() && does_something { + if can_cache && does_something { search_cache.insert(match_pat, Some(regions.clone())); } if does_something { // print!("catch {} at {} on {}", match_pat.regex_str, match_start, line); return Some(regions.clone()); } - } else if refs_regex.is_none() { + } else if can_cache { search_cache.insert(match_pat, None); } return None; diff --git a/src/parsing/syntax_definition.rs b/src/parsing/syntax_definition.rs index 8ee55157..2829c88a 100644 --- a/src/parsing/syntax_definition.rs +++ b/src/parsing/syntax_definition.rs @@ -5,6 +5,7 @@ //! into this data structure? use std::collections::{BTreeMap, HashMap}; use std::hash::Hash; +use lazycell::AtomicLazyCell; use onig::{Regex, RegexOptions, Region, Syntax}; use std::rc::{Rc, Weak}; use std::cell::RefCell; @@ -86,16 +87,17 @@ pub struct MatchIter { index_stack: Vec, } -#[derive(Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct MatchPattern { pub has_captures: bool, pub regex_str: String, - #[serde(skip_serializing, skip_deserializing)] - pub regex: Option, pub scope: Vec, pub captures: Option, pub operation: MatchOperation, pub with_prototype: Option, + + #[serde(skip_serializing, skip_deserializing, default = "AtomicLazyCell::new")] + regex: AtomicLazyCell, } /// This wrapper only exists so that I can implement a serialization @@ -186,14 +188,6 @@ impl Context { _ => panic!("bad index to match_at"), } } - - /// Returns a mutable reference, otherwise like `match_at` - pub fn match_at_mut(&mut self, index: usize) -> &mut MatchPattern { - match self.patterns[index] { - Pattern::Match(ref mut match_pat) => match_pat, - _ => panic!("bad index to match_at"), - } - } } impl ContextReference { @@ -233,6 +227,25 @@ pub(crate) fn substitute_backrefs_in_regex(regex_str: &str, substituter: F) - impl MatchPattern { + pub fn new( + has_captures: bool, + regex_str: String, + scope: Vec, + captures: Option, + operation: MatchOperation, + with_prototype: Option, + ) -> MatchPattern { + MatchPattern { + has_captures, + regex_str, + scope, + captures, + operation, + with_prototype, + regex: AtomicLazyCell::new(), + } + } + /// substitutes back-refs in Regex with regions from s /// used for match patterns which refer to captures from the pattern /// that pushed them. @@ -244,7 +257,7 @@ impl MatchPattern { /// Used by the parser to compile a regex which needs to reference /// regions from another matched pattern. - pub fn compile_with_refs(&self, region: &Region, s: &str) -> Regex { + pub fn regex_with_refs(&self, region: &Region, s: &str) -> Regex { // TODO don't panic on invalid regex Regex::with_options(&self.regex_with_substitutes(region, s), RegexOptions::REGEX_OPTION_CAPTURE_GROUP, @@ -252,25 +265,39 @@ impl MatchPattern { .unwrap() } - fn compile_regex(&mut self) { - // TODO don't panic on invalid regex - let compiled = Regex::with_options(&self.regex_str, - RegexOptions::REGEX_OPTION_CAPTURE_GROUP, - Syntax::default()) - .unwrap(); - self.regex = Some(compiled); + pub fn regex(&self) -> &Regex { + if let Some(regex) = self.regex.borrow() { + regex + } else { + // TODO don't panic on invalid regex + let regex = Regex::with_options( + &self.regex_str, + RegexOptions::REGEX_OPTION_CAPTURE_GROUP, + Syntax::default(), + ).unwrap(); + // Fill returns an error if it has already been filled. This might + // happen if two threads race here. In that case, just use the value + // that won and is now in the cell. + self.regex.fill(regex).ok(); + self.regex.borrow().unwrap() + } } +} - /// Makes sure the regex is compiled if it doesn't have captures. - /// May compile the regex if it isn't, panicing if compilation fails. - #[inline] - pub fn ensure_compiled_if_possible(&mut self) { - if self.regex.is_none() && !self.has_captures { - self.compile_regex(); - } +impl Eq for MatchPattern {} + +impl PartialEq for MatchPattern { + fn eq(&self, other: &MatchPattern) -> bool { + self.has_captures == other.has_captures && + self.regex_str == other.regex_str && + self.scope == other.scope && + self.captures == other.captures && + self.operation == other.operation && + self.with_prototype == other.with_prototype } } + impl Eq for LinkerLink {} impl PartialEq for LinkerLink { @@ -307,17 +334,18 @@ fn ordered_map(map: &HashMap, serializer: S) -> Result Date: Fri, 1 Jun 2018 12:02:27 +1000 Subject: [PATCH 02/58] WIP use arena and indices for contexts --- src/easy.rs | 21 ++-- src/highlighting/highlighter.rs | 2 +- src/html.rs | 12 +-- src/parsing/parser.rs | 123 +++++++++++------------- src/parsing/syntax_definition.rs | 125 +++++++++++++----------- src/parsing/syntax_set.rs | 158 ++++++++++++++++++------------- src/parsing/yaml_load.rs | 130 +++++++++++++++---------- 7 files changed, 312 insertions(+), 259 deletions(-) diff --git a/src/easy.rs b/src/easy.rs index 027f5a9d..f222723e 100644 --- a/src/easy.rs +++ b/src/easy.rs @@ -39,17 +39,18 @@ use std::path::Path; /// ``` pub struct HighlightLines<'a> { highlighter: Highlighter<'a>, - parse_state: ParseState, + parse_state: ParseState<'a>, highlight_state: HighlightState, } impl<'a> HighlightLines<'a> { - pub fn new(syntax: &SyntaxDefinition, theme: &'a Theme) -> HighlightLines<'a> { + // TODO: should syntax come first or the set? + pub fn new(syntax_set: &'a SyntaxSet, syntax: &'a SyntaxDefinition, theme: &'a Theme) -> HighlightLines<'a> { let highlighter = Highlighter::new(theme); let hstate = HighlightState::new(&highlighter, ScopeStack::new()); HighlightLines { highlighter: highlighter, - parse_state: ParseState::new(syntax), + parse_state: ParseState::new(syntax_set, syntax), highlight_state: hstate, } } @@ -101,7 +102,7 @@ impl<'a> HighlightFile<'a> { /// } /// ``` pub fn new>(path_obj: P, - ss: &SyntaxSet, + ss: &'a SyntaxSet, theme: &'a Theme) -> io::Result> { let path: &Path = path_obj.as_ref(); @@ -111,7 +112,7 @@ impl<'a> HighlightFile<'a> { Ok(HighlightFile { reader: BufReader::new(f), - highlight_lines: HighlightLines::new(syntax, theme), + highlight_lines: HighlightLines::new(ss, syntax, theme), }) } } @@ -184,10 +185,10 @@ mod tests { #[test] fn can_highlight_lines() { - let ps = SyntaxSet::load_defaults_nonewlines(); + let ss = SyntaxSet::load_defaults_nonewlines(); let ts = ThemeSet::load_defaults(); - let syntax = ps.find_syntax_by_extension("rs").unwrap(); - let mut h = HighlightLines::new(syntax, &ts.themes["base16-ocean.dark"]); + let syntax = ss.find_syntax_by_extension("rs").unwrap(); + let mut h = HighlightLines::new(&ss, syntax, &ts.themes["base16-ocean.dark"]); let ranges = h.highlight("pub struct Wow { hi: u64 }"); assert!(ranges.len() > 4); } @@ -205,7 +206,7 @@ mod tests { #[test] fn can_find_regions() { let ss = SyntaxSet::load_defaults_nonewlines(); - let mut state = ParseState::new(ss.find_syntax_by_extension("rb").unwrap()); + let mut state = ParseState::new(&ss, ss.find_syntax_by_extension("rb").unwrap()); let line = "lol =5+2"; let ops = state.parse_line(line); @@ -229,7 +230,7 @@ mod tests { #[test] fn can_find_regions_with_trailing_newline() { let ss = SyntaxSet::load_defaults_newlines(); - let mut state = ParseState::new(ss.find_syntax_by_extension("rb").unwrap()); + let mut state = ParseState::new(&ss, ss.find_syntax_by_extension("rb").unwrap()); let lines = ["# hello world\n", "lol=5+2\n"]; let mut stack = ScopeStack::new(); diff --git a/src/highlighting/highlighter.rs b/src/highlighting/highlighter.rs index 8f270f4c..50461925 100644 --- a/src/highlighting/highlighter.rs +++ b/src/highlighting/highlighter.rs @@ -283,7 +283,7 @@ mod tests { let ps = SyntaxSet::load_from_folder("testdata/Packages").unwrap(); let mut state = { let syntax = ps.find_syntax_by_name("Ruby on Rails").unwrap(); - ParseState::new(syntax) + ParseState::new(&ps, syntax) }; let ts = ThemeSet::load_defaults(); let highlighter = Highlighter::new(&ts.themes["base16-ocean.dark"]); diff --git a/src/html.rs b/src/html.rs index 6ae38b8a..2f724510 100644 --- a/src/html.rs +++ b/src/html.rs @@ -38,9 +38,9 @@ fn scope_to_classes(s: &mut String, scope: Scope, style: ClassStyle) { /// Note that the `syntax` passed in must be from a `SyntaxSet` compiled for no newline characters. /// This is easy to get with `SyntaxSet::load_defaults_nonewlines()`. If you think this is the wrong /// choice of `SyntaxSet` to accept, I'm not sure of it either, email me. -pub fn highlighted_snippet_for_string(s: &str, syntax: &SyntaxDefinition, theme: &Theme) -> String { +pub fn highlighted_snippet_for_string(s: &str, ss: &SyntaxSet, syntax: &SyntaxDefinition, theme: &Theme) -> String { let mut output = String::new(); - let mut highlighter = HighlightLines::new(syntax, theme); + let mut highlighter = HighlightLines::new(ss, syntax, theme); let c = theme.settings.background.unwrap_or(Color::WHITE); write!(output, "
\n",
@@ -241,9 +241,9 @@ mod tests {
     use highlighting::{ThemeSet, Style, Highlighter, HighlightIterator, HighlightState};
     #[test]
     fn tokens() {
-        let ps = SyntaxSet::load_defaults_nonewlines();
-        let syntax = ps.find_syntax_by_name("Markdown").unwrap();
-        let mut state = ParseState::new(syntax);
+        let ss = SyntaxSet::load_defaults_nonewlines();
+        let syntax = ss.find_syntax_by_name("Markdown").unwrap();
+        let mut state = ParseState::new(&ss, syntax);
         let line = "[w](t.co) *hi* **five**";
         let ops = state.parse_line(line);
 
@@ -271,7 +271,7 @@ mod tests {
         let ts = ThemeSet::load_defaults();
         let s = include_str!("../testdata/highlight_test.erb");
         let syntax = ss.find_syntax_by_extension("erb").unwrap();
-        let html = highlighted_snippet_for_string(s, syntax, &ts.themes["base16-ocean.dark"]);
+        let html = highlighted_snippet_for_string(s, &ss, syntax, &ts.themes["base16-ocean.dark"]);
         assert_eq!(html, include_str!("../testdata/test3.html"));
         let html2 = highlighted_snippet_for_file("testdata/highlight_test.erb",
                                                  &ss,
diff --git a/src/parsing/parser.rs b/src/parsing/parser.rs
index 5a8c6d95..6e189fa9 100644
--- a/src/parsing/parser.rs
+++ b/src/parsing/parser.rs
@@ -7,6 +7,7 @@ use std::i32;
 use std::hash::BuildHasherDefault;
 use std::ptr;
 use fnv::FnvHasher;
+use parsing::syntax_set::SyntaxSet;
 
 /// Keeps the current parser state (the internal syntax interpreter stack) between lines of parsing.
 /// If you are parsing an entire file you create one of these at the start and use it
@@ -24,9 +25,11 @@ use fnv::FnvHasher;
 ///
 /// **Note:** Caching is for advanced users who have tons of time to maximize performance or want to do so eventually.
 /// It is not recommended that you try caching the first time you implement highlighting.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct ParseState {
-    stack: Vec,
+// TODO: removed PartialEq, Eq from this because SyntaxSet is not
+#[derive(Debug, Clone)]
+pub struct ParseState<'a> {
+    syntax_set: &'a SyntaxSet,
+    stack: Vec>,
     first_line: bool,
     // See issue #101. Contains indices of frames pushed by `with_prototype`s.
     // Doesn't look at `with_prototype`s below top of stack.
@@ -34,21 +37,17 @@ pub struct ParseState {
 }
 
 #[derive(Debug, Clone)]
-struct StateLevel {
-    context: ContextPtr,
-    prototype: Option,
+struct StateLevel<'a> {
+    context: &'a Context,
+    prototype: Option<&'a Context>,
     captures: Option<(Region, String)>,
 }
 
-fn context_ptr_eq(a: &ContextPtr, b: &ContextPtr) -> bool {
-    ptr::eq(a.as_ptr(), b.as_ptr())
-}
-
-impl PartialEq for StateLevel {
+impl<'a> PartialEq for StateLevel<'a> {
     fn eq(&self, other: &StateLevel) -> bool {
-        context_ptr_eq(&self.context, &other.context) &&
+        self.context == other.context &&
         match (&self.prototype, &other.prototype) {
-            (&Some(ref a), &Some(ref b)) => context_ptr_eq(a, b),
+            (&Some(ref a), &Some(ref b)) => a == b,
             (&None, &None) => true,
             _ => false,
         } &&
@@ -56,12 +55,12 @@ impl PartialEq for StateLevel {
     }
 }
 
-impl Eq for StateLevel {}
+impl<'a> Eq for StateLevel<'a> {}
 
 #[derive(Debug)]
-struct RegexMatch {
+struct RegexMatch<'a> {
     regions: Region,
-    context: ContextPtr,
+    context: &'a Context,
     pat_index: usize,
     from_with_prototype: bool,
     would_loop: bool,
@@ -162,17 +161,18 @@ type SearchCache = HashMap<*const MatchPattern, Option, BuildHasherDefau
 // So in our input string, we'd skip one character and try to match the rules
 // again. This time, the "\w+" wins because it comes first.
 
-impl ParseState {
+impl<'a> ParseState<'a> {
     /// Create a state from a syntax, keeps its own reference counted
     /// pointer to the main context of the syntax.
-    pub fn new(syntax: &SyntaxDefinition) -> ParseState {
+    pub fn new(syntax_set: &'a SyntaxSet, syntax: &'a SyntaxDefinition) -> ParseState<'a> {
         let start_state = StateLevel {
             // __start is a special context we add in yaml_load.rs
-            context: syntax.contexts["__start"].clone(),
+            context: &syntax.start_context,
             prototype: None,
             captures: None,
         };
         ParseState {
+            syntax_set,
             stack: vec![start_state],
             first_line: true,
             proto_starts: Vec::new(),
@@ -197,7 +197,7 @@ impl ParseState {
 
         if self.first_line {
             let cur_level = &self.stack[self.stack.len() - 1];
-            let context = cur_level.context.borrow();
+            let context = cur_level.context;
             if !context.meta_content_scope.is_empty() {
                 res.push((0, ScopeStackOp::Push(context.meta_content_scope[0])));
             }
@@ -270,7 +270,7 @@ impl ParseState {
                 // "push", remember the position and stack size so that we can
                 // check the next "pop" for loops. Otherwise leave the state,
                 // e.g. non-consuming "set" could also result in a loop.
-                let context = reg_match.context.borrow();
+                let context = reg_match.context;
                 let match_pattern = context.match_at(reg_match.pat_index);
                 if let MatchOperation::Push(_) = match_pattern.operation {
                     *non_consuming_push_at = (match_end, self.stack.len() + 1);
@@ -285,7 +285,7 @@ impl ParseState {
                 self.proto_starts.push(self.stack.len());
             }
 
-            let level_context = self.stack[self.stack.len() - 1].context.clone();
+            let level_context = self.stack[self.stack.len() - 1].context;
             self.exec_pattern(line, reg_match, level_context, ops);
 
             true
@@ -300,12 +300,9 @@ impl ParseState {
                        search_cache: &mut SearchCache,
                        regions: &mut Region,
                        check_pop_loop: bool)
-                       -> Option {
+                       -> Option> {
         let cur_level = &self.stack[self.stack.len() - 1];
-        let prototype: Option = {
-            let ctx_ref = cur_level.context.borrow();
-            ctx_ref.prototype.clone()
-        };
+        let prototype = cur_level.context.prototype.map(|p| p.resolve(self.syntax_set));
 
         // Build an iterator for the contexts we want to visit in order
         let context_chain = {
@@ -313,7 +310,7 @@ impl ParseState {
             // Sublime applies with_prototypes from bottom to top
             let with_prototypes = self.stack[proto_start..].iter().filter_map(|lvl| lvl.prototype.as_ref().map(|ctx| (true, ctx.clone(), lvl.captures.as_ref())));
             let cur_prototype = prototype.into_iter().map(|ctx| (false, ctx, None));
-            let cur_context = Some((false, cur_level.context.clone(), cur_level.captures.as_ref())).into_iter();
+            let cur_context = Some((false, cur_level.context, cur_level.captures.as_ref())).into_iter();
             with_prototypes.chain(cur_prototype).chain(cur_context)
         };
 
@@ -325,8 +322,7 @@ impl ParseState {
         let mut pop_would_loop = false;
 
         for (from_with_proto, ctx, captures) in context_chain {
-            for (pat_context_ptr, pat_index) in context_iter(ctx) {
-                let pat_context = pat_context_ptr.borrow();
+            for (pat_context, pat_index) in context_iter(self.syntax_set, ctx) {
                 let match_pat = pat_context.match_at(pat_index);
 
                 if let Some(match_region) = self.search(
@@ -353,7 +349,7 @@ impl ParseState {
 
                         best_match = Some(RegexMatch {
                             regions: match_region,
-                            context: pat_context_ptr.clone(),
+                            context: pat_context,
                             pat_index,
                             from_with_prototype: from_with_proto,
                             would_loop: pop_would_loop,
@@ -447,17 +443,16 @@ impl ParseState {
     /// Returns true if the stack was changed
     fn exec_pattern(&mut self,
                     line: &str,
-                    reg_match: RegexMatch,
-                    level_context_ptr: ContextPtr,
+                    reg_match: RegexMatch<'a>,
+                    level_context: &Context,
                     ops: &mut Vec<(usize, ScopeStackOp)>)
                     -> bool {
         let (match_start, match_end) = reg_match.regions.pos(0).unwrap();
-        let context = reg_match.context.borrow();
+        let context = reg_match.context;
         let pat = context.match_at(reg_match.pat_index);
-        let level_context = level_context_ptr.borrow();
         // println!("running pattern {:?} on '{}' at {}, operation {:?}", pat.regex_str, line, match_start, pat.operation);
 
-        self.push_meta_ops(true, match_start, &*level_context, &pat.operation, ops);
+        self.push_meta_ops(true, match_start, level_context, &pat.operation, ops);
         for s in &pat.scope {
             // println!("pushing {:?} at {}", s, match_start);
             ops.push((match_start, ScopeStackOp::Push(*s)));
@@ -535,8 +530,7 @@ impl ParseState {
                 if initial {
                     // add each context's meta scope
                     for r in context_refs.iter() {
-                        let ctx_ptr = r.resolve();
-                        let ctx = ctx_ptr.borrow();
+                        let ctx = r.resolve(self.syntax_set);
 
                         if !is_set {
                             if let Some(clear_amount) = ctx.clear_scopes {
@@ -550,16 +544,14 @@ impl ParseState {
                     }
                 } else {
                     let repush = (is_set && (!cur_context.meta_scope.is_empty() || !cur_context.meta_content_scope.is_empty())) || context_refs.iter().any(|r| {
-                        let ctx_ptr = r.resolve();
-                        let ctx = ctx_ptr.borrow();
+                        let ctx = r.resolve(self.syntax_set);
 
                         !ctx.meta_content_scope.is_empty() || (ctx.clear_scopes.is_some() && is_set)
                     });
                     if repush {
                         // remove previously pushed meta scopes, so that meta content scopes will be applied in the correct order
                         let mut num_to_pop : usize = context_refs.iter().map(|r| {
-                            let ctx_ptr = r.resolve();
-                            let ctx = ctx_ptr.borrow();
+                            let ctx = r.resolve(self.syntax_set);
                             ctx.meta_scope.len()
                         }).sum();
 
@@ -575,8 +567,7 @@ impl ParseState {
 
                         // now we push meta scope and meta context scope for each context pushed
                         for r in context_refs {
-                            let ctx_ptr = r.resolve();
-                            let ctx = ctx_ptr.borrow();
+                            let ctx = r.resolve(self.syntax_set);
 
                             // for some reason, contrary to my reading of the docs, set does this after the token
                             if is_set {
@@ -600,7 +591,7 @@ impl ParseState {
     }
 
     /// Returns true if the stack was changed
-    fn perform_op(&mut self, line: &str, regions: &Region, pat: &MatchPattern) -> bool {
+    fn perform_op(&mut self, line: &str, regions: &Region, pat: &'a MatchPattern) -> bool {
         let (ctx_refs, old_proto) = match pat.operation {
             MatchOperation::Push(ref ctx_refs) => (ctx_refs, None),
             MatchOperation::Set(ref ctx_refs) => {
@@ -623,15 +614,15 @@ impl ParseState {
                 // top most on the stack after all the contexts are pushed - this is also
                 // referred to as the "target" of the push by sublimehq - see
                 // https://forum.sublimetext.com/t/dev-build-3111/19240/17 for more info
-                pat.with_prototype.clone().or_else(|| old_proto.clone())
+                pat.with_prototype.as_ref().clone().or_else(|| old_proto.clone())
             } else {
                 None
             };
-            let ctx_ptr = r.resolve();
+            let context = r.resolve(self.syntax_set);
             let captures = {
-                let mut uses_backrefs = ctx_ptr.borrow().uses_backrefs;
+                let mut uses_backrefs = context.uses_backrefs;
                 if let Some(ref proto) = proto {
-                    uses_backrefs = uses_backrefs || proto.borrow().uses_backrefs;
+                    uses_backrefs = uses_backrefs || proto.uses_backrefs;
                 }
                 if uses_backrefs {
                     Some((regions.clone(), line.to_owned()))
@@ -640,7 +631,7 @@ impl ParseState {
                 }
             };
             self.stack.push(StateLevel {
-                context: ctx_ptr,
+                context,
                 prototype: proto,
                 captures,
             });
@@ -661,10 +652,10 @@ mod tests {
 
     #[test]
     fn can_parse_simple() {
-        let ps = SyntaxSet::load_from_folder("testdata/Packages").unwrap();
+        let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap();
         let mut state = {
-            let syntax = ps.find_syntax_by_name("Ruby on Rails").unwrap();
-            ParseState::new(syntax)
+            let syntax = ss.find_syntax_by_name("Ruby on Rails").unwrap();
+            ParseState::new(&ss, syntax)
         };
 
         let ops1 = ops("module Bob::Wow::Troll::Five; 5; end", &mut state);
@@ -697,10 +688,10 @@ mod tests {
 
     #[test]
     fn can_parse_includes() {
-        let ps = SyntaxSet::load_from_folder("testdata/Packages").unwrap();
+        let ss = SyntaxSet::load_from_folder("testdata/Packages").unwrap();
         let mut state = {
-            let syntax = ps.find_syntax_by_name("HTML (Rails)").unwrap();
-            ParseState::new(syntax)
+            let syntax = ss.find_syntax_by_name("HTML (Rails)").unwrap();
+            ParseState::new(&ss, syntax)
         };
 
         let ops = ops("