From fc30e75b54dae158d48750265d412bd67a6bf9f4 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Mon, 2 Jul 2018 15:31:39 +0300
Subject: [PATCH 01/12] initial rework of syntest to be usable internally

---
 examples/syntest.rs | 218 ++++-------------------------------------
 src/easy.rs         | 232 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 248 insertions(+), 202 deletions(-)

diff --git a/examples/syntest.rs b/examples/syntest.rs
index 0b09b475..6c56ecce 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -15,16 +15,14 @@ extern crate regex;
 extern crate getopts;
 
 //extern crate onig;
-use syntect::parsing::{SyntaxSet, ParseState, ScopeStack, Scope};
-use syntect::highlighting::ScopeSelectors;
-use syntect::easy::{ScopeRegionIterator};
+use syntect::parsing::{SyntaxSet};
+use syntect::easy::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions};
 
 use std::path::Path;
+use std::io::prelude::*;
 use std::io::{BufRead, BufReader};
 use std::fs::File;
-use std::cmp::{min, max};
 use std::time::Instant;
-use std::str::FromStr;
 
 use getopts::Options;
 use regex::Regex;
@@ -36,12 +34,6 @@ pub enum SyntaxTestHeaderError {
     SyntaxDefinitionNotFound,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum SyntaxTestFileResult {
-    FailedAssertions(usize, usize),
-    Success(usize),
-}
-
 lazy_static! {
     pub static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm)
             ^(?P<testtoken_start>\s*\S+)
@@ -49,113 +41,19 @@ lazy_static! {
             "(?P<syntax_file>[^"]+)"
             \s*(?P<testtoken_end>\S+)?$
         "#).unwrap();
-    pub static ref SYNTAX_TEST_ASSERTION_PATTERN: Regex = Regex::new(r#"(?xm)
-        \s*(?:
-            (?P<begin_of_token><-)|(?P<range>\^+)
-        )(.*)$"#).unwrap();
-}
-
-#[derive(Clone, Copy)]
-struct OutputOptions {
-    time: bool,
-    debug: bool,
-    summary: bool,
-}
-
-#[derive(Debug)]
-struct AssertionRange<'a> {
-    begin_char: usize,
-    end_char: usize,
-    scope_selector_text: &'a str,
-    is_pure_assertion_line: bool,
-}
-
-#[derive(Debug)]
-struct ScopedText {
-    scope: Vec<Scope>,
-    char_start: usize,
-    text_len: usize,
-}
-
-#[derive(Debug)]
-struct RangeTestResult {
-    column_begin: usize,
-    column_end: usize,
-    success: bool,
-}
-
-fn get_line_assertion_details<'a>(testtoken_start: &str, testtoken_end: Option<&str>, line: &'a str) -> Option<AssertionRange<'a>> {
-    // if the test start token specified in the test file's header is on the line
-    if let Some(index) = line.find(testtoken_start) {
-        let (before_token_start, token_and_rest_of_line) = line.split_at(index);
-
-        if let Some(captures) = SYNTAX_TEST_ASSERTION_PATTERN.captures(&token_and_rest_of_line[testtoken_start.len()..]) {
-            let mut sst = captures.get(3).unwrap().as_str(); // get the scope selector text
-            let mut only_whitespace_after_token_end = true;
-
-            if let Some(token) = testtoken_end { // if there is an end token defined in the test file header
-                if let Some(end_token_pos) = sst.find(token) { // and there is an end token in the line
-                    let (ss, after_token_end) = sst.split_at(end_token_pos); // the scope selector text ends at the end token
-                    sst = &ss;
-                    only_whitespace_after_token_end = after_token_end.trim_right().is_empty();
-                }
-            }
-            return Some(AssertionRange {
-                begin_char: index + if captures.get(2).is_some() { testtoken_start.len() + captures.get(2).unwrap().start() } else { 0 },
-                end_char: index + if captures.get(2).is_some() { testtoken_start.len() + captures.get(2).unwrap().end() } else { 1 },
-                scope_selector_text: sst,
-                is_pure_assertion_line: before_token_start.trim_left().is_empty() && only_whitespace_after_token_end, // if only whitespace surrounds the test tokens on the line, then it is a pure assertion line
-            });
-        }
-    }
-    None
-}
-
-fn process_assertions(assertion: &AssertionRange, test_against_line_scopes: &Vec<ScopedText>) -> Vec<RangeTestResult> {
-    // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -"
-    // and they are sometimes in the syntax test as ^^^-comment, for example
-    let selector = ScopeSelectors::from_str(&format!(" {}", &assertion.scope_selector_text)).unwrap();
-    // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached
-    let mut results = Vec::new();
-    for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) {
-        let match_value = selector.does_match(scoped_text.scope.as_slice());
-        let result = RangeTestResult {
-            column_begin: max(scoped_text.char_start, assertion.begin_char),
-            column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char),
-            success: match_value.is_some()
-        };
-        results.push(result);
-    }
-    // don't ignore assertions after the newline, they should be treated as though they are asserting against the newline
-    let last = test_against_line_scopes.last().unwrap();
-    if last.char_start + last.text_len < assertion.end_char {
-        let match_value = selector.does_match(last.scope.as_slice());
-        let result = RangeTestResult {
-            column_begin: max(last.char_start + last.text_len, assertion.begin_char),
-            column_end: assertion.end_char,
-            success: match_value.is_some()
-        };
-        results.push(result);
-    }
-    results
 }
 
-/// If `parse_test_lines` is `false` then lines that only contain assertions are not parsed
-fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: OutputOptions) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
-    use syntect::util::debug_print_ops;
+fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
     let f = File::open(path).unwrap();
     let mut reader = BufReader::new(f);
-    let mut line = String::new();
+    let mut header_line = String::new();
 
     // read the first line from the file - if we have reached EOF already, it's an invalid file
-    if reader.read_line(&mut line).unwrap() == 0 {
+    if reader.read_line(&mut header_line).unwrap() == 0 {
         return Err(SyntaxTestHeaderError::MalformedHeader);
     }
 
-    line = line.replace("\r", &"");
-
     // parse the syntax test header in the first line of the file
-    let header_line = line.clone();
     let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line);
     let captures = search_result.ok_or(SyntaxTestHeaderError::MalformedHeader)?;
 
@@ -165,101 +63,19 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: Outp
 
     // find the relevant syntax definition to parse the file with - case is important!
     if !out_opts.summary {
-        println!("The test file references syntax definition file: {}", syntax_file);
+        println!("The test file references syntax definition file: {}", syntax_file); //" and the start test token is {} and the end token is {:?}", testtoken_start, testtoken_end);
     }
     let syntax = ss.find_syntax_by_path(syntax_file).ok_or(SyntaxTestHeaderError::SyntaxDefinitionNotFound)?;
 
-    // iterate over the lines of the file, testing them
-    let mut state = ParseState::new(syntax);
-    let mut stack = ScopeStack::new();
+    let mut contents = String::new();
+    contents.push_str(&header_line);
+    reader.read_to_string(&mut contents).expect("Unable to read file");
+    contents = contents.replace("\r", &"");
 
-    let mut current_line_number = 1;
-    let mut test_against_line_number = 1;
-    let mut scopes_on_line_being_tested = Vec::new();
-    let mut previous_non_assertion_line = line.to_string();
-
-    let mut assertion_failures: usize = 0;
-    let mut total_assertions: usize = 0;
-
-    loop { // over lines of file, starting with the header line
-        let mut line_only_has_assertion = false;
-        let mut line_has_assertion = false;
-        if let Some(assertion) = get_line_assertion_details(testtoken_start, testtoken_end, &line) {
-            let result = process_assertions(&assertion, &scopes_on_line_being_tested);
-            total_assertions += &assertion.end_char - &assertion.begin_char;
-            for failure in result.iter().filter(|r|!r.success) {
-                let length = failure.column_end - failure.column_begin;
-                let text: String = previous_non_assertion_line.chars().skip(failure.column_begin).take(length).collect();
-                if !out_opts.summary {
-                    println!("  Assertion selector {:?} \
-                        from line {:?} failed against line {:?}, column range {:?}-{:?} \
-                        (with text {:?}) \
-                        has scope {:?}",
-                        assertion.scope_selector_text.trim(),
-                        current_line_number, test_against_line_number, failure.column_begin, failure.column_end,
-                        text,
-                        scopes_on_line_being_tested.iter().skip_while(|s|s.char_start + s.text_len <= failure.column_begin).next().unwrap_or(scopes_on_line_being_tested.last().unwrap()).scope
-                    );
-                }
-                assertion_failures += failure.column_end - failure.column_begin;
-            }
-            line_only_has_assertion = assertion.is_pure_assertion_line;
-            line_has_assertion = true;
-        }
-        if !line_only_has_assertion || parse_test_lines {
-            if !line_has_assertion { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against
-                scopes_on_line_being_tested.clear();
-                test_against_line_number = current_line_number;
-                previous_non_assertion_line = line.to_string();
-            }
-            if out_opts.debug && !line_only_has_assertion {
-                println!("-- debugging line {} -- scope stack: {:?}", current_line_number, stack);
-            }
-            let ops = state.parse_line(&line);
-            if out_opts.debug && !line_only_has_assertion {
-                if ops.is_empty() && !line.is_empty() {
-                    println!("no operations for this line...");
-                } else {
-                    debug_print_ops(&line, &ops);
-                }
-            }
-            let mut col: usize = 0;
-            for (s, op) in ScopeRegionIterator::new(&ops, &line) {
-                stack.apply(op);
-                if s.is_empty() { // in this case we don't care about blank tokens
-                    continue;
-                }
-                if !line_has_assertion {
-                    // if the line has no assertions on it, remember the scopes on the line so we can test against them later
-                    let len = s.chars().count();
-                    scopes_on_line_being_tested.push(
-                        ScopedText {
-                            char_start: col,
-                            text_len: len,
-                            scope: stack.as_slice().to_vec()
-                        }
-                    );
-                    // TODO: warn when there are duplicate adjacent (non-meta?) scopes, as it is almost always undesired
-                    col += len;
-                }
-            }
-        }
-
-        line.clear();
-        current_line_number += 1;
-        if reader.read_line(&mut line).unwrap() == 0 {
-            break;
-        }
-        line = line.replace("\r", &"");
-    }
-    let res = if assertion_failures > 0 {
-        Ok(SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions))
-    } else {
-        Ok(SyntaxTestFileResult::Success(total_assertions))
-    };
+    let res = process_syntax_test_assertions(&syntax, &contents, testtoken_start, testtoken_end, &out_opts);
 
     if out_opts.summary {
-        if let Ok(SyntaxTestFileResult::FailedAssertions(failures, _)) = res {
+        if let SyntaxTestFileResult::FailedAssertions(failures, _) = res {
             // Don't print total assertion count so that diffs don't pick up new succeeding tests
             println!("FAILED {}: {}", path.display(), failures);
         }
@@ -267,7 +83,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, parse_test_lines: bool, out_opts: Outp
         println!("{:?}", res);
     }
 
-    res
+    Ok(res)
 }
 
 fn main() {
@@ -309,7 +125,7 @@ fn main() {
         ss.link_syntaxes();
     }
 
-    let out_opts = OutputOptions {
+    let out_opts = SyntaxTestOutputOptions {
         debug: matches.opt_present("debug"),
         time: matches.opt_present("time"),
         summary: matches.opt_present("summary"),
@@ -322,7 +138,7 @@ fn main() {
 }
 
 
-fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 {
+fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions) -> i32 {
     let mut exit_code: i32 = 0; // exit with code 0 by default, if all tests pass
     let walker = WalkDir::new(path).into_iter();
 
@@ -341,7 +157,7 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: OutputOptions) -> i32 {
             println!("Testing file {}", path.display());
         }
         let start = Instant::now();
-        let result = test_file(&ss, path, true, out_opts);
+        let result = test_file(&ss, path, out_opts);
         let elapsed = start.elapsed();
         if out_opts.time {
             let ms = (elapsed.as_secs() * 1_000) + (elapsed.subsec_nanos() / 1_000_000) as u64;
diff --git a/src/easy.rs b/src/easy.rs
index 027f5a9d..58acc8fb 100644
--- a/src/easy.rs
+++ b/src/easy.rs
@@ -2,7 +2,7 @@
 //! files without caring about intermediate semantic representation
 //! and caching.
 
-use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp};
+use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp, Scope};
 use highlighting::{Highlighter, HighlightState, HighlightIterator, Theme, Style};
 use std::io::{self, BufReader};
 use std::fs::File;
@@ -174,6 +174,236 @@ impl<'a> Iterator for ScopeRegionIterator<'a> {
     }
 }
 
+#[derive(Clone, Copy)]
+pub struct SyntaxTestOutputOptions {
+    pub time: bool,
+    pub debug: bool,
+    pub summary: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SyntaxTestFileResult {
+    FailedAssertions(usize, usize),
+    Success(usize),
+}
+
+pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult {
+    use std::collections::VecDeque;
+    use highlighting::ScopeSelectors;
+
+    #[derive(Debug)]
+    struct SyntaxTestAssertionRange {
+        test_line_offset: usize,
+        line_number: usize,
+        begin_char: usize,
+        end_char: usize,
+        scope_selector: ScopeSelectors,
+        scope_selector_text: String,
+    }
+
+    fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque<SyntaxTestAssertionRange> {
+        use std::str::FromStr;
+        
+        let mut assertions = VecDeque::new();
+        let mut test_line_offset = 0;
+        //let mut test_line_len = 0;
+        let mut line_number = 0;
+        let mut offset = 0;
+        //let mut remainder = None;
+        for line in text.lines() {
+            line_number += 1;
+            let mut line_has_assertions = false;
+            
+            // if the test start token specified is on the line
+            if let Some(index) = line.find(token_start) {
+                let token_and_rest_of_line = line.split_at(index).1;
+
+                let rest_of_line = &token_and_rest_of_line[token_start.len()..];
+                if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) {
+                    let mut assertion_range = 0;
+                    while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') {
+                        assertion_range += 1;
+                    }
+                    let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range };
+
+                    let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text
+
+                    if let Some(token) = token_end { // if there is an end token defined in the test file header
+                        if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line
+                            selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token
+                        }
+                    }
+
+                    let assertion = SyntaxTestAssertionRange {
+                        test_line_offset: test_line_offset,
+                        line_number: line_number,
+                        begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 },
+                        end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 },
+
+                        // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -"
+                        // and they are sometimes in the syntax test as ^^^-comment, for example
+                        scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)),
+                        scope_selector_text: selector_text,
+                    };
+                    /*if assertion.end_char > test_line_len {
+                        remainder = Some(SyntaxTestAssertionRange {
+                            test_line_offset: test_line_offset + test_line_len,
+                            line_number: line_number,
+                            begin_char: assertion.begin_char - test_line_len,
+                            end_char: assertion.end_char - test_line_len,
+                            scope_selector: assertion.scope_selector.clone(),
+                            scope_selector_text: assertion.scope_selector_text.clone(),
+                        });
+                    }*/
+                    assertions.push_back(assertion);
+                    
+                    line_has_assertions = true;
+                }
+            }
+            if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text
+                test_line_offset = offset;
+                //test_line_len = line.len() + 1;
+            }
+            offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405
+        }
+        assertions
+    }
+    
+    #[derive(Debug)]
+    struct ScopedText {
+        scope: Vec<Scope>,
+        char_start: usize,
+        text_len: usize,
+    }
+    
+    #[derive(Debug)]
+    struct RangeTestResult {
+        column_begin: usize,
+        column_end: usize,
+        success: bool,
+        actual_scope: String,
+    }
+    
+    fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec<ScopedText>) -> Vec<RangeTestResult> {
+        use std::cmp::{min, max};
+        // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached
+        let mut results = Vec::new();
+        for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) {
+            let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice());
+            let result = RangeTestResult {
+                column_begin: max(scoped_text.char_start, assertion.begin_char),
+                column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char),
+                success: match_value.is_some(),
+                actual_scope: format!("{:?}", scoped_text.scope.as_slice()),
+            };
+            results.push(result);
+        }
+        results
+    }
+    
+    let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
+    //println!("{:?}", assertions);
+    use util::debug_print_ops;
+    
+    // iterate over the lines of the file, testing them
+    let mut state = ParseState::new(syntax);
+    let mut stack = ScopeStack::new();
+
+    let mut offset = 0;
+    let mut scopes_on_line_being_tested = Vec::new();
+    let mut line_number = 0;
+    let mut relevant_assertions = Vec::new();
+    
+    let mut assertion_failures: usize = 0;
+    let mut total_assertions: usize = 0;
+
+    for line_without_char in text.lines() {
+        let line = &(line_without_char.to_owned() + "\n");
+        line_number += 1;
+        
+        let eol_offset = offset + line.len();
+        
+        // parse the line
+        let ops = state.parse_line(&line);
+        // find assertions that relate to the current line
+        relevant_assertions.clear();
+        while let Some(assertion) = assertions.pop_front() {
+            let pos = assertion.test_line_offset + assertion.begin_char;
+            if pos >= offset && pos < eol_offset {
+                relevant_assertions.push(assertion);
+            } else {
+                assertions.push_front(assertion);
+                break;
+            }
+        }
+        if !relevant_assertions.is_empty() {
+            scopes_on_line_being_tested.clear();
+            if out_opts.debug {
+                println!("-- debugging line {} -- scope stack: {:?}", line_number, stack);
+                if ops.is_empty() && !line.is_empty() {
+                    println!("no operations for this line...");
+                } else {
+                    debug_print_ops(&line, &ops);
+                }
+            }
+        }
+        
+        {
+            let mut col: usize = 0;
+            for (s, op) in ScopeRegionIterator::new(&ops, &line) {
+                stack.apply(op);
+                if s.is_empty() { // in this case we don't care about blank tokens
+                    continue;
+                }
+                if !relevant_assertions.is_empty() {
+                    let len = s.chars().count();
+                    scopes_on_line_being_tested.push(
+                        ScopedText {
+                            char_start: col,
+                            text_len: len,
+                            scope: stack.as_slice().to_vec()
+                        }
+                    );
+                    col += len;
+                }
+            }
+        }
+        
+        for assertion in &relevant_assertions {
+            let results = process_assertions(&assertion, &scopes_on_line_being_tested);
+            
+            for result in results {
+                let length = result.column_end - result.column_begin;
+                total_assertions += length;
+                if !result.success {
+                    assertion_failures += length;
+                    let text: String = line.chars().skip(result.column_begin).take(length).collect();
+                    if !out_opts.summary {
+                        println!("  Assertion selector {:?} \
+                            from line {:?} failed against line {:?}, column range {:?}-{:?} \
+                            (with text {:?}) \
+                            has scope {:?}",
+                            &assertion.scope_selector_text.trim(),
+                            &assertion.line_number, line_number, result.column_begin, result.column_end,
+                            text,
+                            result.actual_scope,
+                        );
+                    }
+                }
+            }
+        }
+        
+        offset = eol_offset;
+    }
+    
+    let res = if assertion_failures > 0 {
+        SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions)
+    } else {
+        SyntaxTestFileResult::Success(total_assertions)
+    };
+    res
+}
+
 #[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))]
 #[cfg(test)]
 mod tests {

From fdaf8bd9e384deadf3b471d04e747301cb486dc4 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Mon, 2 Jul 2018 19:39:41 +0300
Subject: [PATCH 02/12] put syntax test functionality in it's own module

---
 examples/syntest.rs |   2 +-
 src/easy.rs         | 232 +-------------------------------------
 src/lib.rs          |   2 +
 src/syntax_tests.rs | 263 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 267 insertions(+), 232 deletions(-)
 create mode 100644 src/syntax_tests.rs

diff --git a/examples/syntest.rs b/examples/syntest.rs
index 6c56ecce..a5b9b84b 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -16,7 +16,7 @@ extern crate getopts;
 
 //extern crate onig;
 use syntect::parsing::{SyntaxSet};
-use syntect::easy::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions};
+use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions};
 
 use std::path::Path;
 use std::io::prelude::*;
diff --git a/src/easy.rs b/src/easy.rs
index 58acc8fb..027f5a9d 100644
--- a/src/easy.rs
+++ b/src/easy.rs
@@ -2,7 +2,7 @@
 //! files without caring about intermediate semantic representation
 //! and caching.
 
-use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp, Scope};
+use parsing::{ScopeStack, ParseState, SyntaxDefinition, SyntaxSet, ScopeStackOp};
 use highlighting::{Highlighter, HighlightState, HighlightIterator, Theme, Style};
 use std::io::{self, BufReader};
 use std::fs::File;
@@ -174,236 +174,6 @@ impl<'a> Iterator for ScopeRegionIterator<'a> {
     }
 }
 
-#[derive(Clone, Copy)]
-pub struct SyntaxTestOutputOptions {
-    pub time: bool,
-    pub debug: bool,
-    pub summary: bool,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub enum SyntaxTestFileResult {
-    FailedAssertions(usize, usize),
-    Success(usize),
-}
-
-pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult {
-    use std::collections::VecDeque;
-    use highlighting::ScopeSelectors;
-
-    #[derive(Debug)]
-    struct SyntaxTestAssertionRange {
-        test_line_offset: usize,
-        line_number: usize,
-        begin_char: usize,
-        end_char: usize,
-        scope_selector: ScopeSelectors,
-        scope_selector_text: String,
-    }
-
-    fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque<SyntaxTestAssertionRange> {
-        use std::str::FromStr;
-        
-        let mut assertions = VecDeque::new();
-        let mut test_line_offset = 0;
-        //let mut test_line_len = 0;
-        let mut line_number = 0;
-        let mut offset = 0;
-        //let mut remainder = None;
-        for line in text.lines() {
-            line_number += 1;
-            let mut line_has_assertions = false;
-            
-            // if the test start token specified is on the line
-            if let Some(index) = line.find(token_start) {
-                let token_and_rest_of_line = line.split_at(index).1;
-
-                let rest_of_line = &token_and_rest_of_line[token_start.len()..];
-                if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) {
-                    let mut assertion_range = 0;
-                    while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') {
-                        assertion_range += 1;
-                    }
-                    let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range };
-
-                    let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text
-
-                    if let Some(token) = token_end { // if there is an end token defined in the test file header
-                        if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line
-                            selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token
-                        }
-                    }
-
-                    let assertion = SyntaxTestAssertionRange {
-                        test_line_offset: test_line_offset,
-                        line_number: line_number,
-                        begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 },
-                        end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 },
-
-                        // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -"
-                        // and they are sometimes in the syntax test as ^^^-comment, for example
-                        scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)),
-                        scope_selector_text: selector_text,
-                    };
-                    /*if assertion.end_char > test_line_len {
-                        remainder = Some(SyntaxTestAssertionRange {
-                            test_line_offset: test_line_offset + test_line_len,
-                            line_number: line_number,
-                            begin_char: assertion.begin_char - test_line_len,
-                            end_char: assertion.end_char - test_line_len,
-                            scope_selector: assertion.scope_selector.clone(),
-                            scope_selector_text: assertion.scope_selector_text.clone(),
-                        });
-                    }*/
-                    assertions.push_back(assertion);
-                    
-                    line_has_assertions = true;
-                }
-            }
-            if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text
-                test_line_offset = offset;
-                //test_line_len = line.len() + 1;
-            }
-            offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405
-        }
-        assertions
-    }
-    
-    #[derive(Debug)]
-    struct ScopedText {
-        scope: Vec<Scope>,
-        char_start: usize,
-        text_len: usize,
-    }
-    
-    #[derive(Debug)]
-    struct RangeTestResult {
-        column_begin: usize,
-        column_end: usize,
-        success: bool,
-        actual_scope: String,
-    }
-    
-    fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec<ScopedText>) -> Vec<RangeTestResult> {
-        use std::cmp::{min, max};
-        // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached
-        let mut results = Vec::new();
-        for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) {
-            let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice());
-            let result = RangeTestResult {
-                column_begin: max(scoped_text.char_start, assertion.begin_char),
-                column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char),
-                success: match_value.is_some(),
-                actual_scope: format!("{:?}", scoped_text.scope.as_slice()),
-            };
-            results.push(result);
-        }
-        results
-    }
-    
-    let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
-    //println!("{:?}", assertions);
-    use util::debug_print_ops;
-    
-    // iterate over the lines of the file, testing them
-    let mut state = ParseState::new(syntax);
-    let mut stack = ScopeStack::new();
-
-    let mut offset = 0;
-    let mut scopes_on_line_being_tested = Vec::new();
-    let mut line_number = 0;
-    let mut relevant_assertions = Vec::new();
-    
-    let mut assertion_failures: usize = 0;
-    let mut total_assertions: usize = 0;
-
-    for line_without_char in text.lines() {
-        let line = &(line_without_char.to_owned() + "\n");
-        line_number += 1;
-        
-        let eol_offset = offset + line.len();
-        
-        // parse the line
-        let ops = state.parse_line(&line);
-        // find assertions that relate to the current line
-        relevant_assertions.clear();
-        while let Some(assertion) = assertions.pop_front() {
-            let pos = assertion.test_line_offset + assertion.begin_char;
-            if pos >= offset && pos < eol_offset {
-                relevant_assertions.push(assertion);
-            } else {
-                assertions.push_front(assertion);
-                break;
-            }
-        }
-        if !relevant_assertions.is_empty() {
-            scopes_on_line_being_tested.clear();
-            if out_opts.debug {
-                println!("-- debugging line {} -- scope stack: {:?}", line_number, stack);
-                if ops.is_empty() && !line.is_empty() {
-                    println!("no operations for this line...");
-                } else {
-                    debug_print_ops(&line, &ops);
-                }
-            }
-        }
-        
-        {
-            let mut col: usize = 0;
-            for (s, op) in ScopeRegionIterator::new(&ops, &line) {
-                stack.apply(op);
-                if s.is_empty() { // in this case we don't care about blank tokens
-                    continue;
-                }
-                if !relevant_assertions.is_empty() {
-                    let len = s.chars().count();
-                    scopes_on_line_being_tested.push(
-                        ScopedText {
-                            char_start: col,
-                            text_len: len,
-                            scope: stack.as_slice().to_vec()
-                        }
-                    );
-                    col += len;
-                }
-            }
-        }
-        
-        for assertion in &relevant_assertions {
-            let results = process_assertions(&assertion, &scopes_on_line_being_tested);
-            
-            for result in results {
-                let length = result.column_end - result.column_begin;
-                total_assertions += length;
-                if !result.success {
-                    assertion_failures += length;
-                    let text: String = line.chars().skip(result.column_begin).take(length).collect();
-                    if !out_opts.summary {
-                        println!("  Assertion selector {:?} \
-                            from line {:?} failed against line {:?}, column range {:?}-{:?} \
-                            (with text {:?}) \
-                            has scope {:?}",
-                            &assertion.scope_selector_text.trim(),
-                            &assertion.line_number, line_number, result.column_begin, result.column_end,
-                            text,
-                            result.actual_scope,
-                        );
-                    }
-                }
-            }
-        }
-        
-        offset = eol_offset;
-    }
-    
-    let res = if assertion_failures > 0 {
-        SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions)
-    } else {
-        SyntaxTestFileResult::Success(total_assertions)
-    };
-    res
-}
-
 #[cfg(all(feature = "assets", any(feature = "dump-load", feature = "dump-load-rs")))]
 #[cfg(test)]
 mod tests {
diff --git a/src/lib.rs b/src/lib.rs
index c2e9860d..3b8c53a1 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -49,6 +49,8 @@ pub mod util;
 pub mod dumps;
 #[cfg(feature = "parsing")]
 pub mod easy;
+#[cfg(feature = "parsing")]
+pub mod syntax_tests;
 #[cfg(feature = "html")]
 pub mod html;
 #[cfg(feature = "html")]
diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
new file mode 100644
index 00000000..f8122387
--- /dev/null
+++ b/src/syntax_tests.rs
@@ -0,0 +1,263 @@
+//! API for running syntax tests.
+//! See http://www.sublimetext.com/docs/3/syntax.html#testing
+
+use parsing::{ScopeStack, ParseState, SyntaxDefinition, Scope};
+//use std::io::Write;
+use std::str::FromStr;
+use util::debug_print_ops;
+use easy::{ScopeRegionIterator};
+
+#[derive(Clone, Copy)]
+pub struct SyntaxTestOutputOptions {
+    pub time: bool,
+    pub debug: bool,
+    pub summary: bool,
+    //pub output: &'a Write,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SyntaxTestFileResult {
+    FailedAssertions(usize, usize),
+    Success(usize),
+}
+
+use std::collections::VecDeque;
+use highlighting::ScopeSelectors;
+
+#[derive(Debug)]
+struct SyntaxTestAssertionRange {
+    test_line_offset: usize,
+    line_number: usize,
+    begin_char: usize,
+    end_char: usize,
+    scope_selector: ScopeSelectors,
+    scope_selector_text: String,
+}
+
+fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque<SyntaxTestAssertionRange> {
+    let mut assertions = VecDeque::new();
+    let mut test_line_offset = 0;
+    //let mut test_line_len = 0;
+    let mut line_number = 0;
+    let mut offset = 0;
+    //let mut remainder = None;
+    for line in text.lines() {
+        line_number += 1;
+        let mut line_has_assertions = false;
+        
+        // if the test start token specified is on the line
+        if let Some(index) = line.find(token_start) {
+            let token_and_rest_of_line = line.split_at(index).1;
+
+            let rest_of_line = &token_and_rest_of_line[token_start.len()..];
+            if let Some(assertion_index) = rest_of_line.find("<-").or_else(|| rest_of_line.find('^')) {
+                let mut assertion_range = 0;
+                while rest_of_line.chars().nth(assertion_index + assertion_range) == Some('^') {
+                    assertion_range += 1;
+                }
+                let skip_assertion_chars = if assertion_range == 0 { 2 } else { assertion_range };
+
+                let mut selector_text : String = rest_of_line.chars().skip(assertion_index + skip_assertion_chars).collect(); // get the scope selector text
+
+                if let Some(token) = token_end { // if there is an end token defined in the test file header
+                    if let Some(end_token_pos) = selector_text.find(token) { // and there is an end token in the line
+                        selector_text = selector_text.chars().take(end_token_pos).collect(); // the scope selector text ends at the end token
+                    }
+                }
+
+                let assertion = SyntaxTestAssertionRange {
+                    test_line_offset: test_line_offset,
+                    line_number: line_number,
+                    begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 },
+                    end_char: index + if assertion_range > 0 { token_start.len() + assertion_index + assertion_range } else { 1 },
+
+                    // format the scope selector to include a space at the beginning, because, currently, ScopeSelector expects excludes to begin with " -"
+                    // and they are sometimes in the syntax test as ^^^-comment, for example
+                    scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)),
+                    scope_selector_text: selector_text,
+                };
+                /*if assertion.end_char > test_line_len {
+                    remainder = Some(SyntaxTestAssertionRange {
+                        test_line_offset: test_line_offset + test_line_len,
+                        line_number: line_number,
+                        begin_char: assertion.begin_char - test_line_len,
+                        end_char: assertion.end_char - test_line_len,
+                        scope_selector: assertion.scope_selector.clone(),
+                        scope_selector_text: assertion.scope_selector_text.clone(),
+                    });
+                }*/
+                assertions.push_back(assertion);
+                
+                line_has_assertions = true;
+            }
+        }
+        if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text
+            test_line_offset = offset;
+            //test_line_len = line.len() + 1;
+        }
+        offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405
+    }
+    assertions
+}
+
+/// Process the syntax test assertions in the given text, for the given syntax definition, using the test token(s) specified.
+/// `text` is the code containing syntax test assertions to be parsed and checked.
+/// `testtoken_start` is the token (normally a comment in the given syntax) that represents that assertions could be on the line.
+/// `testtoken_end` is an optional token that will be stripped from the line when retrieving the scope selector. Useful for syntaxes when the start token represents a block comment, to make the tests easier to construct.
+pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult {
+    #[derive(Debug)]
+    struct ScopedText {
+        scope: Vec<Scope>,
+        char_start: usize,
+        text_len: usize,
+    }
+    
+    #[derive(Debug)]
+    struct RangeTestResult {
+        column_begin: usize,
+        column_end: usize,
+        success: bool,
+        actual_scope: String,
+    }
+    
+    fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec<ScopedText>) -> Vec<RangeTestResult> {
+        use std::cmp::{min, max};
+        // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached
+        let mut results = Vec::new();
+        for scoped_text in test_against_line_scopes.iter().skip_while(|s|s.char_start + s.text_len <= assertion.begin_char).take_while(|s|s.char_start < assertion.end_char) {
+            let match_value = assertion.scope_selector.does_match(scoped_text.scope.as_slice());
+            let result = RangeTestResult {
+                column_begin: max(scoped_text.char_start, assertion.begin_char),
+                column_end: min(scoped_text.char_start + scoped_text.text_len, assertion.end_char),
+                success: match_value.is_some(),
+                actual_scope: format!("{:?}", scoped_text.scope.as_slice()),
+            };
+            results.push(result);
+        }
+        results
+    }
+    
+    let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
+    //println!("{:?}", assertions);
+    
+    // iterate over the lines of the file, testing them
+    let mut state = ParseState::new(syntax);
+    let mut stack = ScopeStack::new();
+
+    let mut offset = 0;
+    let mut scopes_on_line_being_tested = Vec::new();
+    let mut line_number = 0;
+    let mut relevant_assertions = Vec::new();
+    
+    let mut assertion_failures: usize = 0;
+    let mut total_assertions: usize = 0;
+
+    for line_without_char in text.lines() {
+        let line = &(line_without_char.to_owned() + "\n");
+        line_number += 1;
+        
+        let eol_offset = offset + line.len();
+        
+        // parse the line
+        let ops = state.parse_line(&line);
+        // find assertions that relate to the current line
+        relevant_assertions.clear();
+        while let Some(assertion) = assertions.pop_front() {
+            let pos = assertion.test_line_offset + assertion.begin_char;
+            if pos >= offset && pos < eol_offset {
+                relevant_assertions.push(assertion);
+            } else {
+                assertions.push_front(assertion);
+                break;
+            }
+        }
+        if !relevant_assertions.is_empty() {
+            scopes_on_line_being_tested.clear();
+            if out_opts.debug {
+                println!("-- debugging line {} -- scope stack: {:?}", line_number, stack);
+                if ops.is_empty() && !line.is_empty() {
+                    println!("no operations for this line...");
+                } else {
+                    debug_print_ops(&line, &ops);
+                }
+            }
+        }
+        
+        {
+            let mut col: usize = 0;
+            for (s, op) in ScopeRegionIterator::new(&ops, &line) {
+                stack.apply(op);
+                if s.is_empty() { // in this case we don't care about blank tokens
+                    continue;
+                }
+                if !relevant_assertions.is_empty() {
+                    let len = s.chars().count();
+                    scopes_on_line_being_tested.push(
+                        ScopedText {
+                            char_start: col,
+                            text_len: len,
+                            scope: stack.as_slice().to_vec()
+                        }
+                    );
+                    col += len;
+                }
+            }
+        }
+        
+        for assertion in &relevant_assertions {
+            let results = process_assertions(&assertion, &scopes_on_line_being_tested);
+            
+            for result in results {
+                let length = result.column_end - result.column_begin;
+                total_assertions += length;
+                if !result.success {
+                    assertion_failures += length;
+                    let text: String = line.chars().skip(result.column_begin).take(length).collect();
+                    if !out_opts.summary {
+                        println!("  Assertion selector {:?} \
+                            from line {:?} failed against line {:?}, column range {:?}-{:?} \
+                            (with text {:?}) \
+                            has scope {:?}",
+                            &assertion.scope_selector_text.trim(),
+                            &assertion.line_number, line_number, result.column_begin, result.column_end,
+                            text,
+                            result.actual_scope,
+                        );
+                    }
+                }
+            }
+        }
+        
+        offset = eol_offset;
+    }
+    
+    let res = if assertion_failures > 0 {
+        SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions)
+    } else {
+        SyntaxTestFileResult::Success(total_assertions)
+    };
+    res
+}
+
+// #[cfg(test)]
+// mod tests {
+    #[test]
+    fn can_find_test_assertions() {
+        let result = get_syntax_test_assertions(&"//", None,
+            "
+            hello world
+            // <- assertion1
+            // ^^ assertion2
+            
+            foobar
+            //    ^ - assertion3
+            ");
+        
+        assert_eq!(result.len(), 3);
+        assert_eq!(result[0].line_number, 3);
+        assert_eq!(result[1].line_number, 4);
+        assert_eq!(result[2].line_number, 7);
+        assert_eq!(result[0].test_line_offset, result[1].test_line_offset);
+        assert!(result[2].test_line_offset > result[0].test_line_offset);
+    }
+// }

From 2dee5fcd5177861eb7b6f02479e6cf0553333607 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Tue, 3 Jul 2018 09:30:17 +0300
Subject: [PATCH 03/12] add more tests for finding syntax test assertions

---
 src/syntax_tests.rs | 82 +++++++++++++++++++++++++++++++++------------
 1 file changed, 60 insertions(+), 22 deletions(-)

diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index f8122387..5f4c2a82 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -44,7 +44,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
     for line in text.lines() {
         line_number += 1;
         let mut line_has_assertions = false;
-        
+
         // if the test start token specified is on the line
         if let Some(index) = line.find(token_start) {
             let token_and_rest_of_line = line.split_at(index).1;
@@ -87,7 +87,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
                     });
                 }*/
                 assertions.push_back(assertion);
-                
+
                 line_has_assertions = true;
             }
         }
@@ -111,7 +111,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
         char_start: usize,
         text_len: usize,
     }
-    
+
     #[derive(Debug)]
     struct RangeTestResult {
         column_begin: usize,
@@ -119,7 +119,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
         success: bool,
         actual_scope: String,
     }
-    
+
     fn process_assertions(assertion: &SyntaxTestAssertionRange, test_against_line_scopes: &Vec<ScopedText>) -> Vec<RangeTestResult> {
         use std::cmp::{min, max};
         // find the scope at the specified start column, and start matching the selector through the rest of the tokens on the line from there until the end column is reached
@@ -136,10 +136,10 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
         }
         results
     }
-    
+
     let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
     //println!("{:?}", assertions);
-    
+
     // iterate over the lines of the file, testing them
     let mut state = ParseState::new(syntax);
     let mut stack = ScopeStack::new();
@@ -148,16 +148,16 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
     let mut scopes_on_line_being_tested = Vec::new();
     let mut line_number = 0;
     let mut relevant_assertions = Vec::new();
-    
+
     let mut assertion_failures: usize = 0;
     let mut total_assertions: usize = 0;
 
     for line_without_char in text.lines() {
         let line = &(line_without_char.to_owned() + "\n");
         line_number += 1;
-        
+
         let eol_offset = offset + line.len();
-        
+
         // parse the line
         let ops = state.parse_line(&line);
         // find assertions that relate to the current line
@@ -182,7 +182,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
                 }
             }
         }
-        
+
         {
             let mut col: usize = 0;
             for (s, op) in ScopeRegionIterator::new(&ops, &line) {
@@ -206,7 +206,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
         
         for assertion in &relevant_assertions {
             let results = process_assertions(&assertion, &scopes_on_line_being_tested);
-            
+
             for result in results {
                 let length = result.column_end - result.column_begin;
                 total_assertions += length;
@@ -227,7 +227,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
                 }
             }
         }
-        
+
         offset = eol_offset;
     }
     
@@ -243,21 +243,59 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
 // mod tests {
     #[test]
     fn can_find_test_assertions() {
-        let result = get_syntax_test_assertions(&"//", None,
-            "
-            hello world
-            // <- assertion1
-            // ^^ assertion2
-            
-            foobar
-            //    ^ - assertion3
-            ");
+        let text = "\
+            hello world\n\
+            // <- assertion1\n\
+            // ^^ assertion2\n\
+            \n\
+            foobar\n\
+            //    ^ - assertion3\n\
+            ";
+        let result = get_syntax_test_assertions(&"//", None, &text);
+
+        assert_eq!(result.len(), 3);
+        assert_eq!(result[0].line_number, 2);
+        assert_eq!(result[1].line_number, 3);
+        assert_eq!(result[2].line_number, 6);
+        assert_eq!(result[0].test_line_offset, result[1].test_line_offset);
+        assert_eq!(result[2].test_line_offset, text.find("foobar").unwrap());
+        assert_eq!(result[0].scope_selector_text, " assertion1");
+        assert_eq!(result[1].scope_selector_text, " assertion2");
+        assert_eq!(result[2].scope_selector_text, " - assertion3");
+        assert_eq!(result[0].begin_char, 0);
+        assert_eq!(result[0].end_char, 1);
+        assert_eq!(result[1].begin_char, 3);
+        assert_eq!(result[1].end_char, 5);
+        assert_eq!(result[2].begin_char, 6);
+        assert_eq!(result[2].end_char, 7);
+    }
+
+    #[test]
+    fn can_find_test_assertions_with_end_tokens() {
+        let text = "
+hello world
+ <!-- <- assertion1 -->
+<!--  ^^assertion2
+
+foobar
+<!-- ^ - assertion3 -->
+";
+        let result = get_syntax_test_assertions(&"<!--", Some(&"-->"), &text);
         
         assert_eq!(result.len(), 3);
         assert_eq!(result[0].line_number, 3);
         assert_eq!(result[1].line_number, 4);
         assert_eq!(result[2].line_number, 7);
         assert_eq!(result[0].test_line_offset, result[1].test_line_offset);
-        assert!(result[2].test_line_offset > result[0].test_line_offset);
+        assert_eq!(result[2].test_line_offset, text.find("foobar").unwrap());
+        assert_eq!(result[0].scope_selector_text, " assertion1 ");
+        assert_eq!(result[1].scope_selector_text, "assertion2");
+        assert_eq!(result[2].scope_selector_text, " - assertion3 ");
+        assert_eq!(result[0].begin_char, 1);
+        assert_eq!(result[0].end_char, 2);
+        assert_eq!(result[1].begin_char, 6);
+        assert_eq!(result[1].end_char, 8);
+        assert_eq!(result[2].begin_char, 5);
+        assert_eq!(result[2].end_char, 6);
     }
 // }

From 91af5e9aa2422f3763b455aa992f3050b46e5b9f Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Tue, 3 Jul 2018 15:04:20 +0300
Subject: [PATCH 04/12] add tests to syntest to prove it works with assertions
 than span lines

---
 src/syntax_tests.rs | 77 +++++++++++++++++++++++++++++++++++++--------
 1 file changed, 64 insertions(+), 13 deletions(-)

diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 5f4c2a82..9b47bdb2 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -37,10 +37,10 @@ struct SyntaxTestAssertionRange {
 fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque<SyntaxTestAssertionRange> {
     let mut assertions = VecDeque::new();
     let mut test_line_offset = 0;
-    //let mut test_line_len = 0;
+    let mut test_line_len = 0;
     let mut line_number = 0;
     let mut offset = 0;
-    //let mut remainder = None;
+
     for line in text.lines() {
         line_number += 1;
         let mut line_has_assertions = false;
@@ -65,7 +65,7 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
                     }
                 }
 
-                let assertion = SyntaxTestAssertionRange {
+                let mut assertion = SyntaxTestAssertionRange {
                     test_line_offset: test_line_offset,
                     line_number: line_number,
                     begin_char: index + if assertion_range > 0 { token_start.len() + assertion_index } else { 0 },
@@ -76,24 +76,30 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
                     scope_selector: ScopeSelectors::from_str(&format!(" {}", &selector_text)).expect(&format!("Scope selector invalid on line {}", line_number)),
                     scope_selector_text: selector_text,
                 };
-                /*if assertion.end_char > test_line_len {
-                    remainder = Some(SyntaxTestAssertionRange {
+                // if the assertion spans over the line being tested
+                if assertion.end_char > test_line_len {
+                    // calculate where on the next line the assertions will occur
+                    let remainder = SyntaxTestAssertionRange {
                         test_line_offset: test_line_offset + test_line_len,
                         line_number: line_number,
-                        begin_char: assertion.begin_char - test_line_len,
+                        begin_char: 0,
                         end_char: assertion.end_char - test_line_len,
                         scope_selector: assertion.scope_selector.clone(),
                         scope_selector_text: assertion.scope_selector_text.clone(),
-                    });
-                }*/
-                assertions.push_back(assertion);
+                    };
+                    assertion.end_char = test_line_len;
+                    assertions.push_back(assertion);
+                    assertions.push_back(remainder);
+                } else {
+                    assertions.push_back(assertion);
+                }
 
                 line_has_assertions = true;
             }
         }
         if !line_has_assertions { // ST seems to ignore lines that have assertions when calculating which line the assertion tests against, regardless of whether they contain any other text
             test_line_offset = offset;
-            //test_line_len = line.len() + 1;
+            test_line_len = line.len() + 1;
         }
         offset += line.len() + 1; // the +1 is for the `\n`. TODO: maybe better to loop over the lines including the newline chars, using https://stackoverflow.com/a/40457615/4473405
     }
@@ -203,7 +209,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
                 }
             }
         }
-        
+
         for assertion in &relevant_assertions {
             let results = process_assertions(&assertion, &scopes_on_line_being_tested);
 
@@ -230,7 +236,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
 
         offset = eol_offset;
     }
-    
+
     let res = if assertion_failures > 0 {
         SyntaxTestFileResult::FailedAssertions(assertion_failures, total_assertions)
     } else {
@@ -281,7 +287,7 @@ foobar
 <!-- ^ - assertion3 -->
 ";
         let result = get_syntax_test_assertions(&"<!--", Some(&"-->"), &text);
-        
+
         assert_eq!(result.len(), 3);
         assert_eq!(result[0].line_number, 3);
         assert_eq!(result[1].line_number, 4);
@@ -298,4 +304,49 @@ foobar
         assert_eq!(result[2].begin_char, 5);
         assert_eq!(result[2].end_char, 6);
     }
+
+    #[test]
+    fn can_find_test_assertions_that_spans_lines() {
+        let text = "
+hello world
+<!--  ^^^^^^^^^ assertion1
+<!--    ^^^^^^^ assertion2 -->
+foobar
+<!-- ^^^ -assertion3-->
+";
+        let result = get_syntax_test_assertions(&"<!--", Some(&"-->"), &text);
+        println!("{:?}", result);
+
+        assert_eq!(result.len(), 6);
+        assert_eq!(result[0].line_number, 3);
+        assert_eq!(result[1].line_number, 3);
+        assert_eq!(result[2].line_number, 4);
+        assert_eq!(result[3].line_number, 4);
+        assert_eq!(result[4].line_number, 6);
+        assert_eq!(result[5].line_number, 6);
+        assert_eq!(result[0].scope_selector_text, " assertion1");
+        assert_eq!(result[1].scope_selector_text, " assertion1");
+        assert_eq!(result[2].scope_selector_text, " assertion2 ");
+        assert_eq!(result[3].scope_selector_text, " assertion2 ");
+        assert_eq!(result[4].scope_selector_text, " -assertion3");
+        assert_eq!(result[5].scope_selector_text, " -assertion3");
+        assert_eq!(result[0].begin_char, 6);
+        assert_eq!(result[0].end_char, 12);
+        assert_eq!(result[0].test_line_offset, 1);
+        assert_eq!(result[1].begin_char, 0);
+        assert_eq!(result[1].end_char, 3);
+        assert_eq!(result[1].test_line_offset, "\nhello world\n".len());
+
+        assert_eq!(result[2].begin_char, 8);
+        assert_eq!(result[2].end_char, 12);
+        assert_eq!(result[2].test_line_offset, 1);
+        assert_eq!(result[3].begin_char, 0);
+        assert_eq!(result[3].end_char, 3);
+        assert_eq!(result[3].test_line_offset, "\nhello world\n".len());
+
+        assert_eq!(result[4].begin_char, 5);
+        assert_eq!(result[4].end_char, 7);
+        assert_eq!(result[5].begin_char, 0);
+        assert_eq!(result[5].end_char, 1);
+    }
 // }

From 67c0e05367f261663d3ebfa0ddfceb5213d5a205 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Tue, 3 Jul 2018 15:10:41 +0300
Subject: [PATCH 05/12] fix syntest seemingly skipping some files with crlf
 line endings

---
 examples/syntest.rs | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/examples/syntest.rs b/examples/syntest.rs
index a5b9b84b..97224f43 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -52,6 +52,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) ->
     if reader.read_line(&mut header_line).unwrap() == 0 {
         return Err(SyntaxTestHeaderError::MalformedHeader);
     }
+    header_line = header_line.replace("\r", &"");
 
     // parse the syntax test header in the first line of the file
     let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line);
@@ -165,6 +166,7 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions)
         }
         if exit_code != 2 { // leave exit code 2 if there was an error
             if let Err(_) = result { // set exit code 2 if there was an error
+                println!("{:?}", result);
                 exit_code = 2;
             } else if let Ok(ok) = result {
                 if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {

From f485533b23ad407db857bfabc99c8eab31c18b20 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Wed, 4 Jul 2018 14:12:20 +0300
Subject: [PATCH 06/12] add failfast mode to syntest, add more comments

---
 examples/syntest.rs |  8 ++++++++
 src/syntax_tests.rs | 21 ++++++++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/examples/syntest.rs b/examples/syntest.rs
index 97224f43..4b9121d2 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -93,6 +93,7 @@ fn main() {
     opts.optflag("d", "debug", "Show parsing results for each test line");
     opts.optflag("t", "time", "Time execution as a more broad-ranging benchmark");
     opts.optflag("s", "summary", "Print only summary of test failures");
+    opts.optflag("f", "failfast", "Stop at first failure");
 
     let matches = match opts.parse(&args[1..]) {
         Ok(m) => { m }
@@ -130,6 +131,7 @@ fn main() {
         debug: matches.opt_present("debug"),
         time: matches.opt_present("time"),
         summary: matches.opt_present("summary"),
+        failfast: matches.opt_present("failfast"),
     };
 
     let exit_code = recursive_walk(&ss, &tests_path, out_opts);
@@ -168,9 +170,15 @@ fn recursive_walk(ss: &SyntaxSet, path: &str, out_opts: SyntaxTestOutputOptions)
             if let Err(_) = result { // set exit code 2 if there was an error
                 println!("{:?}", result);
                 exit_code = 2;
+                if out_opts.failfast {
+                    break;
+                }
             } else if let Ok(ok) = result {
                 if let SyntaxTestFileResult::FailedAssertions(_, _) = ok {
                     exit_code = 1; // otherwise, if there were failures, exit with code 1
+                    if out_opts.failfast {
+                        break;
+                    }
                 }
             }
         }
diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 9b47bdb2..c29cb1f6 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -12,6 +12,7 @@ pub struct SyntaxTestOutputOptions {
     pub time: bool,
     pub debug: bool,
     pub summary: bool,
+    pub failfast: bool,
     //pub output: &'a Write,
 }
 
@@ -107,10 +108,13 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
 }
 
 /// Process the syntax test assertions in the given text, for the given syntax definition, using the test token(s) specified.
+/// It works by finding all the syntax test assertions, then parsing the text line by line. If the line has some assertions against it, those are checked.
+/// Assertions are counted according to their status - succeeded or failed. Failures are also logged to stdout, depending on the output options.
+/// When there are no assertions left to check, it returns those counts.
 /// `text` is the code containing syntax test assertions to be parsed and checked.
 /// `testtoken_start` is the token (normally a comment in the given syntax) that represents that assertions could be on the line.
 /// `testtoken_end` is an optional token that will be stripped from the line when retrieving the scope selector. Useful for syntaxes when the start token represents a block comment, to make the tests easier to construct.
-pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult {
+pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, testtoken_start: &str, testtoken_end: Option<&str>, out_opts: &SyntaxTestOutputOptions) -> SyntaxTestFileResult {
     #[derive(Debug)]
     struct ScopedText {
         scope: Vec<Scope>,
@@ -166,7 +170,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
 
         // parse the line
         let ops = state.parse_line(&line);
-        // find assertions that relate to the current line
+        // find all the assertions that relate to the current line
         relevant_assertions.clear();
         while let Some(assertion) = assertions.pop_front() {
             let pos = assertion.test_line_offset + assertion.begin_char;
@@ -178,6 +182,8 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
             }
         }
         if !relevant_assertions.is_empty() {
+            // if there are assertions for the line, show the operations for debugging purposes if specified in the output options
+            // (if there are no assertions, or the line contains assertions, debugging it would probably just add noise)
             scopes_on_line_being_tested.clear();
             if out_opts.debug {
                 println!("-- debugging line {} -- scope stack: {:?}", line_number, stack);
@@ -196,6 +202,7 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
                 if s.is_empty() { // in this case we don't care about blank tokens
                     continue;
                 }
+                // if there are assertions against this line, store the scopes for comparison with the assertions
                 if !relevant_assertions.is_empty() {
                     let len = s.chars().count();
                     scopes_on_line_being_tested.push(
@@ -218,8 +225,8 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
                 total_assertions += length;
                 if !result.success {
                     assertion_failures += length;
-                    let text: String = line.chars().skip(result.column_begin).take(length).collect();
                     if !out_opts.summary {
+                        let text: String = line.chars().skip(result.column_begin).take(length).collect();
                         println!("  Assertion selector {:?} \
                             from line {:?} failed against line {:?}, column range {:?}-{:?} \
                             (with text {:?}) \
@@ -235,6 +242,14 @@ pub fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text: &str, tes
         }
 
         offset = eol_offset;
+        
+        // no point continuing to parse the file if there are no syntax test assertions left
+        // (unless we want to prove that no panics etc. occur while parsing the rest of the file ofc...)
+        if assertions.is_empty() || (assertion_failures > 0 && out_opts.failfast) {
+            // NOTE: the total counts only really show how many assertions were checked when failing fast
+            //       - they are not accurate total counts
+            break;
+        }
     }
 
     let res = if assertion_failures > 0 {

From 6cba0699e5813ffbbea1f7f56f43833f5db50cc9 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+gitkraken@gmail.com>
Date: Thu, 5 Jul 2018 09:14:40 +0300
Subject: [PATCH 07/12] switch syntax tests back to a normal Vec instead of a
 VecDeque

---
 src/syntax_tests.rs | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index c29cb1f6..528c1feb 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -22,7 +22,6 @@ pub enum SyntaxTestFileResult {
     Success(usize),
 }
 
-use std::collections::VecDeque;
 use highlighting::ScopeSelectors;
 
 #[derive(Debug)]
@@ -35,8 +34,8 @@ struct SyntaxTestAssertionRange {
     scope_selector_text: String,
 }
 
-fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> VecDeque<SyntaxTestAssertionRange> {
-    let mut assertions = VecDeque::new();
+fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec<SyntaxTestAssertionRange> {
+    let mut assertions = Vec::new();
     let mut test_line_offset = 0;
     let mut test_line_len = 0;
     let mut line_number = 0;
@@ -89,10 +88,10 @@ fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text:
                         scope_selector_text: assertion.scope_selector_text.clone(),
                     };
                     assertion.end_char = test_line_len;
-                    assertions.push_back(assertion);
-                    assertions.push_back(remainder);
+                    assertions.push(assertion);
+                    assertions.push(remainder);
                 } else {
-                    assertions.push_back(assertion);
+                    assertions.push(assertion);
                 }
 
                 line_has_assertions = true;
@@ -147,7 +146,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text
         results
     }
 
-    let mut assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
+    let assertions = get_syntax_test_assertions(testtoken_start, testtoken_end, &text);
     //println!("{:?}", assertions);
 
     // iterate over the lines of the file, testing them
@@ -158,6 +157,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text
     let mut scopes_on_line_being_tested = Vec::new();
     let mut line_number = 0;
     let mut relevant_assertions = Vec::new();
+    let mut assertion_index = 0;
 
     let mut assertion_failures: usize = 0;
     let mut total_assertions: usize = 0;
@@ -172,12 +172,13 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text
         let ops = state.parse_line(&line);
         // find all the assertions that relate to the current line
         relevant_assertions.clear();
-        while let Some(assertion) = assertions.pop_front() {
+        while assertion_index < assertions.len() {
+            let assertion = &assertions[assertion_index];
             let pos = assertion.test_line_offset + assertion.begin_char;
             if pos >= offset && pos < eol_offset {
                 relevant_assertions.push(assertion);
+                assertion_index += 1;
             } else {
-                assertions.push_front(assertion);
                 break;
             }
         }
@@ -245,7 +246,7 @@ pub/*(crate)*/ fn process_syntax_test_assertions(syntax: &SyntaxDefinition, text
         
         // no point continuing to parse the file if there are no syntax test assertions left
         // (unless we want to prove that no panics etc. occur while parsing the rest of the file ofc...)
-        if assertions.is_empty() || (assertion_failures > 0 && out_opts.failfast) {
+        if assertion_index == assertions.len() || (assertion_failures > 0 && out_opts.failfast) {
             // NOTE: the total counts only really show how many assertions were checked when failing fast
             //       - they are not accurate total counts
             break;

From 1d754cf9a0620ea7673e7257a16170f430286f03 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Mon, 15 Oct 2018 14:46:00 +0300
Subject: [PATCH 08/12] make the get_syntax_test_assertions method public

this will allow other functionality to be developed that can use the same syntax test implementation and show the test results in a more friendly manner etc.
---
 src/syntax_tests.rs | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 8a5a14d1..a47e0460 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -5,7 +5,8 @@ use parsing::{ScopeStack, ParseState, SyntaxReference, SyntaxSet, Scope};
 //use std::io::Write;
 use std::str::FromStr;
 use util::debug_print_ops;
-use easy::{ScopeRegionIterator};
+use easy::ScopeRegionIterator;
+use highlighting::ScopeSelectors;
 
 #[derive(Clone, Copy)]
 pub struct SyntaxTestOutputOptions {
@@ -22,10 +23,8 @@ pub enum SyntaxTestFileResult {
     Success(usize),
 }
 
-use highlighting::ScopeSelectors;
-
 #[derive(Debug)]
-struct SyntaxTestAssertionRange {
+pub struct SyntaxTestAssertionRange {
     test_line_offset: usize,
     line_number: usize,
     begin_char: usize,
@@ -34,7 +33,7 @@ struct SyntaxTestAssertionRange {
     scope_selector_text: String,
 }
 
-fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec<SyntaxTestAssertionRange> {
+pub fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec<SyntaxTestAssertionRange> {
     let mut assertions = Vec::new();
     let mut test_line_offset = 0;
     let mut test_line_len = 0;

From b39cd4e6be1ba08f985ac9a40966312c7dff189d Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Sun, 21 Oct 2018 12:06:25 +0300
Subject: [PATCH 09/12] add doc comment for get_syntax_test_assertions

---
 src/syntax_tests.rs | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index a47e0460..28771436 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -25,14 +25,18 @@ pub enum SyntaxTestFileResult {
 
 #[derive(Debug)]
 pub struct SyntaxTestAssertionRange {
-    test_line_offset: usize,
-    line_number: usize,
-    begin_char: usize,
-    end_char: usize,
-    scope_selector: ScopeSelectors,
-    scope_selector_text: String,
+    pub test_line_offset: usize,
+    pub line_number: usize,
+    pub begin_char: usize,
+    pub end_char: usize,
+    pub scope_selector: ScopeSelectors,
+    pub scope_selector_text: String,
 }
 
+/// Given a start token, option end token and text, parse the syntax tests in the text
+/// that follow the format described at http://www.sublimetext.com/docs/3/syntax.html#testing
+/// and return the scope selector assertions found, so that when the text is parsed,
+/// the assertions can be checked
 pub fn get_syntax_test_assertions(token_start: &str, token_end: Option<&str>, text: &str) -> Vec<SyntaxTestAssertionRange> {
     let mut assertions = Vec::new();
     let mut test_line_offset = 0;

From 6edd291e13729fd1f33281aec02fb7d94049b9e5 Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Sun, 21 Oct 2018 13:52:42 +0300
Subject: [PATCH 10/12] make syntax test header parsing public also

---
 Cargo.toml          |  4 +--
 examples/syntest.rs | 21 ++-------------
 src/syntax_tests.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 68 insertions(+), 21 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index d998980d..fd481033 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,6 +18,7 @@ yaml-rust = { version = "0.4", optional = true }
 onig = { version = "4.1", optional = true }
 walkdir = "2.0"
 regex-syntax = { version = "0.6", optional = true }
+regex = { version = "1.0", optional = true }
 lazy_static = "1.0"
 lazycell = "1.0"
 bitflags = "1.0"
@@ -32,7 +33,6 @@ serde_json = "1.0"
 [dev-dependencies]
 criterion = "0.2"
 rayon = "1.0.0"
-regex = "1.0"
 getopts = "0.2"
 pretty_assertions = "0.5.0"
 
@@ -51,7 +51,7 @@ dump-create = ["flate2/default", "bincode"]
 # Pure Rust dump creation, worse compressor so produces larger dumps than dump-create
 dump-create-rs = ["flate2/rust_backend", "bincode"]
 
-parsing = ["onig", "regex-syntax", "fnv"]
+parsing = ["onig", "regex-syntax", "fnv", "regex"]
 # The `assets` feature enables inclusion of the default theme and syntax packages.
 # For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set.
 assets = []
diff --git a/examples/syntest.rs b/examples/syntest.rs
index 5d51f466..468164c3 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -9,13 +9,10 @@
 // cargo run --example syntest testdata/Packages/JavaScript/syntax_test_json.json testdata/Packages/JavaScript/
 extern crate syntect;
 extern crate walkdir;
-#[macro_use]
-extern crate lazy_static;
-extern crate regex;
 extern crate getopts;
 
 use syntect::parsing::{SyntaxSet, SyntaxSetBuilder};
-use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions};
+use syntect::syntax_tests::{SyntaxTestFileResult, SyntaxTestOutputOptions, process_syntax_test_assertions, parse_syntax_test_header_line, SyntaxTestHeader};
 
 use std::path::Path;
 use std::io::prelude::*;
@@ -24,7 +21,6 @@ use std::fs::File;
 use std::time::Instant;
 
 use getopts::Options;
-use regex::Regex;
 use walkdir::{DirEntry, WalkDir};
 
 #[derive(Debug, Clone, PartialEq, Eq)]
@@ -33,14 +29,6 @@ pub enum SyntaxTestHeaderError {
     SyntaxDefinitionNotFound,
 }
 
-lazy_static! {
-    pub static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm)
-            ^(?P<testtoken_start>\s*\S+)
-            \s+SYNTAX\sTEST\s+
-            "(?P<syntax_file>[^"]+)"
-            \s*(?P<testtoken_end>\S+)?$
-        "#).unwrap();
-}
 
 fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) -> Result<SyntaxTestFileResult, SyntaxTestHeaderError> {
     let f = File::open(path).unwrap();
@@ -54,12 +42,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) ->
     header_line = header_line.replace("\r", &"");
 
     // parse the syntax test header in the first line of the file
-    let search_result = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line);
-    let captures = search_result.ok_or(SyntaxTestHeaderError::MalformedHeader)?;
-
-    let testtoken_start = captures.name("testtoken_start").unwrap().as_str();
-    let testtoken_end = captures.name("testtoken_end").map_or(None, |c|Some(c.as_str()));
-    let syntax_file = captures.name("syntax_file").unwrap().as_str();
+    let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?;
 
     // find the relevant syntax definition to parse the file with - case is important!
     if !out_opts.summary {
diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 28771436..8e87ac81 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -8,6 +8,12 @@ use util::debug_print_ops;
 use easy::ScopeRegionIterator;
 use highlighting::ScopeSelectors;
 
+// #[macro_use]
+// extern crate lazy_static;
+extern crate regex;
+
+use self::regex::Regex;
+
 #[derive(Clone, Copy)]
 pub struct SyntaxTestOutputOptions {
     pub time: bool,
@@ -33,6 +39,32 @@ pub struct SyntaxTestAssertionRange {
     pub scope_selector_text: String,
 }
 
+lazy_static! {
+    static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm)
+            ^(?P<testtoken_start>\s*\S+)
+            \s+SYNTAX\sTEST\s+
+            "(?P<syntax_file>[^"]+)"
+            \s*(?P<testtoken_end>\S+)?\r?$
+        "#).unwrap();
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SyntaxTestHeader<'a> {
+    pub testtoken_start: &'a str,
+    pub testtoken_end: Option<&'a str>,
+    pub syntax_file: &'a str,
+}
+
+pub fn parse_syntax_test_header_line(header_line: &str) -> Option<SyntaxTestHeader> {
+    let captures = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line/*.replace("\r", &"")*/)?;
+    
+    Some(SyntaxTestHeader {
+        testtoken_start: captures.name("testtoken_start").unwrap().as_str(),
+        testtoken_end: captures.name("testtoken_end").map_or(None, |c|Some(c.as_str())),
+        syntax_file: captures.name("syntax_file").unwrap().as_str(),
+    })
+}
+
 /// Given a start token, option end token and text, parse the syntax tests in the text
 /// that follow the format described at http://www.sublimetext.com/docs/3/syntax.html#testing
 /// and return the scope selector assertions found, so that when the text is parsed,
@@ -368,4 +400,36 @@ foobar
         assert_eq!(result[5].begin_char, 0);
         assert_eq!(result[5].end_char, 1);
     }
+
+    #[test]
+    fn can_parse_syntax_test_header_with_end_token() {
+        let header = parse_syntax_test_header_line(&"<!-- SYNTAX TEST \"XML.sublime-syntax\" -->").unwrap();
+        assert_eq!(&header.testtoken_start, &"<!--");
+        assert_eq!(&header.testtoken_end.unwrap(), &"-->");
+        assert_eq!(&header.syntax_file, &"XML.sublime-syntax");
+    }
+
+    #[test]
+    fn can_parse_syntax_test_header_with_end_token_and_carriage_return() {
+        let header = parse_syntax_test_header_line(&"<!-- SYNTAX TEST \"XML.sublime-syntax\" -->\r\n").unwrap();
+        assert_eq!(&header.testtoken_start, &"<!--");
+        assert_eq!(&header.testtoken_end.unwrap(), &"-->");
+        assert_eq!(&header.syntax_file, &"XML.sublime-syntax");
+    }
+
+    #[test]
+    fn can_parse_syntax_test_header_with_no_end_token() {
+        let header = parse_syntax_test_header_line(&"// SYNTAX TEST \"Packages/Example/Example.sublime-syntax\"\n").unwrap();
+        assert_eq!(&header.testtoken_start, &"//");
+        assert!(!header.testtoken_end.is_some());
+        assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax");
+    }
+
+    #[test]
+    fn can_parse_syntax_test_header_with_no_end_token_and_carriage_return() {
+        let header = parse_syntax_test_header_line(&"// SYNTAX TEST \"Packages/Example/Example.sublime-syntax\"\r").unwrap();
+        assert_eq!(&header.testtoken_start, &"//");
+        assert!(header.testtoken_end.is_none());
+        assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax");
+    }
 // }

From 77f0de12130a20b01d471d1380932b59324b2c6d Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Mon, 22 Oct 2018 09:31:29 +0300
Subject: [PATCH 11/12] remove regex dependency

---
 Cargo.toml          |  3 +--
 src/syntax_tests.rs | 39 ++++++++++++++++-----------------------
 2 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index fd481033..0e1c20f2 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,6 @@ yaml-rust = { version = "0.4", optional = true }
 onig = { version = "4.1", optional = true }
 walkdir = "2.0"
 regex-syntax = { version = "0.6", optional = true }
-regex = { version = "1.0", optional = true }
 lazy_static = "1.0"
 lazycell = "1.0"
 bitflags = "1.0"
@@ -51,7 +50,7 @@ dump-create = ["flate2/default", "bincode"]
 # Pure Rust dump creation, worse compressor so produces larger dumps than dump-create
 dump-create-rs = ["flate2/rust_backend", "bincode"]
 
-parsing = ["onig", "regex-syntax", "fnv", "regex"]
+parsing = ["onig", "regex-syntax", "fnv"]
 # The `assets` feature enables inclusion of the default theme and syntax packages.
 # For `assets` to do anything, it requires one of `dump-load-rs` or `dump-load` to be set.
 assets = []
diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 8e87ac81..81671d51 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -8,12 +8,6 @@ use util::debug_print_ops;
 use easy::ScopeRegionIterator;
 use highlighting::ScopeSelectors;
 
-// #[macro_use]
-// extern crate lazy_static;
-extern crate regex;
-
-use self::regex::Regex;
-
 #[derive(Clone, Copy)]
 pub struct SyntaxTestOutputOptions {
     pub time: bool,
@@ -39,15 +33,6 @@ pub struct SyntaxTestAssertionRange {
     pub scope_selector_text: String,
 }
 
-lazy_static! {
-    static ref SYNTAX_TEST_HEADER_PATTERN: Regex = Regex::new(r#"(?xm)
-            ^(?P<testtoken_start>\s*\S+)
-            \s+SYNTAX\sTEST\s+
-            "(?P<syntax_file>[^"]+)"
-            \s*(?P<testtoken_end>\S+)?\r?$
-        "#).unwrap();
-}
-
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct SyntaxTestHeader<'a> {
     pub testtoken_start: &'a str,
@@ -55,14 +40,22 @@ pub struct SyntaxTestHeader<'a> {
     pub syntax_file: &'a str,
 }
 
-pub fn parse_syntax_test_header_line(header_line: &str) -> Option<SyntaxTestHeader> {
-    let captures = SYNTAX_TEST_HEADER_PATTERN.captures(&header_line/*.replace("\r", &"")*/)?;
-    
-    Some(SyntaxTestHeader {
-        testtoken_start: captures.name("testtoken_start").unwrap().as_str(),
-        testtoken_end: captures.name("testtoken_end").map_or(None, |c|Some(c.as_str())),
-        syntax_file: captures.name("syntax_file").unwrap().as_str(),
-    })
+pub fn parse_syntax_test_header_line(header_line: &str) -> Option<SyntaxTestHeader> { // TODO: use a "impl<'a> From<&'a str> for SyntaxTestHeader<'a>" instead?
+    if let Some(pos) = &header_line.find(&" SYNTAX TEST \"") {
+        let filename_part = &header_line[*pos + " SYNTAX TEST \"".len()..];
+        if let Some(close_pos) = filename_part.find(&"\"") {
+            let end_token = filename_part[close_pos + 1..].trim();
+            Some(SyntaxTestHeader {
+                testtoken_start: &header_line[0..*pos],
+                testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) },
+                syntax_file: &filename_part[0..close_pos],
+            })
+        } else {
+            None
+        }
+    } else {
+        None
+    }
 }
 
 /// Given a start token, option end token and text, parse the syntax tests in the text

From 59ddb378947c1b07e25a5915b04c45cfa16f653d Mon Sep 17 00:00:00 2001
From: Keith Hall <kingkeith+github@gmail.com>
Date: Tue, 14 Apr 2020 12:17:16 +0300
Subject: [PATCH 12/12] handle new syntax test header format which caters for
 reindentation

---
 examples/syntest.rs |  2 +-
 src/syntax_tests.rs | 42 +++++++++++++++++++++++++++++-------------
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/examples/syntest.rs b/examples/syntest.rs
index 468164c3..c120ea99 100644
--- a/examples/syntest.rs
+++ b/examples/syntest.rs
@@ -42,7 +42,7 @@ fn test_file(ss: &SyntaxSet, path: &Path, out_opts: SyntaxTestOutputOptions) ->
     header_line = header_line.replace("\r", &"");
 
     // parse the syntax test header in the first line of the file
-    let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?;
+    let SyntaxTestHeader { testtoken_start, testtoken_end, syntax_file, reindent_text: _ } = parse_syntax_test_header_line(&header_line).ok_or(SyntaxTestHeaderError::MalformedHeader)?;
 
     // find the relevant syntax definition to parse the file with - case is important!
     if !out_opts.summary {
diff --git a/src/syntax_tests.rs b/src/syntax_tests.rs
index 87789c68..1a32ee65 100644
--- a/src/syntax_tests.rs
+++ b/src/syntax_tests.rs
@@ -38,24 +38,27 @@ pub struct SyntaxTestHeader<'a> {
     pub testtoken_start: &'a str,
     pub testtoken_end: Option<&'a str>,
     pub syntax_file: &'a str,
+    pub reindent_text: Option<&'a str>,
 }
 
 pub fn parse_syntax_test_header_line(header_line: &str) -> Option<SyntaxTestHeader> { // TODO: use a "impl<'a> From<&'a str> for SyntaxTestHeader<'a>" instead?
-    if let Some(pos) = &header_line.find(&" SYNTAX TEST \"") {
-        let filename_part = &header_line[*pos + " SYNTAX TEST \"".len()..];
-        if let Some(close_pos) = filename_part.find(&"\"") {
-            let end_token = filename_part[close_pos + 1..].trim();
-            Some(SyntaxTestHeader {
-                testtoken_start: &header_line[0..*pos],
-                testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) },
-                syntax_file: &filename_part[0..close_pos],
-            })
-        } else {
-            None
+    if let Some(pos) = &header_line.find(&" SYNTAX TEST ") {
+        let after_text_pos = *pos + &" SYNTAX TEST ".len();
+        if let Some(quote_start_pos) = &header_line[after_text_pos..].find("\"") {
+            let reindent_text = if *quote_start_pos == 0 { None } else { Some(header_line[after_text_pos..after_text_pos + *quote_start_pos].trim()) };
+            let filename_part = &header_line[after_text_pos + *quote_start_pos + 1..];
+            if let Some(close_pos) = filename_part.find(&"\"") {
+                let end_token = filename_part[close_pos + 1..].trim();
+                return Some(SyntaxTestHeader {
+                    testtoken_start: &header_line[0..*pos],
+                    testtoken_end: if end_token.len() == 0 { None } else { Some(end_token) },
+                    syntax_file: &filename_part[0..close_pos],
+                    reindent_text: reindent_text,
+                });
+            }
         }
-    } else {
-        None
     }
+    None
 }
 
 /// Given a start token, option end token and text, parse the syntax tests in the text
@@ -400,6 +403,7 @@ foobar
         assert_eq!(&header.testtoken_start, &"<!--");
         assert_eq!(&header.testtoken_end.unwrap(), &"-->");
         assert_eq!(&header.syntax_file, &"XML.sublime-syntax");
+        assert!(&header.reindent_text.is_none());
     }
 
     #[test]
@@ -408,6 +412,7 @@ foobar
         assert_eq!(&header.testtoken_start, &"<!--");
         assert_eq!(&header.testtoken_end.unwrap(), &"-->");
         assert_eq!(&header.syntax_file, &"XML.sublime-syntax");
+        assert!(&header.reindent_text.is_none());
     }
 
     #[test]
@@ -416,6 +421,7 @@ foobar
         assert_eq!(&header.testtoken_start, &"//");
         assert!(!header.testtoken_end.is_some());
         assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax");
+        assert!(&header.reindent_text.is_none());
     }
 
     #[test]
@@ -424,5 +430,15 @@ foobar
         assert_eq!(&header.testtoken_start, &"//");
         assert!(header.testtoken_end.is_none());
         assert_eq!(&header.syntax_file, &"Packages/Example/Example.sublime-syntax");
+        assert!(&header.reindent_text.is_none());
+    }
+    
+    #[test]
+    fn can_parse_syntax_test_reindentation_header() {
+        let header = parse_syntax_test_header_line(&"// SYNTAX TEST reindent-unchanged reindent-unindented \"Packages/PHP/PHP.sublime-syntax\"").unwrap();
+        assert_eq!(&header.testtoken_start, &"//");
+        assert!(header.testtoken_end.is_none());
+        assert_eq!(&header.syntax_file, &"Packages/PHP/PHP.sublime-syntax");
+        assert_eq!(&header.reindent_text.unwrap(), &"reindent-unchanged reindent-unindented");
     }
 // }