Tons more documentation comments

Readme.md

@@ -1,6 +1,6 @@
 # syntect
 
-`syntect` is a work-in-progress syntax highlighting library for Rust that uses [Sublime Text syntax definitions](http://www.sublimetext.com/docs/3/syntax.html#include-syntax). It is not quite complete but eventually the goal is for it to be used in code analysis tools and text editors.
+`syntect` is a syntax highlighting library for Rust that uses [Sublime Text syntax definitions](http://www.sublimetext.com/docs/3/syntax.html#include-syntax). It aims to be a good solution for any Rust project that needs syntax highlighting, including deep integration with text editors written in Rust.
 
 If you are writing a text editor (or something else needing highlighting) in Rust and this library doesn't fit your needs, I consider that a bug and you should file an issue or email me.
 
@@ -20,6 +20,7 @@ Note: the build is currently failing on Travis Linux stable, but succeeding on n
 - [x] Expose internals of the parsing process so text editors can do things like cache parse states and use semantic info for code intelligence
 - [x] High quality highlighting, supporting things like heredocs and complex syntaxes (like Rust's).
 - [x] Include a compressed dump of all the default syntax definitions in the library binary so users don't have to manage a folder of syntaxes.
+- [x] Well documented, I've tried to add a useful documentation comment to everything that isn't utterly self explanatory.
 
 ## Screenshots
 

src/dumps.rs

@@ -1,3 +1,12 @@
+//! Methods for dumping serializable structs to a compressed binary format
+//! These are used to load and store the dumps used for fast startup times.
+//!
+//! Currently syntect serializes `SyntaxSet` structs with `dump_to_file`
+//! into `.packdump` files and likewise `ThemeSet` structs to `.themedump` files.
+//!
+//! You can use these methods to manage your own caching of compiled syntaxes and
+//! themes. And even your own rustc_serialize::Encodable structures if you want to
+//! be consistent with your format.
 use bincode::SizeLimit;
 use bincode::rustc_serialize::*;
 use std::fs::File;
@@ -10,6 +19,7 @@ use flate2::read::ZlibDecoder;
 use flate2::Compression;
 use rustc_serialize::{Encodable, Decodable};
 
+/// Dumps an object to a binary array in the same format as `dump_to_file`
 pub fn dump_binary<T: Encodable>(o: &T) -> Vec<u8> {
     let mut v = Vec::new();
     {
@@ -19,6 +29,9 @@ pub fn dump_binary<T: Encodable>(o: &T) -> Vec<u8> {
     v
 }
 
+/// Dumps an encodable object to a file at a given path. If a file already exists at that path
+/// it will be overwritten. The files created are encoded with the `bincode` crate and then
+/// compressed with the `flate2` crate.
 pub fn dump_to_file<T: Encodable, P: AsRef<Path>>(o: &T, path: P) -> EncodingResult<()> {
     let f = BufWriter::new(try!(File::create(path).map_err(EncodingError::IoError)));
     let mut encoder = ZlibEncoder::new(f, Compression::Best);

src/highlighting/mod.rs

@@ -10,7 +10,7 @@ mod highlighter;
 mod theme_set;
 
 pub use self::selector::*;
-pub use self::settings::*;
+pub use self::settings::{SettingsError};
 pub use self::style::*;
 pub use self::theme::*;
 pub use self::highlighter::*;

src/highlighting/selector.rs

@@ -17,10 +17,13 @@ pub struct ScopeSelector {
 /// work.
 #[derive(Debug, Clone, PartialEq, Eq, Default, RustcEncodable, RustcDecodable)]
 pub struct ScopeSelectors {
+    /// the selectors, if any of them match, this matches
     pub selectors: Vec<ScopeSelector>,
 }
 
 impl ScopeSelector {
+    /// Checks if this selector matches a given scope stack.
+    /// See `ScopeSelectors#does_match` for more info.
     pub fn does_match(&self, stack: &[Scope]) -> Option<MatchPower> {
         if self.exclude.as_ref().and_then(|sel| sel.does_match(stack)).is_some() {
             return None;
@@ -32,6 +35,7 @@ impl ScopeSelector {
 impl FromStr for ScopeSelector {
     type Err = ParseScopeError;
 
+    /// Parses a scope prefix followed optionally by a " - " and then a scope prefix to exclude
     fn from_str(s: &str) -> Result<ScopeSelector, ParseScopeError> {
         match s.find(" - ") {
             Some(index) => {
@@ -53,14 +57,6 @@ impl FromStr for ScopeSelector {
 }
 
 impl ScopeSelectors {
-    pub fn does_match(&self, stack: &[Scope]) -> Option<MatchPower> {
-        self.selectors.iter().filter_map(|sel| sel.does_match(stack)).max()
-    }
-}
-
-impl FromStr for ScopeSelectors {
-    type Err = ParseScopeError;
-
     /// checks if any of these selectors match the given scope stack
     /// if so it returns a match score, higher match scores are stronger
     /// matches. Scores are ordered according to the rules found
@@ -76,6 +72,15 @@ impl FromStr for ScopeSelectors {
     ///     .does_match(ScopeStack::from_str("a.b c.d j e.f").unwrap().as_slice()),
     ///     Some(MatchPower(0o2001u64 as f64)));
     /// ```
+    pub fn does_match(&self, stack: &[Scope]) -> Option<MatchPower> {
+        self.selectors.iter().filter_map(|sel| sel.does_match(stack)).max()
+    }
+}
+
+impl FromStr for ScopeSelectors {
+    type Err = ParseScopeError;
+
+    /// Parses a series of selectors separated by commas
     fn from_str(s: &str) -> Result<ScopeSelectors, ParseScopeError> {
         let mut selectors = Vec::new();
         for selector in s.split(',') {

src/highlighting/settings.rs

@@ -17,8 +17,11 @@ pub trait ParseSettings: Sized {
     fn parse_settings(settings: Settings) -> Result<Self, Self::Error>;
 }
 
+
+/// An error parsing a settings file
 #[derive(Debug)]
 pub enum SettingsError {
+    /// Incorrect Plist syntax
     Plist(PlistError),
 }
 

src/highlighting/style.rs

@@ -23,33 +23,48 @@ pub struct StyleModifier {
     pub font_style: Option<FontStyle>,
 }
 
+
+/// Pre-defined convenience colour
 pub const BLACK: Color = Color {
     r: 0x00,
     g: 0x00,
     b: 0x00,
     a: 0x00,
 };
+
+/// Pre-defined convenience colour
 pub const WHITE: Color = Color {
     r: 0xFF,
     g: 0xFF,
     b: 0xFF,
     a: 0xFF,
 };
 
+/// RGBA colour, these numbers come directly from the theme so
+/// for now you might have to do your own colour space conversion if you are outputting
+/// a different colour space from the theme. This can be a problem because some Sublime
+/// themes use sRGB and some don't. This is specified in an attribute syntect doesn't parse yet.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, RustcEncodable, RustcDecodable)]
 pub struct Color {
+    /// Red component
     pub r: u8,
+    /// Green component
     pub g: u8,
+    /// Blue component
     pub b: u8,
+    /// Alpha component
     pub a: u8,
 }
 
 bitflags! {
     /// This can be a combination of `FONT_STYLE_BOLD`, `FONT_STYLE_UNDERLINE` and `FONT_STYLE_ITALIC`
     #[derive(RustcEncodable, RustcDecodable)]
     flags FontStyle: u8 {
+        /// A bitfield constant FontStyle
         const FONT_STYLE_BOLD = 1,
+        /// A bitfield constant FontStyle
         const FONT_STYLE_UNDERLINE = 2,
+        /// A bitfield constant FontStyle
         const FONT_STYLE_ITALIC = 4,
     }
 }

src/highlighting/theme_set.rs

@@ -12,6 +12,7 @@ pub struct ThemeSet {
     pub themes: BTreeMap<String, Theme>,
 }
 
+/// A set of themes, includes convenient methods for loading and discovering themes.
 impl ThemeSet {
     /// Returns all the themes found in a folder, good for enumerating before loading one with get_theme
     pub fn discover_theme_paths<P: AsRef<Path>>(folder: P) -> Result<Vec<PathBuf>, LoadingError> {

src/lib.rs

@@ -37,11 +37,18 @@ use highlighting::{ParseThemeError, SettingsError};
 /// Common error type used by syntax and theme loading
 #[derive(Debug)]
 pub enum LoadingError {
+    /// error finding all the files in a directory
     WalkDir(walkdir::Error),
+    /// error reading a file
     Io(IoError),
+    /// a syntax file was invalid in some way
     ParseSyntax(ParseSyntaxError),
+    /// a theme file was invalid in some way
     ParseTheme(ParseThemeError),
+    /// a theme's Plist syntax was invalid in some way
     ReadSettings(SettingsError),
+    /// A path given to a method was invalid.
+    /// Possibly because it didn't reference a file or wasn't UTF-8.
     BadPath,
 }
 

src/parsing/parser.rs

@@ -44,6 +44,8 @@ struct RegexMatch {
 type MatchCache = Vec<bool>;
 
 impl ParseState {
+    /// Create a state from a syntax, keeps its own reference counted
+    /// pointer to the main context of the syntax.
     pub fn new(syntax: &SyntaxDefinition) -> ParseState {
         let start_state = StateLevel {
             context: syntax.contexts["main"].clone(),

src/parsing/scope.rs

@@ -9,6 +9,8 @@ use rustc_serialize::{Encodable, Encoder, Decodable, Decoder};
 use std::cmp::Ordering;
 
 lazy_static! {
+    /// The global scope repo, exposed in case you want to minimize locking and unlocking.
+    /// Shouldn't be necessary for you to use. See the `ScopeRepository` docs.
     pub static ref SCOPE_REPO: Mutex<ScopeRepository> = Mutex::new(ScopeRepository::new());
 }
 
@@ -32,6 +34,7 @@ pub struct Scope {
     b: u64,
 }
 
+/// Not all strings are valid scopes
 #[derive(Debug)]
 pub enum ParseScopeError {
     /// Due to a limitation of the current optimized internal representation
@@ -42,6 +45,12 @@ pub enum ParseScopeError {
     TooManyAtoms,
 }
 
+/// The structure used to keep of the mapping between scope atom numbers
+/// and their string names. It is only exposed in case you want to lock
+/// `SCOPE_REPO` and then allocate a whole bunch of scopes at once
+/// without thrashing the lock. It is recommended you just use `Scope::new()`
+///
+/// Only `Scope`s created by the same repository have valid comparison results.
 #[derive(Debug)]
 pub struct ScopeRepository {
     atoms: Vec<String>,
@@ -139,11 +148,16 @@ impl ScopeRepository {
 }
 
 impl Scope {
+    /// Parses a `Scope` from a series of atoms separated by
+    /// `.` characters. Example: `Scope::new("meta.rails.controller")`
     pub fn new(s: &str) -> Result<Scope, ParseScopeError> {
         let mut repo = SCOPE_REPO.lock().unwrap();
         repo.build(s.trim())
     }
 
+    /// Gets the atom number at a given index.
+    /// I can't think of any reason you'd find this useful.
+    /// It is used internally for turning a scope back into a string.
     pub fn atom_at(self, index: usize) -> u16 {
         let shifted = if index < 4 {
             (self.a >> ((3 - index) * 16))
@@ -285,6 +299,10 @@ impl ScopeStack {
     pub fn pop(&mut self) {
         self.scopes.pop();
     }
+
+    /// Modifies this stack according to the operation given
+    /// use this to create a stack from a `Vec` of changes
+    /// given by the parser.
     pub fn apply(&mut self, op: &ScopeStackOp) {
         match op {
             &ScopeStackOp::Push(scope) => self.scopes.push(scope),
@@ -296,6 +314,9 @@ impl ScopeStack {
             &ScopeStackOp::Noop => (),
         }
     }
+
+    /// Prints out each scope in the stack separated by spaces
+    /// and then a newline. Top of the stack at the end.
     pub fn debug_print(&self, repo: &ScopeRepository) {
         for s in self.scopes.iter() {
             print!("{} ", repo.to_string(*s));
@@ -363,6 +384,7 @@ impl ScopeStack {
 impl FromStr for ScopeStack {
     type Err = ParseScopeError;
 
+    /// Parses a scope stack from a whitespace separated list of scopes.
     fn from_str(s: &str) -> Result<ScopeStack, ParseScopeError> {
         let mut scopes = Vec::new();
         for name in s.split_whitespace() {

src/parsing/syntax_definition.rs

@@ -146,13 +146,15 @@ pub fn context_iter(ctx: ContextPtr) -> MatchIter {
 }
 
 impl Context {
+    /// Returns the match pattern at an index, panics if the thing isn't a match pattern
     pub fn match_at(&self, index: usize) -> &MatchPattern {
         match self.patterns[index] {
             Pattern::Match(ref match_pat) => match_pat,
             _ => panic!("bad index to match_at"),
         }
     }
 
+    /// Returns a mutable reference, otherwise like `match_at`
     pub fn match_at_mut(&mut self, index: usize) -> &mut MatchPattern {
         match self.patterns[index] {
             Pattern::Match(ref mut match_pat) => match_pat,

src/parsing/yaml_load.rs

@@ -10,13 +10,23 @@ use std::ops::DerefMut;
 
 #[derive(Debug)]
 pub enum ParseSyntaxError {
+    /// Invalid YAML file syntax, or at least something yaml_rust can't handle
     InvalidYaml(ScanError),
+    /// The file must contain at least on YAML document
     EmptyFile,
+    /// Some keys are required for something to be a valid `.sublime-syntax`
     MissingMandatoryKey(&'static str),
+    /// Invalid regex
     RegexCompileError(onig::Error),
+    /// A scope that syntect's scope implementation can't handle
     InvalidScope(ParseScopeError),
+    /// A reference to another file that is invalid
     BadFileRef,
+    /// Syntaxes must have a context named "main"
     MainMissing,
+    /// Some part of the YAML file is the wrong type (e.g a string but should be a list)
+    /// Sorry this doesn't give you any way to narrow down where this is.
+    /// Maybe use Sublime Text to figure it out.
     TypeMismatch,
 }
 
@@ -47,6 +57,8 @@ struct ParserState<'a> {
 }
 
 impl SyntaxDefinition {
+    /// In case you want to create your own SyntaxDefinition's in memory from strings.
+    /// Generally you should use a `SyntaxSet`
     pub fn load_from_str(s: &str,
                          lines_include_newline: bool)
                          -> Result<SyntaxDefinition, ParseSyntaxError> {

src/util.rs

@@ -1,3 +1,5 @@
+//! Convenient utility methods, mostly for printing `syntect` data structures
+//! prettily to the terminal.
 use highlighting::Style;
 use std::fmt::Write;
 use parsing::ScopeStackOp;
@@ -6,6 +8,11 @@ use parsing::ScopeStackOp;
 /// terminal escape codes. Meant for debugging and testing.
 /// It's currently fairly inefficient in its use of escape codes.
 ///
+/// Note that this does not currently ever un-set the colour so that
+/// the end of a line will also get highlighted with the background.
+/// This means if you might want to use `println!("\x1b[0m");` after
+/// to clear the colouring.
+///
 /// If `bg` is true then the background is also set
 pub fn as_24_bit_terminal_escaped(v: &[(Style, &str)], bg: bool) -> String {
     let mut s: String = String::new();