Error recovery

.gitattributes

@@ -0,0 +1 @@
+*.dot diff=-astextplain

.gitignore

@@ -32,3 +32,6 @@ dependency-reduced-pom.xml
 
 *.iml
 .idea/
+
+# generated by error recovery benchmark
+rascal-recovery-stats.csv

.vscode/launch.json

@@ -38,7 +38,7 @@
             "request": "launch",
             "mainClass": "org.rascalmpl.shell.RascalShell",
             "projectName": "rascal",
-            "cwd" : "${workspaceFolder}/../rascal-tutor",
+            "cwd": "${workspaceFolder}/../rascal-tutor",
             "vmArgs": "-Xss80m -Xmx2g -ea"
         },
         {

pom.xml

@@ -168,6 +168,7 @@
                             <goal>package</goal>
                         </goals>
                     </execution>
+		    <!--
                     <execution>
                         <id>default-cli</id>
                         <phase>compile</phase>
@@ -190,6 +191,7 @@
                             </ignores>
                         </configuration>
                         </execution>
+		    -->
                 </executions>
             </plugin>
             <plugin>

src/org/rascalmpl/interpreter/Evaluator.java

@@ -989,7 +989,7 @@ private IFunction parserForCurrentModule(RascalFunctionValueFactory vf, ModuleEn
         IMap syntaxDefinition = curMod.getSyntaxDefinition();
         IMap grammar = (IMap) getParserGenerator().getGrammarFromModules(getMonitor(), curMod.getName(), syntaxDefinition).get("rules");
         IConstructor reifiedType = vf.reifiedType(dummy, grammar);
-        return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); 
+        return vf.parsers(reifiedType, vf.bool(false), vf.bool(false), vf.bool(false), vf.bool(false), vf.set()); 
     }
 
     private Result<IValue> evalMore(String command, ISourceLocation location)

src/org/rascalmpl/library/ParseTree.rsc

@@ -143,7 +143,6 @@ extend Type;
 extend Message;
 extend List;
 
-
 @synopsis{The Tree data type as produced by the parser.}
 @description{
 A `Tree` defines the trees normally found after parsing; additional constructors exist for execptional cases:
@@ -177,6 +176,8 @@ construct ordered and un-ordered compositions, and associativity groups.
 <4> `assoc`  means all alternatives are acceptable, but nested on the declared side;
 <5> `reference` means a reference to another production rule which should be substituted there,
     for extending priority chains and such.
+<6> `error` means a node produced by error recovery.
+<7> `skipped` means characters skipped during error recovery, always the last child of an `appl` with a `error` production.
 } 
 data Production 
      = prod(Symbol def, list[Symbol] symbols, set[Attr] attributes) // <1>
@@ -189,6 +190,9 @@ data Production
      | \reference(Symbol def, str cons) // <5>
      ;
 
+data Production
+     = \error(Symbol def, Production prod, int dot)
+     | \skipped(Symbol def);
 
 @synopsis{Attributes in productions.}
 @description{
@@ -348,6 +352,16 @@ The latter option terminates much faster, i.e. always in cubic time, and always
 while constructing ambiguous parse forests may grow to O(n^p+1), where p is the length of the longest production rule and n 
 is the length of the input.
 
+The `allowRecovery` can be set to `true` to enable error recovery. This is an experimental feature.
+When error recovery is enabled, the parser will attempt to recover from parse errors and continue parsing.
+If successful, a parse tree with error and skipped productions is returned (see the definition of `Production` above).
+A number of functions is provided to analyze trees with errors, for example `hasErrors`, `getSkipped`, and `getErrorText`.
+Note that the resulting parse forest can contain a lot of error nodes. `disambiguateErrors` can be used to prune the forest
+and leave a tree with a single (or even zero) errors based on simple heuristics.
+When `allowAmbiguity` is set to false, `allowRecovery` is set to true, and `filters` is empty, this disambiguation is done
+automatically so you should end up with a tree with no error ambiguities. Regular ambiguities can still occur
+and will result in an error.
+
 The `filters` set contains functions which may be called optionally after the parse algorithm has finished and just before
 the Tree representation is built. The set of functions contain alternative functions, only on of them is successfully applied
 to each node in a tree. If such a function fails to apply, the other ones are tried. There is no fixed-point computation, so
@@ -392,14 +406,15 @@ catch ParseError(loc l): {
 }
 ```
 }
-&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
-  = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|);
 
-&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
-  = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin);
+&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
+  = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|);
+
+&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
+  = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, origin);
 
-&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
-  = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input);
+&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={})
+  = parser(begin, allowAmbiguity=allowAmbiguity, allowRecovery=allowRecovery, hasSideEffects=hasSideEffects, filters=filters)(input, input);
 
 
 @synopsis{Generates a parser from an input grammar.}
@@ -415,15 +430,16 @@ So the parse function reads either directly from a str or via the contents of a
 which leads to the prefix of the `src` fields of the resulting tree.
 
 The parse function behaves differently depending of the given keyword parameters:
-     *  `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false,
+     * `allowAmbiguity`: if true then no exception is thrown in case of ambiguity and a parse forest is returned. if false,
                          the parser throws an exception during tree building and produces only the first ambiguous subtree in its message.
                          if set to `false`, the parse constructs trees in linear time. if set to `true` the parser constructs trees in polynomial time.
-     * 
+     * 'allowRecovery`: ***experimental*** if true, the parser tries to recover when it encounters a parse error. if a parse error is encountered that can be recovered from,
+                         special `error` and `skipped` nodes are included in the resulting parse tree. More documentation will be added here when this feature matures.
      *  `hasSideEffects`: if false then the parser is a lot faster when constructing trees, since it does not execute the parse _actions_ in an
                          interpreted environment to make side effects (like a symbol table) and it can share more intermediate results as a result.
 }
 @javaClass{org.rascalmpl.library.Prelude}
-java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); 
+java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); 
 
 @javaClass{org.rascalmpl.library.Prelude}
 @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.}
@@ -436,15 +452,15 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole
 * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. 
 The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar.
 }
-java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool hasSideEffects=false, set[Tree(Tree)] filters={}); 
+java Tree (value input, loc origin) firstAmbiguityFinder(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}); 
 
 @synopsis{Generates parsers from a grammar (reified type), where all non-terminals in the grammar can be used as start-symbol.}
 @description{
 This parser generator behaves the same as the `parser` function, but it produces parser functions which have an additional
 nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing.
 }
 @javaClass{org.rascalmpl.library.Prelude}
-java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false,  set[Tree(Tree)] filters={}); 
+java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false,  set[Tree(Tree)] filters={}); 
 
 @javaClass{org.rascalmpl.library.Prelude}
 @synopsis{Generates a parser function that can be used to find the left-most deepest ambiguous sub-sentence.}
@@ -457,7 +473,7 @@ the tree that exhibits ambiguity. This can be done very quickly, while the whole
 * The returned sub-tree usually has a different type than the parameter of the type[] symbol that was passed in. 
 The reason is that sub-trees typically have a different non-terminal than the start non-terminal of a grammar.
 }
-java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool hasSideEffects=false,  set[Tree(Tree)] filters={}); 
+java Tree (type[Tree] nonterminal, value input, loc origin) firstAmbiguityFinders(type[Tree] grammar, bool allowRecovery=false, bool hasSideEffects=false,  set[Tree(Tree)] filters={}); 
 
 @synopsis{Parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring.}
 @description{
@@ -535,15 +551,15 @@ p(type(sort("E"), ()), "e+e", |src:///|);
 * reifiying types (use of `#`) will trigger the loading of a parser generator anyway. You have to use
 this notation for types to avoid that: `type(\start(sort("MySort")), ())` to avoid the computation for `#start[A]`
 }
-java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={});
+java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={});
 
 @synopsis{Load a previously serialized parser, for a specific non-terminal, from disk for usage}
 @description{
 This loader behaves just like ((loadParsers)), except that the resulting parser function is already
 bound to a specific non-terminal. 
 }
 @javaClass{org.rascalmpl.library.Prelude}
-java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={});
+java &U (value input, loc origin) loadParser(type[&U] nonterminal, loc savedParsers, bool allowAmbiguity=false, bool allowRecovery=false, bool hasSideEffects=false, set[Tree(Tree)] filters={});
 
 @synopsis{Yield the string of characters that form the leafs of the given parse tree.}
 @description{

src/org/rascalmpl/library/Prelude.java

@@ -2377,20 +2377,24 @@ public INode arbNode() {
 
 	protected final TypeReifier tr;
 
+	public IFunction parser(IValue start,  IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) {
+	    return rascalValues.parser(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters);
+	}
+
 	public IFunction parser(IValue start,  IBool allowAmbiguity, IBool hasSideEffects, ISet filters) {
-	    return rascalValues.parser(start, allowAmbiguity, hasSideEffects, values.bool(false), filters);
+	    return rascalValues.parser(start, allowAmbiguity, values.bool(false), hasSideEffects, values.bool(false), filters);
 	}
 
-	public IFunction firstAmbiguityFinder(IValue start, IBool hasSideEffects, ISet filters) {
-	    return rascalValues.parser(start, values.bool(true), hasSideEffects, values.bool(true), filters);
+	public IFunction firstAmbiguityFinder(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) {
+	    return rascalValues.parser(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters);
 	}
 
-	public IFunction parsers(IValue start,  IBool allowAmbiguity, IBool hasSideEffects, ISet filters) {
-        return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, values.bool(false), filters);
+	public IFunction parsers(IValue start,  IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) {
+        return rascalValues.parsers(start, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters);
     }
 
-	public IFunction firstAmbiguityFinders(IValue start, IBool hasSideEffects, ISet filters) {
-        return rascalValues.parsers(start, values.bool(true), hasSideEffects, values.bool(true), filters);
+	public IFunction firstAmbiguityFinders(IValue start, IBool allowRecovery, IBool hasSideEffects, ISet filters) {
+        return rascalValues.parsers(start, values.bool(true), allowRecovery, hasSideEffects, values.bool(true), filters);
     }
 
 	public void storeParsers(IValue start, ISourceLocation saveLocation) {
@@ -2405,18 +2409,18 @@ public void storeParsers(IValue start, ISourceLocation saveLocation) {
 		}
 	}
 
-	public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) {
+	public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool allowRecovery, IBool hasSideEffects, ISet filters) {
 		try {
-			return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters);
+			return rascalValues.loadParsers(savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters);
 		}
 		catch (IOException | ClassNotFoundException e) {
 			throw RuntimeExceptionFactory.io(e.getMessage());
 		}
 	}
 
-	public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) {
+	public IFunction loadParser(IValue grammar, ISourceLocation savedLocation, IBool allowRecovery, IBool allowAmbiguity, IBool hasSideEffects, ISet filters) {
 		try {
-			return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, hasSideEffects, values.bool(false), filters);
+			return rascalValues.loadParser(grammar, savedLocation, allowAmbiguity, allowRecovery, hasSideEffects, values.bool(false), filters);
 		}
 		catch (IOException | ClassNotFoundException e) {
 			throw RuntimeExceptionFactory.io(e.getMessage());

src/org/rascalmpl/library/lang/c90/examples/hello-world.c

@@ -0,0 +1,19 @@
+
+int print(const char *text); 
+
+void printHello(char *name) {
+  print("Hello ");
+  print(name);
+  print("!");
+}
+
+int main(int argc, char *argv[]) {
+  char *name;
+  if (argc > 1) {
+    name = argv[1];
+  } else {
+    name = "World";
+  }
+
+  printHello(name);
+}

src/org/rascalmpl/library/lang/diff/unified/UnifiedDiff.rsc

@@ -3,6 +3,8 @@
 @contributor{Tijs van der Storm - [email protected] (CWI)}
 module lang::diff::unified::UnifiedDiff
 
+start syntax DiffFile = Diff;
+
 syntax Diff
   = Header old Header new Chunk* chunks
   ;

src/org/rascalmpl/library/lang/diff/unified/examples/example.diff

@@ -0,0 +1,39 @@
+--- a/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java
++++ b/src/org/rascalmpl/parser/uptr/UPTRNodeFactory.java
+@@ -1,6 +1,7 @@
+ package org.rascalmpl.parser.uptr;
+
+ import java.net.URI;
++import java.util.Arrays;
+ import java.util.IdentityHashMap;
+ import java.util.Map;
+
+@@ -21,7 +22,9 @@ import org.rascalmpl.values.parsetrees.ProductionAdapter;
+ import org.rascalmpl.values.parsetrees.TreeAdapter;
+
+ public class UPTRNodeFactory implements INodeConstructorFactory<ITree, ISourceLocation>{
+-	private final static RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
++	private static final RascalValueFactory VF = (RascalValueFactory) ValueFactoryFactory.getValueFactory();
++	private static final IConstructor SKIPPED = VF.constructor(RascalValueFactory.Production_Skipped, VF.constructor(RascalValueFactory.Symbol_IterStar, VF.constructor(RascalValueFactory.Symbol_CharClass, VF.list(VF.constructor(RascalValueFactory.CharRange_Range, VF.integer(1), VF.integer(Character.MAX_CODE_POINT))))));
++
+ 	private boolean allowAmb;
+
+ 	public UPTRNodeFactory(boolean allowAmbiguity){
+@@ -141,7 +144,14 @@ public class UPTRNodeFactory implements INodeConstructorFactory<ITree, ISourceLo
+ 	}
+
+     @Override
+-    public ITree createRecoveryNode(int[] characters) {
+-       throw new UnsupportedOperationException(); 
+-    }
++    public ITree createSkippedNode(int[] characters) {
++		return createLiteralNode(characters, SKIPPED);
++	}
++
++	public ITree createErrorNode(ArrayList<ITree> children, Object production) {
++		IConstructor prod = (IConstructor) production;
++		IConstructor errorProd = VF.constructor(RascalValueFactory.Production_Error, prod.get(0), prod, VF.integer(children.size()-1));
++		return buildAppl(children, errorProd);
++	}
++
+ }