Track the starting and ending position of each token. (VirusTotal#53)

This opens the door for improving the AST in the future, storing the starting and ending position within the source code for each node in the AST. Also improves the way in the line number associated to each rule is computed, until know the line number for rule was the line number where the "rule" keyword was found, but now it takes in to account that a rule definition may start with some modifiers located some lines before the "rule" keyword.

gyp_test.go

@@ -48,28 +48,31 @@ func TestNonAscii(t *testing.T) {
 
 func TestLineNo(t *testing.T) {
 	rs, err := ParseString(`
-rule foo {
-  strings:
-    $a = "foo"
-    // Intentional blank line
-    $b = "bar"
-    $c = "baz"
-  condition:
-    all of them
-}
-rule bar {
-  strings:
-    $a = "foo"
-  condition:
-    all of them
-}`)
+rule foo {                        //  2
+  strings:                        //  3
+    $a = "foo"                    //  4
+    // Intentional blank line     //  5
+    $b = "bar"                    //  6
+    $c = "baz"                    //  7
+  condition:                      //  8
+    all of them                   //  9
+}                                 // 10
+global                            // 11
+private                           // 12
+rule bar {                        // 13
+  strings:                        // 14
+    $a = "foo"                    // 15
+  condition:                      // 16
+    all of them                   // 17
+}                                 // 18
+`)
 	assert.NoError(t, err)
 	assert.Equal(t, 2, rs.Rules[0].LineNo)
 	assert.Equal(t, 4, rs.Rules[0].Strings[0].GetLineNo())
 	assert.Equal(t, 6, rs.Rules[0].Strings[1].GetLineNo())
 	assert.Equal(t, 7, rs.Rules[0].Strings[2].GetLineNo())
 	assert.Equal(t, 11, rs.Rules[1].LineNo)
-	assert.Equal(t, 13, rs.Rules[1].Strings[0].GetLineNo())
+	assert.Equal(t, 15, rs.Rules[1].Strings[0].GetLineNo())
 
 }
 

parser/grammar.y

@@ -271,8 +271,18 @@ rule
             }
         }
 
+        // The line number of the rule is the line number of the first
+        // modifier....
+        $<lineno>$ = $<lineno>1
+
+        // ... or the line number of the "rule" keyword if the rule doesn't
+        // have any modifiers.
+        if $<lineno>$ == -1 {
+           $<lineno>$  = $<lineno>2
+        }
+
         $$ = &ast.Rule{
-            LineNo: $<lineno>2,
+            LineNo: $<lineno>$,
             Global: $1 & ModGlobal == ModGlobal,
             Private: $1 & ModPrivate == ModPrivate,
             Identifier: $3,
@@ -344,10 +354,17 @@ rule_modifiers
     : /* empty */
       {
         $$ = 0
+        $<lineno>$ = -1
       }
     | rule_modifiers rule_modifier
       {
         $$ = $1 | $2
+
+        if $<lineno>1 == -1 {
+          $<lineno>$ = $<lineno>2
+        } else {
+          $<lineno>$ = $<lineno>1
+        }
       }
     ;
 
@@ -356,10 +373,12 @@ rule_modifier
     : _PRIVATE_
       {
         $$ = ModPrivate
+        $<lineno>$ = $<lineno>1
       }
     | _GLOBAL_
       {
         $$ = ModGlobal
+        $<lineno>$ = $<lineno>1
       }
     ;