Fix LU parser rules to support all comments in a file (#6744)

* Update LU parser rules to support only comments * Fix unit tests * Add unit test to cover all comments case --------- Co-authored-by: CeciliaAvila <[email protected]>
microsoft · Feb 6, 2024 · 84df086 · 84df086
1 parent 36927e1
commit 84df086
Show file tree

Hide file tree

Showing 20 changed files with 1,450 additions and 847 deletions.
diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileLexer.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileLexer.cs
diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileLexer.interp b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileLexer.interp
diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParser.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParser.cs
diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParser.interp b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParser.interp
diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserBaseListener.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserBaseListener.cs
@@ -72,6 +72,18 @@ public virtual void EnterNewline([NotNull] LUFileParser.NewlineContext context)
 	/// <param name="context">The parse tree.</param>
 	public virtual void ExitNewline([NotNull] LUFileParser.NewlineContext context) { }
 	/// <summary>
+	/// Enter a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// <para>The default implementation does nothing.</para>
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	public virtual void EnterCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context) { }
+	/// <summary>
+	/// Exit a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// <para>The default implementation does nothing.</para>
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	public virtual void ExitCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context) { }
+	/// <summary>
 	/// Enter a parse tree produced by <see cref="LUFileParser.errorString"/>.
 	/// <para>The default implementation does nothing.</para>
 	/// </summary>

diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserBaseVisitor.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserBaseVisitor.cs
@@ -65,6 +65,16 @@ public partial class LUFileParserBaseVisitor<Result> : AbstractParseTreeVisitor<
 	/// <return>The visitor result.</return>
 	public virtual Result VisitNewline([NotNull] LUFileParser.NewlineContext context) { return VisitChildren(context); }
 	/// <summary>
+	/// Visit a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// <para>
+	/// The default implementation returns the result of calling <see cref="AbstractParseTreeVisitor{Result}.VisitChildren(IRuleNode)"/>
+	/// on <paramref name="context"/>.
+	/// </para>
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	/// <return>The visitor result.</return>
+	public virtual Result VisitCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context) { return VisitChildren(context); }
+	/// <summary>
 	/// Visit a parse tree produced by <see cref="LUFileParser.errorString"/>.
 	/// <para>
 	/// The default implementation returns the result of calling <see cref="AbstractParseTreeVisitor{Result}.VisitChildren(IRuleNode)"/>

diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserListener.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserListener.cs
@@ -61,6 +61,16 @@ public interface ILUFileParserListener : IParseTreeListener {
 	/// <param name="context">The parse tree.</param>
 	void ExitNewline([NotNull] LUFileParser.NewlineContext context);
 	/// <summary>
+	/// Enter a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	void EnterCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context);
+	/// <summary>
+	/// Exit a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	void ExitCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context);
+	/// <summary>
 	/// Enter a parse tree produced by <see cref="LUFileParser.errorString"/>.
 	/// </summary>
 	/// <param name="context">The parse tree.</param>

diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserVisitor.cs b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Generated/LUFileParserVisitor.cs
@@ -50,6 +50,12 @@ public interface ILUFileParserVisitor<Result> : IParseTreeVisitor<Result> {
 	/// <return>The visitor result.</return>
 	Result VisitNewline([NotNull] LUFileParser.NewlineContext context);
 	/// <summary>
+	/// Visit a parse tree produced by <see cref="LUFileParser.commentDefinition"/>.
+	/// </summary>
+	/// <param name="context">The parse tree.</param>
+	/// <return>The visitor result.</return>
+	Result VisitCommentDefinition([NotNull] LUFileParser.CommentDefinitionContext context);
+	/// <summary>
 	/// Visit a parse tree produced by <see cref="LUFileParser.errorString"/>.
 	/// </summary>
 	/// <param name="context">The parse tree.</param>

diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/LUFileLexer.g4 b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/LUFileLexer.g4
@@ -54,7 +54,7 @@ MODEL_INFO
   ;
 
 COMMENT
-  : WS* '>' ~('\r'|'\n')* -> skip
+  : WS* '>' ~('\r'|'\n')*
   ;
 
 QNA

diff --git a/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/LUFileParser.g4 b/libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/LUFileParser.g4
@@ -16,6 +16,7 @@ paragraph
     | referenceSection
     | qnaSection
     | modelInfoSection
+    | commentDefinition
     ;
 
 // Treat EOF as newline to hanle file end gracefully
@@ -25,6 +26,10 @@ newline
     : WS* (NEWLINE | EOF)
     ;
 
+commentDefinition
+    : COMMENT NEWLINE?
+    ;
+
 errorString
     : (WS|INVALID_TOKEN_DEFAULT_MODE)+
     ;
@@ -74,7 +79,7 @@ intentBody
 	;
 
 normalIntentBody
-    : WS* ((normalIntentString newline) | errorString)+
+    : WS* ((normalIntentString newline) | errorString | commentDefinition)+
     ;
 
 normalIntentString

diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/AllComments.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/AllComments.json
@@ -0,0 +1,5 @@
+{
+  "sections": [],
+  "content": "> This is a comment\n> This is another comment",
+  "errors": []
+}
diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/AllComments.txt b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/AllComments.txt
@@ -0,0 +1,2 @@
+> This is a comment
+> This is another comment
diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/LU_Sections_Unix.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/LU_Sections_Unix.json
@@ -161,11 +161,11 @@
                     "errorMsgs": [
 
                     ],
-                    "contextText": "- Please set {deviceTemperature = thermostat to 72}",
+                    "contextText": "    - Please set {deviceTemperature = thermostat to 72}",
                     "range": {
                         "Start": {
                             "Line": 17,
-                            "Character": 4
+                            "Character": 0
                         },
                         "End": {
                             "Line": 17,

diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/LU_Sections_Windows.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/LU_Sections_Windows.json
diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU131.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU131.json
@@ -52,7 +52,7 @@
       "Severity": "ERROR"
     },
     {
-      "Message": "syntax error: extraneous input 'ml' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
+      "Message": "syntax error: extraneous input 'ml' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, COMMENT, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
       "Range": {
         "Start": {
           "Line": 2,

diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU139.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU139.json
@@ -52,7 +52,7 @@
       "Severity": "ERROR"
     },
     {
-      "Message": "syntax error: extraneous input 'regex' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
+      "Message": "syntax error: extraneous input 'regex' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, COMMENT, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
       "Range": {
         "Start": {
           "Line": 2,

diff --git a/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU153.json b/tests/Parsers/Microsoft.Bot.Builder.Parsers.LU.Tests/Fixtures/testLU153.json
@@ -37,7 +37,7 @@
       "Severity": "ERROR"
     },
     {
-      "Message": "syntax error: extraneous input 'abc' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
+      "Message": "syntax error: extraneous input 'abc' expecting {<EOF>, WS, NEWLINE, QNA_SOURCE_INFO, MODEL_INFO, COMMENT, QNA, HASH, '$', '@', IMPORT, REFERENCE, QNA_ID_MARK}",
       "Range": {
         "Start": {
           "Line": 2,
-Original file line number
+Diff line change
@@ Expand Up / @@ -54,7 +54,7 @@ MODEL_INFO @@
       ;
     COMMENT
-      : WS* '>' ~('\r'|'\n')* -> skip
+      : WS* '>' ~('\r'|'\n')*
       ;
     QNA
@@ Expand Down @@