feat: added single and multiline comments to the dsl

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt

@@ -14,10 +14,22 @@ class LexerBuilder internal constructor() {
   internal var ignorePattern: Regex = Lexer.Defaults.ignorePattern
     private set
 
+  internal var singleLineComments: Regex? = Lexer.Defaults.singleLineComments
+    private set
+
+  internal var multilineComments: Pair<Regex, Regex>? = Lexer.Defaults.multilineComments
+    private set
+
   /**
    * The lexer will skip over any strings that match this regex.
    * This acts like a token separator.
    *
+   * ```kt
+   * val myLexer = lexer {
+   *   ignorePatterns(Regex("""\s+"""))
+   * }
+   * ```
+   *
    * @param regex regex of the pattern the lexer will not tokenize.
    *
    * @see Lexer.Defaults.ignorePattern
@@ -27,6 +39,48 @@ class LexerBuilder internal constructor() {
   fun ignorePattern(regex: Regex) {
     ignorePattern = regex
   }
+
+  /**
+   * When the lexer identifies a [singleLineComments] pattern, it will skip to the next line
+   * and return the next token.
+   *
+   * ```kt
+   * val myLexer = lexer {
+   *   singleLineComments(Regex("//"))
+   * }
+   * ```
+   *
+   * @param singleLineComments regex of the pattern the lexer will skip over to the next line.
+   *
+   * @see Lexer.Defaults.singleLineComments
+   * @author Nishant Aanjaney Jalan
+   * @since 0.2.0
+   */
+  fun singleLineComments(singleLineComments: Regex) {
+    this.singleLineComments = singleLineComments
+  }
+
+  /**
+   * There are two parts to [multilineComments]: the starting and the ending pattern.
+   * When the lexer identifies the starting pattern, it will continue to skip to the next possible token
+   * until it meets the ending pattern.
+   *
+   * ```kt
+   * val myLexer = lexer {
+   *   // You don't need the square brackets but KDoc doesn't like it...
+   *   multiLineComments(Regex("[/][*]") to Regex("[*][/]"))
+   * }
+   * ```
+   *
+   * @param singleLineComments regex of the pattern the lexer will skip over to the next line.
+   *
+   * @see Lexer.Defaults.singleLineComments
+   * @author Nishant Aanjaney Jalan
+   * @since 0.2.0
+   */
+  fun multiLineComments(multilineComments: Pair<Regex, Regex>) {
+    this.multilineComments = multilineComments
+  }
 }
 
 /**
@@ -52,6 +106,8 @@ fun lexer(init: LexerBuilder.() -> Unit): Lexer {
   val builder = LexerBuilder()
   builder.init()
   return Lexer(
-    ignorePattern = builder.ignorePattern
+    ignorePattern = builder.ignorePattern,
+    singleLineComments = builder.singleLineComments,
+    multilineComments = builder.multilineComments
   )
 }

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt

@@ -45,8 +45,8 @@ import io.github.cybercodernaj.parkour.utils.Position
  */
 class Lexer internal constructor(
   private val ignorePattern: Regex = Defaults.ignorePattern,
-  private val singleLineComments: Regex? = null,
-  private val multilineComments: Pair<Regex, Regex>? = null,
+  private val singleLineComments: Regex? = Defaults.singleLineComments,
+  private val multilineComments: Pair<Regex, Regex>? = Defaults.multilineComments,
   private val identifiers: Regex = Defaults.identifiers,
   private val hardKeywords: Set<String> = emptySet(),
   private val operators: Set<String> = emptySet(),
@@ -72,6 +72,8 @@ class Lexer internal constructor(
      * @since 0.2.0
      */
     val ignorePattern = Regex("""\s+""")
+    val singleLineComments: Regex? = null
+    val multilineComments: Pair<Regex, Regex>? = null
     val identifiers = Regex("""[a-zA-Z_]\w*""")
     val integerLiterals = Regex("""[-+]?[0-9_]+""")
     val floatingLiterals = Regex("""[-+]?[0-9_]*\.[0-9_]+(?:[eE][-+]?[0-9_]+)?""")

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilderTest.kt

@@ -27,5 +27,37 @@ class LexerBuilderTest {
       )
     )
   }
+
+  @Test
+  fun `sets single line comments`() {
+    val myLexer = lexer {
+      singleLineComments(Regex("//"))
+    }
+
+    myLexer.source = StringSource("hi // hello\nhru")
+    assertTokens(
+      myLexer,
+      listOf(
+        Token.Identifier("hi", Position(0, 0), Position(0, 1)),
+        Token.Identifier("hru", Position(1, 0), Position(1, 2))
+      )
+    )
+  }
+
+  @Test
+  fun `sets multiline comments`() {
+    val myLexer = lexer {
+      multiLineComments(Regex("/\\*") to Regex("\\*/"))
+    }
+
+    myLexer.source = StringSource("hi /* hello */\nhru")
+    assertTokens(
+      myLexer,
+      listOf(
+        Token.Identifier("hi", Position(0, 0), Position(0, 1)),
+        Token.Identifier("hru", Position(1, 0), Position(1, 2))
+      )
+    )
+  }
 }
 

docs/core/io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/escape-sequences.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span>escape</span><wbr><span><span>Sequences</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="escape-sequences.html">escapeSequences</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.collections/-list/index.html">List</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-pair/index.html">Pair</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token punctuation">, </span><span class="token punctuation">(</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-string/index.html">String</a><span class="token punctuation">)</span><span class="token operator"> -&gt; </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-char/index.html">Char</a><span class="token operator">&gt;</span><span class="token operator">&gt;</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L79">source</a>)</span></span></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="escape-sequences.html">escapeSequences</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.collections/-list/index.html">List</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-pair/index.html">Pair</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token punctuation">, </span><span class="token punctuation">(</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-string/index.html">String</a><span class="token punctuation">)</span><span class="token operator"> -&gt; </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-char/index.html">Char</a><span class="token operator">&gt;</span><span class="token operator">&gt;</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L81">source</a>)</span></span></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Copyright</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>

docs/core/io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/floating-literals.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span>floating</span><wbr><span><span>Literals</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="floating-literals.html">floatingLiterals</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L77">source</a>)</span></span></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="floating-literals.html">floatingLiterals</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L79">source</a>)</span></span></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Copyright</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>

docs/core/io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/identifiers.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span><span>identifiers</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="identifiers.html">identifiers</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L75">source</a>)</span></span></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="identifiers.html">identifiers</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L77">source</a>)</span></span></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Copyright</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>

docs/core/io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/index.html

@@ -144,6 +144,36 @@ <h2 class="">Properties</h2>
               </div>
             </div>
           </div>
+<a data-name="-1528482951%2FProperties%2F537371977" anchor-label="multilineComments" id="-1528482951%2FProperties%2F537371977" data-filterable-set=":core:dokkaHtmlPartial/main"></a>
+          <div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main">
+            <div class="main-subrow keyValue ">
+              <div class=""><span class="inline-flex">
+                  <div><a href="multiline-comments.html"><span>multiline</span><wbr><span><span>Comments</span></span></a></div>
+<span class="anchor-wrapper"><span class="anchor-icon" pointing-to="-1528482951%2FProperties%2F537371977"></span>
+                    <div class="copy-popup-wrapper "><span class="copy-popup-icon"></span><span>Link copied to clipboard</span></div>
+                  </span></span></div>
+              <div>
+                <div class="title">
+                  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="multiline-comments.html">multilineComments</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-pair/index.html">Pair</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token punctuation">, </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token operator">&gt;</span><span class="token operator">?</span><span class="token operator"> = </span>null</div></div></div>
+                </div>
+              </div>
+            </div>
+          </div>
+<a data-name="-1941481328%2FProperties%2F537371977" anchor-label="singleLineComments" id="-1941481328%2FProperties%2F537371977" data-filterable-set=":core:dokkaHtmlPartial/main"></a>
+          <div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main">
+            <div class="main-subrow keyValue ">
+              <div class=""><span class="inline-flex">
+                  <div><a href="single-line-comments.html"><span>single</span><wbr><span>Line</span><wbr><span><span>Comments</span></span></a></div>
+<span class="anchor-wrapper"><span class="anchor-icon" pointing-to="-1941481328%2FProperties%2F537371977"></span>
+                    <div class="copy-popup-wrapper "><span class="copy-popup-icon"></span><span>Link copied to clipboard</span></div>
+                  </span></span></div>
+              <div>
+                <div class="title">
+                  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="single-line-comments.html">singleLineComments</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token operator">?</span><span class="token operator"> = </span>null</div></div></div>
+                </div>
+              </div>
+            </div>
+          </div>
 <a data-name="-1379014957%2FProperties%2F537371977" anchor-label="singleLineString" id="-1379014957%2FProperties%2F537371977" data-filterable-set=":core:dokkaHtmlPartial/main"></a>
           <div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main">
             <div class="main-subrow keyValue ">

docs/core/io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/integer-literals.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span>integer</span><wbr><span><span>Literals</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="integer-literals.html">integerLiterals</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L76">source</a>)</span></span></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">val </span><a href="integer-literals.html">integerLiterals</a><span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/internal/Lexer.kt#L78">source</a>)</span></span></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Copyright</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>