feat: added identifiers to the dsl

core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt

@@ -20,10 +20,15 @@ class LexerBuilder internal constructor() {
   internal var multilineComments: Pair<Regex, Regex>? = Lexer.Defaults.multilineComments
     private set
 
+  internal var identifiers: Regex = Lexer.Defaults.identifiers
+    private set
+
   /**
    * The lexer will skip over any strings that match this regex.
    * This acts like a token separator.
    *
+   * ### Usage
+   *
    * ```kt
    * val myLexer = lexer {
    *   ignorePatterns(Regex("""\s+"""))
@@ -44,6 +49,8 @@ class LexerBuilder internal constructor() {
    * When the lexer identifies a [singleLineComments] pattern, it will skip to the next line
    * and return the next token.
    *
+   * ### Usage
+   *
    * ```kt
    * val myLexer = lexer {
    *   singleLineComments(Regex("//"))
@@ -65,14 +72,16 @@ class LexerBuilder internal constructor() {
    * When the lexer identifies the starting pattern, it will continue to skip to the next possible token
    * until it meets the ending pattern.
    *
+   * ### Usage
+   *
    * ```kt
    * val myLexer = lexer {
    *   // You don't need the square brackets but KDoc doesn't like it...
    *   multilineComments(Regex("[/][*]") to Regex("[*][/]"))
    * }
    * ```
    *
-   * @param multilineComments regex of the pattern the lexer will skip over to the next line.
+   * @param multilineComments pair of regexes that define what contents are under comments.
    *
    * @see Lexer.Defaults.multilineComments
    * @author Nishant Aanjaney Jalan
@@ -81,6 +90,30 @@ class LexerBuilder internal constructor() {
   fun multilineComments(multilineComments: Pair<Regex, Regex>) {
     this.multilineComments = multilineComments
   }
+
+  /**
+   * Supply the regex pattern that defines the rules for identifiers.
+   * Identifiers are parts of the program that are named by the user;
+   * for instance, the field name or class name.
+   *
+   * ### Usage
+   *
+   * ```kt
+   * val myLexer = lexer {
+   *   // If all identifiers must be at least 2 characters and start with a lowercase english alphabet.
+   *   identifiers(Regex("[a-z][a-zA-Z0-9]+"))
+   * }
+   * ```
+   *
+   * @param identifiers regex defining the rules of the naming identifiers
+   *
+   * @see Lexer.Defaults.multilineComments
+   * @author Nishant Aanjaney Jalan
+   * @since 0.2.0
+   */
+  fun identifiers(identifiers: Regex) {
+    this.identifiers = identifiers
+  }
 }
 
 /**
@@ -108,6 +141,7 @@ fun lexer(init: LexerBuilder.() -> Unit): Lexer {
   return Lexer(
     ignorePattern = builder.ignorePattern,
     singleLineComments = builder.singleLineComments,
-    multilineComments = builder.multilineComments
+    multilineComments = builder.multilineComments,
+    identifiers = builder.identifiers
   )
 }

core/src/test/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilderTest.kt

@@ -9,7 +9,7 @@ import org.junit.jupiter.api.Test
 class LexerBuilderTest {
   @Test
   fun `initialise a default lexer`() {
-    val myLexer = lexer {}
+    lexer {}
   }
 
   @Test
@@ -59,5 +59,21 @@ class LexerBuilderTest {
       )
     )
   }
+
+  @Test
+  fun `sets identifiers rule`() {
+    val myLexer = lexer {
+      identifiers(Regex("[a-z][a-zA-Z0-9]+"))
+    }
+
+    myLexer.source = StringSource("hi value")
+    assertTokens(
+      myLexer,
+      listOf(
+        Token.Identifier("hi", Position(0, 0), Position(0, 1)),
+        Token.Identifier("value", Position(0, 3), Position(0, 7))
+      )
+    )
+  }
 }
 

docs/core/io.github.cybercodernaj.parkour.lexer/-lexer-builder/multiline-comments.html

@@ -63,7 +63,7 @@
   <div class="cover ">
     <h1 class="cover"><span>multiline</span><wbr><span><span>Comments</span></span></h1>
   </div>
-  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="multiline-comments.html"><span class="token function">multilineComments</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter ">multilineComments<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-pair/index.html">Pair</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token punctuation">, </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token operator">&gt;</span></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L81">source</a>)</span></span></div><p class="paragraph">There are two parts to <a href="multiline-comments.html">multilineComments</a>: the starting and the ending pattern. When the lexer identifies the starting pattern, it will continue to skip to the next possible token until it meets the ending pattern.</p><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  // You don't need the square brackets but KDoc doesn't like it...<br>  multilineComments(Regex("[/][*]") to Regex("[*][/]"))<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span>single</span><wbr><span>Line</span><wbr><span><span>Comments</span></span></u></div></span></div><div><div class="title"><p class="paragraph">regex of the pattern the lexer will skip over to the next line.</p></div></div></div></div></div><h4 class="">See also</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><a href="../../io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/single-line-comments.html"><span>Lexer.</span><wbr><span>Defaults.</span><wbr><span>single</span><wbr><span>Line</span><wbr><span><span>Comments</span></span></a></div></span></div><div></div></div></div></div></div></div>
+  <div class="platform-hinted " data-platform-hinted="data-platform-hinted"><div class="content sourceset-dependent-content" data-active="" data-togglable=":core:dokkaHtmlPartial/main"><div class="symbol monospace"><span class="token keyword">fun </span><a href="multiline-comments.html"><span class="token function">multilineComments</span></a><span class="token punctuation">(</span><span class="parameters "><span class="parameter ">multilineComments<span class="token operator">: </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin/-pair/index.html">Pair</a><span class="token operator">&lt;</span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token punctuation">, </span><a href="https://kotlinlang.org/api/latest/jvm/stdlib/kotlin.text/-regex/index.html">Regex</a><span class="token operator">&gt;</span></span></span><span class="token punctuation">)</span><span class="clearfix"><span class="floating-right">(<a href="https://github.com/cybercoder-naj/parkour/tree/main/core/src/main/kotlin/io/github/cybercodernaj/parkour/lexer/LexerBuilder.kt#L81">source</a>)</span></span></div><p class="paragraph">There are two parts to <a href="multiline-comments.html">multilineComments</a>: the starting and the ending pattern. When the lexer identifies the starting pattern, it will continue to skip to the next possible token until it meets the ending pattern.</p><div class="sample-container"><pre><code class="block lang-kt" theme="idea">val myLexer = lexer {<br>  // You don't need the square brackets but KDoc doesn't like it...<br>  multilineComments(Regex("[/][*]") to Regex("[*][/]"))<br>}</code></pre><span class="top-right-position"><span class="copy-icon"></span><div class="copy-popup-wrapper popup-to-left"><span class="copy-popup-icon"></span><span>Content copied to clipboard</span></div></span></div><span class="kdoc-tag"><h4 class="">Author</h4><p class="paragraph">Nishant Aanjaney Jalan</p></span><span class="kdoc-tag"><h4 class="">Since</h4><p class="paragraph">0.2.0</p></span><h4 class="">Parameters</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><u><span>multiline</span><wbr><span><span>Comments</span></span></u></div></span></div><div><div class="title"><p class="paragraph">regex of the pattern the lexer will skip over to the next line.</p></div></div></div></div></div><h4 class="">See also</h4><div class="table"><div class="table-row" data-filterable-current=":core:dokkaHtmlPartial/main" data-filterable-set=":core:dokkaHtmlPartial/main"><div class="main-subrow keyValue "><div class=""><span class="inline-flex"><div><a href="../../io.github.cybercodernaj.parkour.lexer.internal/-lexer/-defaults/multiline-comments.html"><span>Lexer.</span><wbr><span>Defaults.</span><wbr><span>multiline</span><wbr><span><span>Comments</span></span></a></div></span></div><div></div></div></div></div></div></div>
 </div>
       <div class="footer">
         <span class="go-to-top-icon"><a href="#content" id="go-to-top-link"></a></span><span>© 2024 Nishant Aanjaney Jalan</span><span class="pull-right"><span>Generated by </span><a href="https://github.com/Kotlin/dokka"><span>dokka</span><span class="padded-icon"></span></a></span>