Merge pull request #91 from Jaseci-Labs/comments_v2

New comment approach

jaclang/jac/absyntree.py

@@ -5,7 +5,6 @@
 import pprint
 from typing import Any, Callable, Generic, Optional, Sequence, Type, TypeVar
 
-from jaclang.jac import jac_lark as jl
 from jaclang.jac.codeloc import CodeGenTarget, CodeLocInfo
 from jaclang.jac.constant import Constants as Con, EdgeDir
 from jaclang.jac.constant import Tokens as Tok
@@ -2153,6 +2152,35 @@ class Semi(Token, CodeBlockStmt):
     """Semicolon node type for Jac Ast."""
 
 
+class CommentToken(Token):
+    """CommentToken node type for Jac Ast."""
+
+    def __init__(
+        self,
+        file_path: str,
+        name: str,
+        value: str,
+        line: int,
+        col_start: int,
+        col_end: int,
+        pos_start: int,
+        pos_end: int,
+        kid: Sequence[AstNode],
+        is_inline: bool = False,
+    ) -> None:
+        """Initialize token."""
+        self.file_path = file_path
+        self.name = name
+        self.value = value
+        self.line_no = line
+        self.c_start = col_start
+        self.c_end = col_end
+        self.pos_start = pos_start
+        self.pos_end = pos_end
+        self.is_inline = is_inline
+        AstNode.__init__(self, kid=kid)
+
+
 # ----------------
 class JacSource(EmptyToken):
     """SourceString node type for Jac Ast."""
@@ -2162,7 +2190,7 @@ def __init__(self, source: str, mod_path: str) -> None:
         super().__init__()
         self.value = source
         self.file_path = mod_path
-        self.comments: list[jl.Token] = []
+        self.comments: list[CommentToken] = []
 
     @property
     def code(self) -> str:

jaclang/jac/parser.py

@@ -30,10 +30,11 @@ def __init__(self, input_ir: ast.JacSource) -> None:
     def transform(self, ir: ast.AstNode) -> ast.AstNode:
         """Transform input IR."""
         try:
-            tree, self.source.comments = JacParser.parse(
+            tree, comments = JacParser.parse(
                 self.source.value, on_error=self.error_callback
             )
             mod = JacParser.TreeToAST(parser=self).transform(tree)
+            self.source.comments = [self.proc_comment(i, mod) for i in comments]
         except jl.UnexpectedInput as e:
             catch_error = ast.EmptyToken()
             catch_error.file_path = self.mod_path
@@ -47,6 +48,21 @@ def transform(self, ir: ast.AstNode) -> ast.AstNode:
             self.error(f"Internal Error: {e}")
         return mod
 
+    @staticmethod
+    def proc_comment(token: jl.Token, mod: ast.AstNode) -> ast.CommentToken:
+        """Process comment."""
+        return ast.CommentToken(
+            file_path=mod.loc.mod_path,
+            name=token.type,
+            value=token.value,
+            line=token.line if token.line is not None else 0,
+            col_start=token.column if token.column is not None else 0,
+            col_end=token.end_column if token.end_column is not None else 0,
+            pos_start=token.start_pos if token.start_pos is not None else 0,
+            pos_end=token.end_pos if token.end_pos is not None else 0,
+            kid=[],
+        )
+
     def error_callback(self, e: jl.UnexpectedInput) -> bool:
         """Handle error."""
         return False
@@ -1068,7 +1084,7 @@ def statement(self, kid: list[ast.AstNode]) -> ast.CodeBlockStmt:
                     kid=kid,
                 )
             elif isinstance(kid[0], ast.CodeBlockStmt):
-                kid[0].add_kids_right([kid[1]], pos_update=False)
+                kid[0].add_kids_right([kid[1]])
                 return self.nu(kid[0])
             else:
                 raise self.ice()

jaclang/jac/passes/main/__init__.py

@@ -5,7 +5,6 @@
 from .def_impl_match_pass import DeclDefMatchPass  # noqa: I100
 from .def_use_pass import DefUsePass  # noqa: I100
 from .pyout_pass import PyOutPass  # noqa: I100
-from .jac_formatter_pass import JacFormatPass  # noqa: I100
 from .pyast_load_pass import PyastBuildPass  # noqa: I100
 from .pyast_gen_pass import PyastGenPass  # noqa: I100
 from .schedules import py_code_gen  # noqa: I100
@@ -20,7 +19,6 @@
     "DeclDefMatchPass",
     "DefUsePass",
     "PyOutPass",
-    "JacFormatPass",
     "PyastBuildPass",
     "PyastGenPass",
 ]

jaclang/jac/passes/main/pyast_gen_pass.py

@@ -2333,3 +2333,15 @@ def exit_semi(self, node: ast.Semi) -> None:
         pos_start: int,
         pos_end: int,
         """
+
+    def exit_comment_token(self, node: ast.CommentToken) -> None:
+        """Sub objects.
+
+        file_path: str,
+        name: str,
+        value: str,
+        col_start: int,
+        col_end: int,
+        pos_start: int,
+        pos_end: int,
+        """

jaclang/jac/passes/main/schedules.py

@@ -4,16 +4,13 @@
 """
 from __future__ import annotations
 
-from typing import Type
 
-from ..ir_pass import Pass
 from .sub_node_tab_pass import SubNodeTabPass  # noqa: I100
 from .import_pass import ImportPass  # noqa: I100
 from .sym_tab_build_pass import SymTabBuildPass  # noqa: I100
 from .def_impl_match_pass import DeclDefMatchPass  # noqa: I100
 from .def_use_pass import DefUsePass  # noqa: I100
 from .pyout_pass import PyOutPass  # noqa: I100
-from .jac_formatter_pass import JacFormatPass  # noqa: I100
 from .pyast_gen_pass import PyastGenPass  # noqa: I100
 
 py_code_gen = [
@@ -29,5 +26,3 @@
     *py_code_gen,
     PyOutPass,
 ]
-
-format_pass: list[Type[Pass]] = [JacFormatPass]

jaclang/jac/passes/main/sym_tab_build_pass.py

@@ -1366,3 +1366,7 @@ def enter_builtin_type(self, node: ast.BuiltinType) -> None:
     def enter_semi(self, node: ast.Semi) -> None:
         """Sub objects."""
         self.sync_node_to_scope(node)
+
+    def enter_comment_token(self, node: ast.CommentToken) -> None:
+        """Sub objects."""
+        self.sync_node_to_scope(node)

jaclang/jac/passes/main/tests/test_jac_format_pass.py

@@ -1,9 +1,11 @@
 """Test ast build pass module."""
 import ast as ast3
 
-from jaclang.jac.passes.main import JacFormatPass
+
 from jaclang.jac.passes.main import PyastGenPass
 from jaclang.jac.passes.main.schedules import py_code_gen as without_format
+from jaclang.jac.passes.tool import JacFormatPass
+from jaclang.jac.passes.tool.schedules import format_pass
 from jaclang.jac.transpiler import jac_file_to_pass, jac_str_to_pass
 from jaclang.utils.test import AstSyncTestMixin, TestCaseMicroSuite
 
@@ -38,7 +40,7 @@ def micro_suite_test(self, filename: str) -> None:
             target=PyastGenPass,
             schedule=without_format,
         )
-        with_format = [JacFormatPass]
+        with_format = format_pass
         code_gen_format = jac_file_to_pass(
             self.fixture_abs_path(filename), schedule=with_format
         )
@@ -52,10 +54,17 @@ def micro_suite_test(self, filename: str) -> None:
         for i in range(len(code_gen_pure.ir.gen.py.split("\n"))):
             if "test_" in code_gen_pure.ir.gen.py.split("\n")[i]:
                 continue
-            self.assertEqual(
-                ast3.dump(code_gen_pure.ir.gen.py_ast, indent=2),
-                ast3.dump(code_gen_jac.ir.gen.py_ast, indent=2),
-            )
+            try:
+                self.assertEqual(
+                    ast3.dump(code_gen_pure.ir.gen.py_ast, indent=2),
+                    ast3.dump(code_gen_jac.ir.gen.py_ast, indent=2),
+                )
+            except Exception as e:
+                from jaclang.utils.helpers import add_line_numbers
+
+                print(add_line_numbers(code_gen_pure.ir.source.code))
+                print(add_line_numbers(code_gen_format.ir.gen.jac))
+                raise e
 
 
 JacFormatPassTests.self_attach_micro_tests()

jaclang/jac/passes/tool/__init__.py

@@ -4,11 +4,12 @@
     SymbolTableDotGraphPass,
     SymbolTablePrinterPass,
 )
-
+from .jac_formatter_pass import JacFormatPass  # noqa: I100
 
 __all__ = [
     "AstDotGraphPass",
     "AstPrinterPass",
     "SymbolTablePrinterPass",
     "SymbolTableDotGraphPass",
+    "JacFormatPass",
 ]

jaclang/jac/passes/tool/fuse_comments_pass.py

@@ -0,0 +1,68 @@
+"""JacFormatPass for Jaseci Ast.
+
+This is a pass for formatting Jac code.
+"""
+
+import jaclang.jac.absyntree as ast
+from jaclang.jac.passes import Pass
+
+
+class FuseCommentsPass(Pass):
+    """JacFormat Pass format Jac code."""
+
+    def before_pass(self) -> None:
+        """Before pass."""
+        self.all_tokens: list[ast.Token] = []
+        self.comments: list[ast.Token] = (
+            self.ir.source.comments if isinstance(self.ir, ast.Module) else []
+        )
+        return super().before_pass()
+
+    def exit_node(self, node: ast.AstNode) -> None:
+        """Exit node."""
+        if isinstance(node, ast.Token):
+            self.all_tokens.append(node)
+
+    def after_pass(self) -> None:
+        """Insert comment tokens into all_tokens."""
+        chomp: list[ast.CommentToken] = [*self.comments]
+        new_tokens: list[ast.Token] = []
+        if not len(chomp):
+            return
+        for i in range(len(self.all_tokens)):
+            chomp[0].is_inline = chomp[0].is_inline or (
+                self.all_tokens[i].loc.first_line == chomp[0].loc.first_line
+            )
+            if i == len(self.all_tokens) - 1:
+                if len(chomp):
+                    new_tokens.append(self.all_tokens[i])
+                    new_tokens += chomp
+                break
+            while (i < len(self.all_tokens) - 1) and (
+                (
+                    self.all_tokens[i].loc.first_line == chomp[0].loc.first_line
+                    and self.all_tokens[i].loc.col_start > chomp[0].loc.col_start
+                )
+                or (self.all_tokens[i].loc.first_line > chomp[0].loc.first_line)
+            ):
+                new_tokens.append(chomp[0])
+                chomp = chomp[1:]
+                if not len(chomp):
+                    new_tokens.extend(self.all_tokens[i + 1 :])
+                    break
+            new_tokens.append(self.all_tokens[i])
+            if not len(chomp):
+                break
+        for i in range(len(new_tokens)):
+            if isinstance(new_tokens[i], ast.CommentToken):
+                if i == 0:
+                    self.ir.add_kids_left([new_tokens[i]])
+                    continue
+                new_val = new_tokens[i - 1]
+                if new_val.parent is not None:
+                    new_kids = new_val.parent.kid
+                    new_kids.insert(new_kids.index(new_val) + 1, new_tokens[i])
+                    new_val.parent.set_kids(new_kids)
+                else:
+                    new_val.print()
+                    raise self.ice("Token without parent in AST should be impossible")