Mypy symbol resolve

jaclang/compiler/absyntree.py

@@ -632,6 +632,7 @@ def __init__(
         self.mod_deps: dict[str, Module] = {}
         self.registry = registry
         self.terminals: list[Token] = terminals
+        self.py_raised: bool = False
         AstNode.__init__(self, kid=kid)
         AstDocNode.__init__(self, doc=doc)
 

jaclang/compiler/passes/main/fuse_typeinfo_pass.py

@@ -10,7 +10,6 @@
 
 import jaclang.compiler.absyntree as ast
 from jaclang.compiler.passes import Pass
-from jaclang.compiler.symtable import SymbolTable
 from jaclang.settings import settings
 from jaclang.utils.helpers import pascal_to_snake
 from jaclang.vendor.mypy.nodes import Node as VNode  # bit of a hack
@@ -28,6 +27,7 @@ class FuseTypeInfoPass(Pass):
     """Python and bytecode file self.__debug_printing pass."""
 
     node_type_hash: dict[MypyNodes.Node | VNode, MyType] = {}
+    python_raise_list: set[tuple[str, str]] = set()
 
     def __debug_print(self, *argv: object) -> None:
         if settings.fuse_type_info_debug:
@@ -65,6 +65,36 @@ def __set_sym_table_link(self, node: ast.AstSymbolNode) -> None:
         if typ_sym_table != self.ir.sym_tab:
             node.name_spec.type_sym_tab = typ_sym_table
 
+    def __collect_python_dependencies(self, node: ast.AstNode) -> None:
+        from jaclang.compiler.passes.main.type_check_pass import JacTypeCheckPass
+
+        assert isinstance(node, ast.AstSymbolNode)
+
+        mypy_node = node.gen.mypy_ast[0]
+
+        if isinstance(mypy_node, MypyNodes.RefExpr):
+            node_full_name = mypy_node.node.fullname
+            if "." in node_full_name:
+                mod_name = node_full_name[: node_full_name.rindex(".")]
+            else:
+                mod_name = node_full_name
+
+            if mod_name not in JacTypeCheckPass.graph:
+                self.__debug_print(
+                    f"Can't find a python file associated with {type(node)}::{node.loc}"
+                )
+                return
+
+            mode_path = JacTypeCheckPass.graph[mod_name].xpath
+            if mode_path.endswith(".jac"):
+                return
+
+            FuseTypeInfoPass.python_raise_list.add((mod_name, mode_path))
+        else:
+            self.__debug_print(
+                f"Collect python dependencies is not supported in {type(node)}::{node.loc}"
+            )
+
     @staticmethod
     def __handle_node(
         func: Callable[[FuseTypeInfoPass, T], None]
@@ -86,6 +116,7 @@ def node_handler(self: FuseTypeInfoPass, node: T) -> None:
                 if len(node.gen.mypy_ast) == 1:
                     func(self, node)
                     self.__set_sym_table_link(node)
+                    self.__collect_python_dependencies(node)
 
                 # Jac node has multiple mypy nodes linked to it
                 elif len(node.gen.mypy_ast) > 1:
@@ -109,6 +140,7 @@ def node_handler(self: FuseTypeInfoPass, node: T) -> None:
                         )
                         func(self, node)
                         self.__set_sym_table_link(node)
+                        self.__collect_python_dependencies(node)
 
                 # Jac node doesn't have mypy nodes linked to it
                 else:
@@ -446,20 +478,16 @@ def exit_assignment(self, node: ast.Assignment) -> None:
                 if self_obj.type_sym_tab and isinstance(right_obj, ast.AstSymbolNode):
                     self_obj.type_sym_tab.def_insert(right_obj)
 
-    def exit_name(self, node: ast.Name) -> None:
-        """Add new symbols in the symbol table in case of atom trailer."""
-        if isinstance(node.parent, ast.AtomTrailer):
-            target_node = node.parent.target
-            if isinstance(target_node, ast.AstSymbolNode):
-                parent_symbol_table = target_node.type_sym_tab
-                if isinstance(parent_symbol_table, SymbolTable):
-                    node.sym = parent_symbol_table.lookup(node.sym_name)
-
-    # def exit_in_for_stmt(self, node: ast.InForStmt):
-    #     print(node.loc.mod_path, node.loc)
-    #     print(node.target, node.target.loc)
-    #     # node.sym_tab.def_insert()
-    #     # exit()
-
-    # def after_pass(self) -> None:
-    #     exit()
+    def exit_atom_trailer(self, node: ast.AtomTrailer) -> None:
+        """Adding symbol links to AtomTrailer right nodes."""
+        # This will fix adding the symbol links to nodes in atom trailer
+        # self.x.z = 5  # will add symbol links to both x and z
+        for i in range(1, len(node.as_attr_list)):
+            left = node.as_attr_list[i - 1]
+            right = node.as_attr_list[i]
+            # assert isinstance(left, ast.NameAtom)
+            # assert isinstance(right, ast.NameAtom)
+            if left.type_sym_tab and not isinstance(
+                right, ast.IndexSlice
+            ):  # TODO check why IndexSlice produce an issue
+                right.name_spec.sym = left.type_sym_tab.lookup(right.sym_name)

jaclang/compiler/passes/main/import_pass.py

@@ -6,17 +6,13 @@
 """
 
 import ast as py_ast
-import importlib.util
 import os
-import sys
 from typing import Optional
 
 
 import jaclang.compiler.absyntree as ast
 from jaclang.compiler.passes import Pass
 from jaclang.compiler.passes.main import SubNodeTabPass
-from jaclang.settings import settings
-from jaclang.utils.helpers import is_standard_lib_module
 from jaclang.utils.log import logging
 
 logger = logging.getLogger(__name__)
@@ -213,59 +209,58 @@ class PyImportPass(JacImportPass):
 
     def before_pass(self) -> None:
         """Only run pass if settings are set to raise python."""
+        from jaclang.compiler.passes.main.fuse_typeinfo_pass import FuseTypeInfoPass
+
         super().before_pass()
-        if not settings.py_raise:
-            self.terminate()
 
-    def process_import(self, node: ast.Module, i: ast.ModulePath) -> None:
-        """Process an import."""
-        imp_node = i.parent_of_type(ast.Import)
-        if (
-            imp_node.is_py
-            and not i.sub_module
-            and not is_standard_lib_module(i.path_str)
-        ):
-            mod = self.import_py_module(node=i, mod_path=node.loc.mod_path)
-            if mod:
-                i.sub_module = mod
-                i.add_kids_right([mod], pos_update=False)
-                if settings.py_raise_deep:
-                    self.run_again = True
+        raised_modules: dict = {"Children": {}}
+        python_raise_list = list(FuseTypeInfoPass.python_raise_list)
+        python_raise_list.sort(key=lambda x: x[0], reverse=False)
 
-    def import_py_module(
-        self, node: ast.ModulePath, mod_path: str
-    ) -> Optional[ast.Module]:
+        for mod_name, mod_path in python_raise_list:
+            o = self.import_py_module(mod_path)
+            if o is None:
+                continue
+            if "." not in mod_name:
+                o.name = mod_name
+                self.attach_mod_to_node(self.ir, o)
+                raised_modules["Children"][o.name] = {"Node": o, "Children": {}}
+            else:
+                mod_parents = mod_name[: mod_name.rindex(".")].split(".")
+                parent = raised_modules
+                for m in mod_parents:
+                    print(m, raised_modules)
+                    if m not in parent["Children"]:
+                        raise Exception("Parent module should be raised before")
+                    else:
+                        parent = parent["Children"][m]
+                o.name = mod_name[mod_name.rindex(".") + 1 :]
+                self.attach_mod_to_node(parent["Node"], o)
+                parent["Children"][o.name] = {"Node": o, "Children": {}}
+
+        self.terminate()
+
+    def import_py_module(self, mod_path: str) -> Optional[ast.Module]:
         """Import a module."""
         from jaclang.compiler.passes.main import PyastBuildPass
 
-        base_dir = os.path.dirname(mod_path)
-        sys.path.append(base_dir)
         try:
-            # Dynamically import the module
-            spec = importlib.util.find_spec(node.path_str)
-            sys.path.remove(base_dir)
-            if spec and spec.origin and spec.origin not in {None, "built-in", "frozen"}:
-                if spec.origin in self.import_table:
-                    return self.import_table[spec.origin]
-                with open(spec.origin, "r", encoding="utf-8") as f:
+            if mod_path not in {None, "built-in", "frozen"}:
+                if mod_path in self.import_table:
+                    return self.import_table[mod_path]
+                with open(mod_path, "r", encoding="utf-8") as f:
                     mod = PyastBuildPass(
                         input_ir=ast.PythonModuleAst(
-                            py_ast.parse(f.read()), mod_path=spec.origin
+                            py_ast.parse(f.read()), mod_path=mod_path
                         ),
                     ).ir
                 if mod:
-                    self.import_table[spec.origin] = mod
+                    mod.py_raised = True
+                    self.import_table[mod_path] = mod
                     return mod
                 else:
-                    raise self.ice(
-                        f"Failed to import python module {node.path_str}: {spec.origin}"
-                    )
+                    raise self.ice(f"Failed to import python module {mod_path}")
         except Exception as e:
-            if "Empty kid for Token ModulePath" in str(e) or "utf-8" in str(e):  # FIXME
-                return None
-            self.error(
-                f"Failed to import python module {node.path_str}: {e}",
-                node_override=node,
-            )
+            self.error(f"Failed to import python module {mod_path}")
             raise e
         return None

jaclang/compiler/passes/main/schedules.py

@@ -33,6 +33,12 @@
     PyBytecodeGenPass,
 ]
 
-type_checker_sched = [JacTypeCheckPass, FuseTypeInfoPass, AccessCheckPass]
+type_checker_sched = [
+    JacTypeCheckPass,
+    FuseTypeInfoPass,
+    PyImportPass,
+    SymTabBuildPass,
+    AccessCheckPass,
+]
 py_code_gen_typed = [*py_code_gen, *type_checker_sched]
 py_compiler = [*py_code_gen, PyOutPass]

jaclang/compiler/passes/main/type_check_pass.py

@@ -18,6 +18,8 @@
 class JacTypeCheckPass(Pass):
     """Python and bytecode file printing pass."""
 
+    graph: myab.Graph = {}
+
     def before_pass(self) -> None:
         """Before pass."""
         self.__path = (
@@ -101,7 +103,7 @@ def api(self) -> None:
             mypy_graph[module.name] = st
             new_modules.append(st)
 
-        graph = myab.load_graph(
+        JacTypeCheckPass.graph = myab.load_graph(
             [
                 myab.BuildSource(
                     path=str(self.__path / "typeshed" / "stdlib" / "builtins.pyi"),
@@ -112,4 +114,4 @@ def api(self) -> None:
             old_graph=mypy_graph,
             new_modules=new_modules,  # To parse the dependancies of modules
         )
-        myab.process_graph(graph, manager)
+        myab.process_graph(JacTypeCheckPass.graph, manager)

jaclang/utils/treeprinter.py

@@ -116,6 +116,8 @@ def __node_repr_in_tree(node: AstNode) -> str:
             return out
         elif isinstance(node, Token):
             return f"{node.__class__.__name__} - {node.value}, {access}"
+        elif isinstance(node, ast.Module) and node.py_raised:
+            return f"{node.__class__.__name__} - PythonModuleRaised: {node.name}"
         elif isinstance(node, AstSymbolNode):
             out = (
                 f"{node.__class__.__name__} - {node.sym_name} - "
@@ -184,11 +186,13 @@ def mapper(draw: bool) -> str:
 
     if isinstance(root, ast.AstNode):
         tree_str = f"{root.loc}\t{markers}{__node_repr_in_tree(root)}\n"
-        for i, child in enumerate(root.kid):
-            is_last = i == len(root.kid) - 1
-            tree_str += print_ast_tree(
-                child, marker, [*level_markers, not is_last], output_file, max_depth
-            )
+        if not (isinstance(root, ast.Module) and root.py_raised):
+            tree_str = f"{root.loc}\t{markers}{__node_repr_in_tree(root)}\n"
+            for i, child in enumerate(root.kid):
+                is_last = i == len(root.kid) - 1
+                tree_str += print_ast_tree(
+                    child, marker, [*level_markers, not is_last], output_file, max_depth
+                )
     elif isinstance(root, ast3.AST):
         tree_str = (
             f"{get_location_info(root)}\t{markers}{__node_repr_in_py_tree(root)}\n"