Fix mypy compliance in Ghidra scripts

pyproject.toml

@@ -7,5 +7,7 @@ max-line-length = 120
 
 [tool.mypy]
 check_untyped_defs = true
-# excluded until fixed
-exclude = "reccmp/ghidra_scripts/.*"
+
+[[tool.mypy.overrides]]
+module = ["ghidra", "ghidra.*"]
+ignore_missing_imports = true

reccmp/ghidra_scripts/import_functions_and_types_from_pdb.py

@@ -31,8 +31,7 @@
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
-    import ghidra
-    from lego_util.headers import *  # pylint: disable=wildcard-import # these are just for headers
+    from reccmp.ghidra_scripts.lego_util.headers import *  # pylint: disable=wildcard-import # these are just for headers
 
 
 logger = logging.getLogger(__name__)
@@ -50,9 +49,45 @@ def reload_module(module: str):
     importlib.reload(importlib.import_module(module))
 
 
-reload_module("lego_util.statistics")
-reload_module("lego_util.globals")
-from lego_util.globals import GLOBALS
+def add_python_path(path: Path):
+    """
+    Scripts in Ghidra are executed from the tools/ghidra_scripts directory. We need to add
+    a few more paths to the Python path so we can import the other libraries.
+    """
+    logger.info("Adding %s to Python Path", path)
+    assert path.exists()
+    sys.path.insert(1, str(path))
+
+
+def find_and_add_venv_to_pythonpath():
+    path = Path(__file__).resolve()
+
+    # Add the virtual environment if we are in one, e.g. `.venv/Lib/site-packages/reccmp/ghidra_scripts/import_[...].py`
+    while not path.is_mount():
+        if path.name == "site-packages":
+            add_python_path(path)
+            return
+        path = path.parent
+
+    # Development setup: Running from the reccmp repository. The dependencies must be installed in a venv with name `.venv`.
+
+    # This one is needed when the reccmp project is installed in editable mode and we are running directly from the source
+    add_python_path(Path(__file__).parent.parent.parent)
+
+    # Now we add the virtual environment where the dependencies need to be installed
+    path = Path(__file__).resolve()
+    while not path.is_mount():
+        venv_candidate = path / ".venv"
+        if venv_candidate.exists():
+            site_packages = next(venv_candidate.glob("lib/**/site-packages/"), None)
+            if site_packages is not None:
+                add_python_path(site_packages)
+                return
+        path = path.parent
+
+    logger.warning(
+        "No virtual environment was found. This script might fail to find dependencies."
+    )
 
 
 def setup_logging():
@@ -61,7 +96,6 @@ def setup_logging():
     # formatter = logging.Formatter("%(name)s %(levelname)-8s %(message)s") # use this to identify loggers
     stdout_handler = logging.StreamHandler(sys.stdout)
     stdout_handler.setFormatter(formatter)
-    logging.root.setLevel(GLOBALS.loglevel)
     logging.root.addHandler(stdout_handler)
 
     logger.info("Starting import...")
@@ -70,6 +104,13 @@ def setup_logging():
 # This script can be run both from Ghidra and as a standalone.
 # In the latter case, only the PDB parser will be used.
 setup_logging()
+find_and_add_venv_to_pythonpath()
+reload_module("reccmp.ghidra_scripts.lego_util.statistics")
+reload_module("reccmp.ghidra_scripts.lego_util.globals")
+from reccmp.ghidra_scripts.lego_util.globals import GLOBALS
+
+logging.root.setLevel(GLOBALS.loglevel)
+
 try:
     from ghidra.program.flatapi import FlatProgramAPI
     from ghidra.util.exception import CancelledException
@@ -90,16 +131,6 @@ def get_repository_root():
     return Path(__file__).absolute().parent.parent.parent
 
 
-def add_python_path(path: Path):
-    """
-    Scripts in Ghidra are executed from the tools/ghidra_scripts directory. We need to add
-    a few more paths to the Python path so we can import the other libraries.
-    """
-    logger.info("Adding %s to Python Path", path)
-    assert path.exists()
-    sys.path.insert(1, str(path))
-
-
 # We need to quote the types here because they might not exist when running without Ghidra
 def import_function_into_ghidra(
     api: "FlatProgramAPI",
@@ -207,37 +238,6 @@ def log_and_track_failure(
         )
 
 
-def find_and_add_venv_to_pythonpath():
-    path = Path(__file__).resolve()
-
-    # Add the virtual environment if we are in one, e.g. `.venv/Lib/site-packages/reccmp/ghidra_scripts/import_[...].py`
-    while not path.is_mount():
-        if path.name == "site-packages":
-            add_python_path(path)
-            return
-        path = path.parent
-
-    # Development setup: Running from the reccmp repository. The dependencies must be installed in a venv with name `.venv`.
-
-    # This one is needed when the reccmp project is installed in editable mode and we are running directly from the source
-    add_python_path(Path(__file__).parent.parent.parent)
-
-    # Now we add the virtual environment where the dependencies need to be installed
-    path = Path(__file__).resolve()
-    while not path.is_mount():
-        venv_candidate = path / ".venv"
-        if venv_candidate.exists():
-            site_packages = next(venv_candidate.glob("lib/**/site-packages/"), None)
-            if site_packages is not None:
-                add_python_path(site_packages)
-                return
-        path = path.parent
-
-    logger.warning(
-        "No virtual environment was found. This script might fail to find dependencies."
-    )
-
-
 def find_build_target() -> "RecCmpBuiltTarget":
     """
     Known issue: In order to use this script, `reccmp-build.yml` must be located in the same directory as `reccmp-project.yml`.
@@ -338,9 +338,7 @@ def main():
 # sys.path is not reset after running the script, so we should restore it
 sys_path_backup = sys.path.copy()
 try:
-    find_and_add_venv_to_pythonpath()
-
-    import setuptools  # pylint: disable=unused-import # required to fix a distutils issue in Python 3.12
+    import setuptools  # type: ignore[import-untyped] # pylint: disable=unused-import # required to fix a distutils issue in Python 3.12
 
     # Packages are imported down here because reccmp's dependencies are only available after the venv was added to the pythonpath
     reload_module("reccmp.project.detect")
@@ -359,25 +357,29 @@ def main():
     reload_module("reccmp.isledecomp.compare.db")
 
     reload_module("lego_util.exceptions")
-    from lego_util.exceptions import Lego1Exception
+    from reccmp.ghidra_scripts.lego_util.exceptions import Lego1Exception
 
     reload_module("lego_util.pdb_extraction")
-    from lego_util.pdb_extraction import (
+    from reccmp.ghidra_scripts.lego_util.pdb_extraction import (
         PdbFunctionExtractor,
         PdbFunction,
     )
 
     reload_module("lego_util.vtable_importer")
-    from lego_util.vtable_importer import import_vftables_into_ghidra
+    from reccmp.ghidra_scripts.lego_util.vtable_importer import (
+        import_vftables_into_ghidra,
+    )
 
     if GLOBALS.running_from_ghidra:
         reload_module("lego_util.ghidra_helper")
 
         reload_module("lego_util.function_importer")
-        from lego_util.function_importer import PdbFunctionImporter
+        from reccmp.ghidra_scripts.lego_util.function_importer import (
+            PdbFunctionImporter,
+        )
 
         reload_module("lego_util.type_importer")
-        from lego_util.type_importer import PdbTypeImporter
+        from reccmp.ghidra_scripts.lego_util.type_importer import PdbTypeImporter
 
     if __name__ == "__main__":
         main()

reccmp/ghidra_scripts/lego_util/function_importer.py

@@ -17,19 +17,19 @@
     ComponentOffsetSettingsDefinition,
 )
 
-from lego_util.pdb_extraction import (
+from .pdb_extraction import (
     PdbFunction,
     CppRegisterSymbol,
     CppStackSymbol,
 )
-from lego_util.ghidra_helper import (
+from .ghidra_helper import (
     add_data_type_or_reuse_existing,
     get_namespace_and_name,
     get_or_add_pointer_type,
 )
 
-from lego_util.exceptions import StackOffsetMismatchError, Lego1Exception
-from lego_util.type_importer import PdbTypeImporter
+from .exceptions import StackOffsetMismatchError, Lego1Exception
+from .type_importer import PdbTypeImporter
 
 logger = logging.getLogger(__name__)
 

reccmp/ghidra_scripts/lego_util/ghidra_helper.py

@@ -3,20 +3,21 @@
 import logging
 import re
 
-from lego_util.exceptions import (
-    ClassOrNamespaceNotFoundInGhidraError,
-    TypeNotFoundInGhidraError,
-    MultipleTypesFoundInGhidraError,
-)
-from lego_util.globals import GLOBALS
-
 # Disable spurious warnings in vscode / pylance
 # pyright: reportMissingModuleSource=false
 
 from ghidra.program.flatapi import FlatProgramAPI
 from ghidra.program.model.data import DataType, DataTypeConflictHandler, PointerDataType
 from ghidra.program.model.symbol import Namespace
 
+from .exceptions import (
+    ClassOrNamespaceNotFoundInGhidraError,
+    TypeNotFoundInGhidraError,
+    MultipleTypesFoundInGhidraError,
+)
+from .globals import GLOBALS
+
+
 logger = logging.getLogger(__name__)
 
 

reccmp/ghidra_scripts/lego_util/globals.py

@@ -1,7 +1,7 @@
 import logging
 from enum import Enum
 from dataclasses import dataclass, field
-from lego_util.statistics import Statistics
+from .statistics import Statistics
 
 
 class SupportedModules(Enum):

reccmp/ghidra_scripts/lego_util/pdb_extraction.py

@@ -65,7 +65,7 @@ def __init__(self, compare: IsleCompare):
         "STD Near": "__stdcall",
     }
 
-    def _get_cvdump_type(self, type_name: str | None) -> dict[str, Any | None]:
+    def _get_cvdump_type(self, type_name: str | None) -> dict[str, Any] | None:
         return (
             None
             if type_name is None

reccmp/ghidra_scripts/lego_util/statistics.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass, field
 import logging
 
-from lego_util.exceptions import (
+from .exceptions import (
     TypeNotFoundInGhidraError,
     ClassOrNamespaceNotFoundInGhidraError,
 )

reccmp/ghidra_scripts/lego_util/type_importer.py

@@ -7,21 +7,6 @@
 # pylint: disable=too-many-return-statements # a `match` would be better, but for now we are stuck with Python 3.9
 # pylint: disable=no-else-return # Not sure why this rule even is a thing, this is great for checking exhaustiveness
 
-from lego_util.exceptions import (
-    TypeNotFoundError,
-    TypeNotFoundInGhidraError,
-    TypeNotImplementedError,
-    StructModificationError,
-)
-from lego_util.ghidra_helper import (
-    add_data_type_or_reuse_existing,
-    get_or_add_pointer_type,
-    get_ghidra_type,
-    get_or_create_namespace,
-    sanitize_name,
-)
-from lego_util.pdb_extraction import PdbFunctionExtractor
-
 from ghidra.program.flatapi import FlatProgramAPI
 from ghidra.program.model.data import (
     ArrayDataType,
@@ -39,6 +24,21 @@
 
 from reccmp.isledecomp.cvdump.types import VirtualBasePointer
 
+from .exceptions import (
+    TypeNotFoundError,
+    TypeNotFoundInGhidraError,
+    TypeNotImplementedError,
+    StructModificationError,
+)
+from .ghidra_helper import (
+    add_data_type_or_reuse_existing,
+    get_or_add_pointer_type,
+    get_ghidra_type,
+    get_or_create_namespace,
+    sanitize_name,
+)
+from .pdb_extraction import PdbFunctionExtractor
+
 logger = logging.getLogger(__name__)
 
 

reccmp/ghidra_scripts/lego_util/vtable_importer.py

@@ -9,10 +9,11 @@
 from ghidra.program.flatapi import FlatProgramAPI
 from ghidra.program.model.symbol import SourceType
 
-from lego_util.ghidra_helper import get_namespace_and_name
-
 from reccmp.isledecomp.compare.db import ReccmpEntity
 
+from .ghidra_helper import get_namespace_and_name
+
+
 logger = logging.getLogger(__name__)