Fix Re complile fail mentioning 'ps_d' when using --gitignore (#241)

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: tests/cases/(refactor|source).*
 
 repos:
   - repo: https://github.com/psf/black
-    rev: 22.6.0
+    rev: 22.8.0
     hooks:
       - id: black
         args: [--line-length=79]

action.yml

@@ -10,7 +10,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - run: pip install --upgrade pip && python -m pip install unimport==0.11.1
+    - run: pip install --upgrade pip && python -m pip install unimport==0.11.2
       shell: bash
     - run: unimport --color auto --gitignore --ignore-init ${{ inputs.extra_args }}
       shell: bash

docs/CHANGELOG.md

@@ -2,8 +2,15 @@
 
 All notable changes to this project will be documented in this file.
 
-## [Unreleased] - ././
-
+## [0.11.2] - 4/September/2022
+
+- [Fix Re complile fail mentioning 'ps_d' when using --gitignore by @hakancelikdev](https://github.com/hakancelikdev/unimport/pull/241)
+  - For Python 3.7 and above
+    - Drop support for patspec, 0.5.0 above and below 0.10.0 versions.
+    - Only 0.10.0 and above versions are supported, in these versions the gitignore
+      parameter works more accurately.
+    - For more accurate results when using --gitignore parameter, please do not use
+      Python 3.6 and Windows.
 - Docs update by @hakancelikdev
 - [Refactor main.py and add tests by @hakancelikdev](https://github.com/hakancelikdev/unimport/pull/238)
 

docs/tutorial/command-line-options.md

@@ -110,6 +110,12 @@ It's possible to skip `.gitignore` glob patterns.
 
 - `$ unimport --gitignore`
 
+**Warning:**
+
+For more accurate results when using `--gitignore` parameter, please do not use Python
+3.6 and Windows. For more information, please visit ->
+https://github.com/hakancelikdev/unimport/issues/240
+
 ---
 
 ## Ignore init

setup.cfg

@@ -41,12 +41,13 @@ packages =
 package_dir =
     =src
 install_requires =
-    libcst>=0.3.7; python_version >= '3.9'
-    libcst>=0.3.0; python_version <= '3.8'
-    pathspec>=0.5.0
-    toml>=0.9.0
-    dataclasses>=0.5; python_version < '3.7'
-    typing-extensions>=3.7.4; python_version < '3.8'
+    libcst>=0.3.7, <1; python_version >= '3.9'
+    libcst>=0.3.0, <1; python_version <= '3.8'
+    pathspec>=0.10.1, <1; python_version >= '3.7'
+    pathspec>=0.5.0, <0.10.0; python_version == '3.6'
+    toml>=0.9.0, <1
+    dataclasses>=0.5, <1; python_version < '3.7'
+    typing-extensions>=3.7.4, <4; python_version < '3.8'
 
 [options.entry_points]
 console_scripts =

src/unimport/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.11.1"
+__version__ = "0.11.2"
 __description__ = (
     "A linter, formatter for finding and removing unused import statements."
 )

src/unimport/config.py

@@ -8,6 +8,7 @@
 from typing import Any, ClassVar, Dict, Iterator, List, Optional, Tuple
 
 import toml
+from pathspec.patterns import GitWildMatchPattern
 
 from unimport import constants as C
 from unimport import utils
@@ -23,7 +24,11 @@
 
 @dataclasses.dataclass
 class Config:
-    default_sources: ClassVar[List[Path]] = [Path(".")]
+    default_sources: ClassVar[List[Path]] = [Path(".")]  # Not init attribute
+    gitignore_patterns: List[GitWildMatchPattern] = dataclasses.field(
+        default_factory=list, init=False, repr=False, compare=False
+    )  # Not init attribute
+    use_color: bool = dataclasses.field(init=False)  # Not init attribute
 
     sources: Optional[List[Path]] = None
     include: str = C.INCLUDE_REGEX_PATTERN
@@ -40,12 +45,10 @@ class Config:
     @classmethod
     @functools.lru_cache(maxsize=None)
     def _get_init_fields(cls):
-        import typing
-
         return [
             key
-            for key, value in cls.__annotations__.items()
-            if not dataclasses._is_classvar(value, typing)
+            for key, field in cls.__dataclass_fields__.items()
+            if field._field_type == dataclasses._FIELD and field.init
         ]
 
     def __post_init__(self):
@@ -56,21 +59,19 @@ def __post_init__(self):
         self.check = self.check or not any((self.diff, self.remove))
         self.use_color: bool = self._use_color(self.color)
 
-        if self.gitignore and self.ignore_init:
-            gitignore_exclude = utils.get_exclude_list_from_gitignore()
-            self.exclude = "|".join(
-                [self.exclude, C.INIT_FILE_IGNORE_REGEX] + gitignore_exclude
-            )
-        elif self.gitignore:
-            gitignore_exclude = utils.get_exclude_list_from_gitignore()
-            self.exclude = "|".join([self.exclude] + gitignore_exclude)
+        if self.gitignore:
+            self.gitignore_patterns = utils.get_exclude_list_from_gitignore()
+
         elif self.ignore_init:
             self.exclude = "|".join([self.exclude, C.INIT_FILE_IGNORE_REGEX])
 
     def get_paths(self) -> Iterator[Path]:
         for source_path in self.sources:
             yield from utils.list_paths(
-                source_path, self.include, self.exclude
+                source_path,
+                include=self.include,
+                exclude=self.exclude,
+                gitignore_patterns=self.gitignore_patterns,
             )
 
     @classmethod
@@ -113,7 +114,7 @@ def build(
                 )
             context[field_name] = config_value
 
-        return cls(**context)
+        return cls(**context)  # Only init attribute values
 
 
 @dataclasses.dataclass

src/unimport/statement.py

@@ -280,18 +280,14 @@ def get_scope_by_current_node(
 
     @property
     def names(self) -> Iterator[Name]:
-        yield from filter(  # type: ignore
-            lambda node: isinstance(node, Name), self.current_nodes
-        )
+        yield from filter(lambda node: isinstance(node, Name), self.current_nodes)  # type: ignore
 
         for child_scope in self.child_scopes:
             yield from child_scope.names
 
     @property
     def imports(self) -> Iterator[Import]:
-        yield from filter(  # type: ignore
-            lambda node: isinstance(node, Import), self.current_nodes
-        )
+        yield from filter(lambda node: isinstance(node, Import), self.current_nodes)  # type: ignore
 
     @classmethod
     def get_previous_scope(cls, scope: "Scope") -> "Scope":

src/unimport/utils.py

@@ -81,18 +81,24 @@ def actiontobool(action: str) -> bool:
     return False
 
 
-def get_exclude_list_from_gitignore() -> List[str]:
-    """Converts .gitignore patterns to regex and return this exclude regex
+def get_exclude_list_from_gitignore(
+    path=Path(".gitignore"),
+) -> List[GitWildMatchPattern]:  # TODO: rename
+    """Converts .gitignore patterns to regex and return this excludes regex
     list."""
-    path = Path(".gitignore")
-    gitignore_regex: List[str] = []
-    if path.is_file():
-        source, _, _ = read(path)
-        for line in source.splitlines():
-            regex = GitWildMatchPattern.pattern_to_regex(line)[0]
-            if regex:
-                gitignore_regex.append(regex)
-    return gitignore_regex
+
+    if not path.is_file():
+        return []
+
+    gitignore_patterns: List[GitWildMatchPattern] = []
+    source, _, _ = read(path)
+    for line in source.splitlines():
+        regex, include = GitWildMatchPattern.pattern_to_regex(line)
+        if regex:
+            pattern = GitWildMatchPattern(re.compile(regex), include)
+            gitignore_patterns.append(pattern)
+
+    return gitignore_patterns
 
 
 def read(path: Path) -> Tuple[str, str, Optional[str]]:
@@ -111,20 +117,39 @@ def read(path: Path) -> Tuple[str, str, Optional[str]]:
 
 def list_paths(
     start: Path,
+    *,
     include: str = C.INCLUDE_REGEX_PATTERN,
     exclude: str = C.EXCLUDE_REGEX_PATTERN,
+    gitignore_patterns: Optional[List[GitWildMatchPattern]] = None
 ) -> Iterator[Path]:
     include_regex, exclude_regex = re.compile(include), re.compile(exclude)
     file_names: Iterable[Path]
     if start.is_dir():
         file_names = start.glob(C.GLOB_PATTERN)
     else:
         file_names = [start]
-    yield from filter(
-        lambda filename: include_regex.search(str(filename))
-        and not exclude_regex.search(str(filename)),
-        file_names,
-    )
+
+    if gitignore_patterns:
+        for file_name in file_names:
+            if include_regex.search(
+                str(file_name)
+            ) and not exclude_regex.search(str(file_name)):
+                for gitignore_pattern in gitignore_patterns:
+                    match_file = (
+                        gitignore_pattern.match_file(str(file_name))
+                        if C.PY37_PLUS
+                        else list(gitignore_pattern.match([str(file_name)]))
+                    )
+                    if match_file:
+                        break
+                else:
+                    yield file_name
+    else:
+        for file_name in file_names:
+            if include_regex.search(
+                str(file_name)
+            ) and not exclude_regex.search(str(file_name)):
+                yield file_name
 
 
 def diff(

tests/test_utils.py

@@ -1,9 +1,12 @@
 import os
+import sys
+import textwrap
 from pathlib import Path
 
 import pytest
 
 from tests.utils import refactor, reopenable_temp_file
+from unimport import constants as C
 from unimport import utils
 
 
@@ -25,6 +28,29 @@ def test_list_paths(path, count):
     assert len(list(utils.list_paths(path))) == count
 
 
+@pytest.mark.skipif(
+    not C.PY37_PLUS or sys.platform == "win32",
+    reason="Patspec version 0.10.0 and above are only supported for Python 3.7 above.",
+)
+def test_list_paths_with_gitignore():
+    gitignore = textwrap.dedent(
+        """\
+        a
+        b
+        spam/**
+        **/api/
+        **/
+        """
+    )
+    with reopenable_temp_file(gitignore) as gitignore_path:
+        gitignore_patterns = utils.get_exclude_list_from_gitignore(
+            gitignore_path
+        )
+        assert list(
+            utils.list_paths(Path("."), gitignore_patterns=gitignore_patterns)
+        ) == [Path("setup.py")]
+
+
 def test_bad_encoding():
     # Make conflict between BOM and encoding Cookie.
     # https://docs.python.org/3/library/tokenize.html#tokenize.detect_encoding