tests + config + packaging #minor (#1)

.github/workflows/python-lint.yml

@@ -19,8 +19,9 @@ jobs:
         path: ~/.cache/pre-commit
         key: ${{ runner.os }}-pre-commit-${{ hashFiles('.pre-commit-config.yaml') }}
 
+    # make sure to also update `pyproject.toml` with the same requirements
     - id: install-requirements
-      run: pip install -e .[dev]
+      run: pip install "pre-commit~=3.7.0"
 
     - name: pre-commit
       id: pre-commit

.github/workflows/python-test.yml

@@ -17,11 +17,11 @@ jobs:
       matrix:
         os: [ubuntu-latest] # [ubuntu-latest, windows-latest, macos-latest]
         python-version: [
-          "3.8",
+#          "3.8",
           "3.9",
-          "3.10",
-          "3.11",
-          "3.12",
+#          "3.10",
+#          "3.11",
+#          "3.12",
         ]
 
     steps:
@@ -35,8 +35,7 @@ jobs:
     - name: Install dependencies
       run: |
         python3 -m pip install --upgrade pip
-        python3 -m pip install -r requirements.txt
-        python3 -m pip install -r requirements-test.txt
+        python3 -m pip install -e .[test]
 
     - name: Test with pytest
       run: |

.github/workflows/version-bump.yml

@@ -0,0 +1,28 @@
+# on merge to master from pull request
+name: Bump version
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - master
+
+jobs:
+  build:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.merge_commit_sha }}
+          fetch-depth: '0'
+
+      - name: Bump version and push tag
+        uses: anothrNick/github-tag-action@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          DEFAULT_BUMP: "patch"
+          WITH_V: true
+          PRERELEASE: true

.gitignore

@@ -127,3 +127,6 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+
+# macos
+.DS_Store

.pre-commit-config.yaml

@@ -28,6 +28,25 @@ repos:
       - id: black
         args: ["--verbose", "--target-version=py39"]
 
+  # local repo
+  - repo: local
+    hooks:
+      - id: pydependence
+        name: PyDependence
+        description: Generate dependencies for your project using the python AST.
+        language: python
+        always_run: true
+        verbose: true
+        entry: python3 -m pydependence.__main__
+        files: ^pyproject\.toml$
+        language_version: "3.9"
+        additional_dependencies:
+          - 'pydantic==2.*'
+          - 'packaging'
+          - 'networkx'
+          - 'stdlib_list'
+          - 'tomlkit'
+
   # - repo: https://github.com/nmichlo/pydependence
   #   # rev: v0.2.0
   #   hooks:

README.md

@@ -1,6 +1,6 @@
 
 <p align="center">
-    <h1 align="center">🕵️ pydependence 🐍</h1>
+    <h1 align="center">🕵️ PyDependence 🐍</h1>
     <p align="center">
         <i>Python local package dependency discovery and resolution</i>
     </p>
@@ -37,8 +37,15 @@
 - [Overview](#overview)
   + [Why](#Why)
   + [How This Works](#how-this-works)
-- [Install](#install)
+- [Configuration](#configuration)
 - [Usage](#usage)
+  + [Usage - Pre-Commit](#usage---pre-commit)
+  + [Usage - CLI](#usage---cli)
+- [Help](#help)
+  + [Version Mapping](#version-mapping)
+  + [Scopes](#scopes)
+    * [Sub-Scopes](#sub-scopes)
+  + [Output Resolvers](#output-resolvers)
 
 ----------------------
 
@@ -47,7 +54,6 @@
 If multiple dependencies are listed in a project, only some of them may actually be required!
 This project finds those dependencies!
 
-
 ### Why
 
 This project was created for multiple reasons
@@ -62,22 +68,160 @@ This project was created for multiple reasons
    - This can either be modules under a folder, similar to PYTHONPATH
    - Or actual paths to modules
 2. The AST of each python file is parsed, and import statements are found
-3. Finally, dependencies are resolved using breadth-first-search and flattened.
-   - imports that redirect to modules within the current namespace
-     are flattened and replaced with imports not in the namespace.
+3. Finally, dependencies are resolved using graph traversal and flattened.
+   - imports that redirect to modules within the current scope
+     are flattened and replaced with imports not in the scope.
 
+----------------------
 
-## Install
+## Configuration
 
-`pydependence` currently requires `python==3.10`, however,
-it can still be run in a virtual environment over legacy python code
+_Check the [pyproject.toml](./pyproject.toml) for detailed explanations of various config options and a working example of `pydependence` applied to itself._
 
-```bash
-pip install pydependence
+It is recommended to specify the config inside your projects existing `pyproject.toml`
+file, however, pydepedence will override whatever is specified here if a `.pydependence.cfg`
+file exists in the same folder. (This behavior is needed if for example a project is still using a
+`setup.py` file, or migrating from this.)
+
+Configuration using the `pyproject.toml` should be placed under the `[tool.pydependence]` table,
+while configuration for the `.pydependence.cfg` should be placed under the root table.
+
+Here is a minimal example:
+
+```toml
+# ... rest of pyproject.toml file ...
+
+[tool.pydependence]  # exclude table definition if inside `.pydependence.cfg`, place all attributes at root instead.
+versions = ["tomlkit>=0.12,<1"]
+scopes = [{name = "pydependence", pkg_paths = "./pydependence"}]
+resolvers = [
+    {strict_requirements_map=false, scope='pydependence', output_mode='dependencies'},
+    {strict_requirements_map=false, scope='pydependence', output_mode='optional-dependencies', output_name='all', visit_lazy=true},
+]
+
+# ... rest of pyproject.toml file ...
 ```
 
+----------------------
+
 ## Usage
 
+`pydependence` can be triggered from both the CLI and using pre-commit, and
+currently requires `python>=3.8`, however, it should still be able to run in
+a virtual environment over legacy python code.
+
+
+### Usage - Pre-Commit
+
+### Usage - CLI
+
+Manually invoke `pydependence `
+
 ```bash
+# install
+pip install pydependence
+
+# manual invocation
 python -m pydependence --help
 ```
+
+----------------------
+
+## Help
+
+pydependence is an AST imports analysis tool that is used to discover the imports of a
+package and generate a dependency graph and requirements/pyproject sections.
+
+pydependence is NOT a package manager or a dependency resolver.
+This is left to the tool of your choice, e.g. `pip`, `poetry`, `pip-compile`, `uv`, etc.
+
+_Check the [pyproject.toml](./pyproject.toml) for detailed explanations of various config options and a working example of `pydependence` applied to itself._
+
+### Version Mapping
+
+Versions are used to specify the version of a package that should be used when generating output requirements.
+- If a version is not specified, an error will be raised.
+
+Versions are also used to construct mappings between package names and import names.
+- e.g. `Pillow` is imported as `PIL`, so the version mapping is `{package="pillow", version="*", import="PIL"}`
+
+
+### Scopes
+
+A scope is a logical collection of packages.
+It is a way to group packages together for the purpose of dependency resolution.
+- NOTE: there cannot be conflicting module imports within a scope.
+
+Scopes can inherit from other scopes.
+Scopes can have filters applied to them, include & exclude.
+Scopes must have unique names.
+
+The order of constructing a single scope is important.
+   1. `parents`, `search_paths`, `pkg_paths`
+      - `parents`: inherit all modules from the specified scopes
+      - `search_paths`: search for packages inside the specified paths (like PYTHONPATH)
+      - `pkg_paths`: add the packages at the specified paths
+   2. `limit`, `include`, `exclude`
+      - `limit`: limit the search space to children of the specified packages
+      - `include`: include packages that match the specified patterns
+      - `exclude`: exclude packages that match the specified patterns
+
+The order of evaluation when constucting multiple scopes is important, and can
+be used to create complex dependency resolution strategies.
+   - all scopes are constructed in order of definition
+
+#### Sub-Scopes
+
+A subscope is simply an alias for constructing a new scope, where:
+- the parent scope is the current scope
+- a filter is applied to limit the packages
+
+e.g.
+```toml
+[[tool.pydependence.scopes]]
+name = "my_pkg"
+pkg_paths = ["my_pkg"]
+subscopes = {mySubPkg="my_pkg.my_sub_pkg"}
+```
+
+is the same as:
+```toml
+[[tool.pydependence.scopes]]
+name = "my_pkg"
+pkg_paths = ["my_pkg"]
+
+[[tool.pydependence.scopes]]
+name = "mySubPkg"
+parents = ["my_pkg"]
+limit = ["my_pkg.my_sub_pkg"]
+```
+
+why?
+- This simplifies syntax for the common pattern of when you want to resolve optional dependencies
+  across an entire package, but only want to traverse starting from the subscope.
+
+### Output Resolvers
+
+Resolvers are used to specify how to resolve dependencies, and where to output the results.
+
+options:
+* `scope`:
+  - is used to determine the search space for the resolver.
+* `start_scope`:
+  - is used to determine the starting point for the resolver, i.e. BFS across all imports occurs from this point.
+* `env`
+  - used to select a specific set of `versions` that are tagged with the same `env` key when resolving.
+* `raw`
+  - manually specify requirements and versions to output, overwriting what was resolved if conflicting.
+* `output_mode`:
+  - is used to determine where to output the results.
+  - valid options are: `dependencies`, `optional-dependencies`, or `requirements`
+* `output_file`:
+  - is used to specify the file to output the results to, by default this is the current `pyproject.toml` file.
+    this usually only needs to be specified when outputting to a different file like `requirements.txt`
+* `output_name`
+  - only applied if using `output_mode="optional-dependencies"`, specifies the extras group name.
+
+Note: We can have multiple resolvers to construct different sets of outputs. For example if you have a library
+      with core dependencies and optional dependencies, you can construct a resolver for each. And limit the results
+      for the optional dependencies to only output the optional dependencies for that resolver.