Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Improve test coverage #64

Merged
merged 18 commits into from
May 9, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,11 @@ jobs:

- name: Run tests
run: make test

- name: Report coverage
id: coverage
if: ${{ !cancelled() }}
uses: MishaKav/pytest-coverage-comment@a1fe18e2b00c64a765568e2edb9f1706eb8fc88b # v1.1.51
with:
title: Coverage Report
pytest-xml-coverage-path: ./coverage.xml
4 changes: 4 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,7 @@ __pycache__/
# Virtual environments
.venv/
env/

# Test coverage output
/.coverage
/coverage.xml
1 change: 1 addition & 0 deletions environment-dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ channels:
dependencies:
- python=3.10
- pytest
- pytest-cov
- pyyaml
- pip
- mypy
Expand Down
6 changes: 6 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,12 @@ files = [
python_version = "3.9"

[tool.pytest.ini_options]
addopts = [
"--cov",
"--color=yes",
"--cov-report=xml:./coverage.xml",
"--cov-report=term-missing"
]
filterwarnings = ["error"]
norecursedirs = ["env"]
pythonpath = "src/"
Expand Down
149 changes: 63 additions & 86 deletions src/anaconda_pre_commit_hooks/add_renovate_annotations.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
#!/usr/bin/env python3
"""Automagically add renovate comments to conda environment files.

Given a number of paths specified as CLI arguments, we extract the unique app directories
Expand All @@ -8,13 +7,10 @@

"""

import contextlib
import json
import os
import re
import shlex
import subprocess
from collections.abc import Iterator
from pathlib import Path
from typing import Annotated, NamedTuple, Optional, TypedDict

Expand All @@ -40,16 +36,36 @@ class Dependencies(NamedTuple):
conda: dict[str, Dependency]


@contextlib.contextmanager
def working_dir(d: Path) -> Iterator[None]:
orig = Path.cwd()
os.chdir(d)
yield
os.chdir(orig)
def setup_conda_environment(command: str, *, cwd: Optional[Path] = None) -> None:
"""Ensure the conda environment is setup and updated."""
cwd = cwd or Path.cwd()
result = subprocess.run(
shlex.split(command), capture_output=True, text=True, cwd=cwd
)
if result.returncode != 0:
print(f"Failed to run setup command in {cwd}")
print(result.stdout)
print(result.stderr)
result.check_returncode()


def list_packages_in_conda_environment(environment_selector: str) -> list[dict]:
# Then we list the actual versions of each package in the environment
result = subprocess.run(
["conda", "list", *shlex.split(environment_selector), "--json"],
capture_output=True,
text=True,
)
if result.returncode != 0:
print(result.stdout)
print(result.stderr)
result.check_returncode()

return json.loads(result.stdout)


def load_dependencies(
project_directory: Path,
project_directory: Optional[Path] = None,
create_command: str = "make setup",
environment_selector: str = "-p ./env",
) -> Dependencies:
Expand All @@ -64,63 +80,44 @@ def load_dependencies(
An object containing all dependencies in the installed environment, split between conda and pip packages.

"""
with working_dir(project_directory):
# First ensure the conda environment exists
result = subprocess.run(
shlex.split(create_command), capture_output=True, text=True
setup_conda_environment(create_command, cwd=project_directory or Path.cwd())

data = list_packages_in_conda_environment(environment_selector)

# We split the list separately into pip & conda dependencies
pip_deps = {
x["name"]: Dependency(
name=x["name"], channel=x["channel"], version=x["version"]
)
if result.returncode != 0:
print(f"Failed to run make setup for {project_directory}")
print(result.stdout)
print(result.stderr)
result.check_returncode()

# Then we list the actual versions of each package in the environment
result = subprocess.run(
["conda", "list", *shlex.split(environment_selector), "--json"],
capture_output=True,
text=True,
for x in data
if x["channel"] == "pypi"
}

# We use endswith to match both pkgs/main and repo/main to main
conda_deps = {
x["name"]: Dependency(
name=x["name"],
channel="main" if x["channel"].endswith("/main") else x["channel"],
version=x["version"],
)
if result.returncode != 0:
print(result.stdout)
print(result.stderr)
result.check_returncode()
data = json.loads(result.stdout)

# We split the list separately into pip & conda dependencies
pip_deps = {
x["name"]: Dependency(
name=x["name"], channel=x["channel"], version=x["version"]
)
for x in data
if x["channel"] == "pypi"
}

# We use endswith to match both pkgs/main and repo/main to main
conda_deps = {
x["name"]: Dependency(
name=x["name"],
channel="main" if x["channel"].endswith("/main") else x["channel"],
version=x["version"],
)
for x in data
if x["channel"] != "pypi"
}
if len(pip_deps) + len(conda_deps) != len(data):
raise ValueError("Mismatch parsing dependencies")
return Dependencies(pip=pip_deps, conda=conda_deps)
for x in data
if x["channel"] != "pypi"
}
if len(pip_deps) + len(conda_deps) != len(data):
raise ValueError("Mismatch parsing dependencies")
return Dependencies(pip=pip_deps, conda=conda_deps)


def process_environment_file(
def add_comments_to_env_file(
env_file: Path,
dependencies: Dependencies,
*,
conda_channel_overrides: Optional[ChannelOverrides] = None,
pypi_index_overrides: Optional[IndexOverrides] = None,
pip_index_overrides: Optional[IndexOverrides] = None,
) -> None:
"""Process an environment file, which entails adding renovate comments and pinning the installed version."""
conda_channel_overrides = conda_channel_overrides or {}
pypi_index_overrides = pypi_index_overrides or {}
pip_index_overrides = pip_index_overrides or {}

with env_file.open() as fp:
in_lines = fp.readlines()
Expand Down Expand Up @@ -176,7 +173,7 @@ def process_environment_file(
if package_name != ".":
if datasource == "conda":
renovate_line = f"{' ' * indentation}# renovate: datasource={datasource} depName={dep_name}\n"
elif (index_url := pypi_index_overrides.get(dep_name)) is not None:
elif (index_url := pip_index_overrides.get(dep_name)) is not None:
renovate_line = f"{' ' * indentation}# renovate: datasource={datasource} registryUrl={index_url}\n"
else:
renovate_line = (
Expand All @@ -202,24 +199,7 @@ def process_environment_file(
fp.writelines(out_lines)


def add_comments_to_env_files(
env_files: list[Path],
dependencies: Dependencies,
*,
conda_channel_overrides: Optional[ChannelOverrides] = None,
pypi_index_overrides: Optional[IndexOverrides] = None,
) -> None:
"""Process each environment file found."""
for f in env_files:
process_environment_file(
f,
dependencies,
conda_channel_overrides=conda_channel_overrides,
pypi_index_overrides=pypi_index_overrides,
)


def _parse_pip_index_overrides(
def parse_pip_index_overrides(
internal_pip_index_url: str, internal_pip_package: list[str]
) -> dict[PackageName, IndexUrl]:
pip_index_overrides = {}
Expand All @@ -235,7 +215,7 @@ def cli(
internal_pip_index_url: Annotated[str, typer.Option()] = "",
) -> None:
# Construct a mapping of package name to index URL based on CLI options
pip_index_overrides = _parse_pip_index_overrides(
pip_index_overrides = parse_pip_index_overrides(
internal_pip_index_url, internal_pip_package or []
)

Expand All @@ -244,15 +224,12 @@ def cli(
project_dirs = sorted({env_file.parent for env_file in env_files})
for project_dir in project_dirs:
deps = load_dependencies(project_dir)
project_env_files = [e for e in env_files if e.parent == project_dir]
add_comments_to_env_files(
project_env_files, deps, pypi_index_overrides=pip_index_overrides
)
project_env_files = (e for e in env_files if e.parent == project_dir)
for env_file in project_env_files:
add_comments_to_env_file(
env_file, deps, pip_index_overrides=pip_index_overrides
)


def main() -> None:
typer.run(cli)


if __name__ == "__main__":
main()
typer.run(cli) # pragma: nocover
Loading
Loading