diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 54c7286..48f89b0 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,25 +7,31 @@ on: jobs: build: + name: Units - ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: + fail-fast: true matrix: - python-version: ["3.9"] + python-version: + - "3.9" steps: - - uses: actions/checkout@v4 + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements-dev.txt -r requirements.txt - continue-on-error: true + pip install -r requirements/requirements-test-${{ matrix.python-version }}.txt -r requirements/requirements-${{ matrix.python-version }}.txt + - name: Test with pytest run: | - pytest -v --cov=app --cov-report=xml + pytest --cov-report=xml - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index 5897b63..d005e61 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -venv __pycache__ -.ruff_cache .install -.venv +.ruff_cache +/.venv* +/venv* diff --git a/Makefile b/Makefile index cd14265..bb55971 100644 --- a/Makefile +++ b/Makefile @@ -1,34 +1,44 @@ # Variables -VENV_DIR=venv +VENV_DIR=.venvs/digital_roadmap PYTHON=$(VENV_DIR)/bin/python -PIP=$(VENV_DIR)/bin/pip -UVICORN=$(VENV_DIR)/bin/uvicorn +PIP=$(VENV_DIR)/bin/python -m pip RUFF=$(VENV_DIR)/bin/ruff +PRE_COMMIT=$(VENV_DIR)/bin/pre-commit +PYTEST=$(VENV_DIR)/bin/pytest PROJECT_DIR=$(shell pwd) +PYTHON_VERSION = $(shell python -V | cut -d ' ' -f 2 | cut -d '.' -f 1,2) +export PIP_DISABLE_PIP_VERSION_CHECK = 1 default: install -.venv: - python3 -m venv $(VENV_DIR) - touch $@ +.PHONY: venv +venv: + python3 -m venv --clear $(VENV_DIR) -.install: - $(PIP) install -r requirements.txt - $(PIP) install -r requirements-dev.txt - touch $@ +.PHONY: install +install: venv + $(PIP) install -r requirements/requirements-$(PYTHON_VERSION).txt -install: .venv .install +.PHONY: install-dev +install-dev: venv + $(PIP) install -r requirements/requirements-dev-$(PYTHON_VERSION).txt -run: install - $(UVICORN) app.main:app --reload --port 8081 +.PHONY: run +run: + $(VENV_DIR)/bin/fastapi run app/main.py --reload --host 127.0.0.1 --port 8081 +.PHONY: clean clean: - rm -rf $(VENV_DIR) - rm -rf .install .venv + @rm -rf $(VENV_DIR) +.PHONY: freeze +freeze: + @$(PROJECT_DIR)/scripts/freeze.py + +.PHONY: lint lint: - @echo "Running lint checks..." - @$(RUFF) check $(PROJECT_DIR) --fix - @echo "Linting completed." + @$(PRE_COMMIT) run --all-files -.PHONY: venv install run clean lint +.PHONY: test +test: + @$(PYTEST) diff --git a/README.md b/README.md index 25c694a..e1965dc 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,74 @@ -# Digital roadmap backend PoC +# Digital roadmap backend + +API server providing access to Red Hat Enterprise Linux roadmap information. -FastAPI application using `Uvicorn` as the ASGI server. ## Prerequisites -Before you begin, ensure you have the following installed: +Python 3.9 or later. +A container runtime such as `docker` or `podman`. -- Python 3.9 or later ## Setup Instructions -1. Clone this repository -2. Create a virtual environment (`make venv`), activate it and install requirements `make install` -3. Run server - `make run` -4. Take a look at the docs endpoint at `http://127.0.0.1:8000/docs` +Create a virtual environment, install the requirements, and run the server. + +```shell +make install +make run +``` + +This runs a server using the default virtual environment. Documentation can be found at `http://127.0.0.1:8081/docs`. + + +## Developer Guide +Install the developer tools and run the server. + +```shell +make install-dev +make run +``` + +Alternatively you may create your own virtual environment, install the requirements, and run the server manually. +``` +# After creating and activating a virtual environment +pip install -r requirements/requirements-dev-{Python version}.txt +fastapi run app/main.py --reload --host 127.0.0.1 --port 8081 +``` + + +### Testing + +Lint and run tests. + +```shell +make lint +make test +``` + +All `make` targets use the default virtual environment. If you want to use your own virtual environment, run the commands directly. + +```shell +ruff check --fix +ruff format +pytest +pre-commit run --all-files +``` + + +### Updating requirements + +Python 3.9, 3.11, and 3.12 must be available in order to generate requirements files. + +The following files are used for updating requiremetns: + +- `requiremetns.in` - Direct project dependencies +- `requiremetns-dev.in` - Requirements for development +- `requiremetns-test.in` - Requirements for running tests +- `constraints.txt` - Indirect project dependencies -## TODO +``` +make freeze +``` -- [ ] Contributing guide -- [ ] Tests -- [ ] CI/CD pipeline +Commit the changes. diff --git a/pyproject.toml b/pyproject.toml index 198a0a5..ceab4d1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,3 +16,19 @@ target-version = "py39" lint.ignore = [ "E501", # Ignore line length (similar to flake8's max-line-length) ] + +[tool.pytest.ini_options] +addopts =""" + -r a + --verbose + --strict-markers + --cov app + --cov-report html + --durations-min 1 + --durations 10 + --color yes + --showlocals +""" +testpaths = [ + "app", +] diff --git a/requirements.txt b/requirements.txt index b7662c8..e093a82 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,38 @@ -fastapi -uvicorn -beautifulsoup4 -pydantic -sqlalchemy -psycopg[binary] +annotated-types==0.7.0 +anyio==4.7.0 +certifi==2024.12.14 +click==8.1.7 +dnspython==2.7.0 +email_validator==2.2.0 +exceptiongroup==1.2.2 +fastapi==0.115.6 +fastapi-cli==0.0.7 +greenlet==3.1.1 +h11==0.14.0 +httpcore==1.0.7 +httptools==0.6.4 +httpx==0.28.1 +idna==3.10 +Jinja2==3.1.4 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +psycopg==3.2.3 +pydantic==2.10.3 +pydantic_core==2.27.1 +Pygments==2.18.0 +python-dotenv==1.0.1 +python-multipart==0.0.20 +PyYAML==6.0.2 +rich==13.9.4 +rich-toolkit==0.12.0 +shellingham==1.5.4 +sniffio==1.3.1 +SQLAlchemy==2.0.36 +starlette==0.41.3 +typer==0.15.1 +typing_extensions==4.12.2 +uvicorn==0.34.0 +uvloop==0.21.0 +watchfiles==1.0.3 +websockets==14.1 diff --git a/requirements/constraints.txt b/requirements/constraints.txt new file mode 100644 index 0000000..b804e85 --- /dev/null +++ b/requirements/constraints.txt @@ -0,0 +1 @@ +# Constraints on indirect dependencies diff --git a/requirements/requirements-3.11.txt b/requirements/requirements-3.11.txt new file mode 100644 index 0000000..b5241dc --- /dev/null +++ b/requirements/requirements-3.11.txt @@ -0,0 +1,37 @@ +annotated-types==0.7.0 +anyio==4.7.0 +certifi==2024.12.14 +click==8.1.7 +dnspython==2.7.0 +email_validator==2.2.0 +fastapi==0.115.6 +fastapi-cli==0.0.7 +greenlet==3.1.1 +h11==0.14.0 +httpcore==1.0.7 +httptools==0.6.4 +httpx==0.28.1 +idna==3.10 +Jinja2==3.1.4 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +psycopg==3.2.3 +pydantic==2.10.3 +pydantic_core==2.27.1 +Pygments==2.18.0 +python-dotenv==1.0.1 +python-multipart==0.0.20 +PyYAML==6.0.2 +rich==13.9.4 +rich-toolkit==0.12.0 +shellingham==1.5.4 +sniffio==1.3.1 +SQLAlchemy==2.0.36 +starlette==0.41.3 +typer==0.15.1 +typing_extensions==4.12.2 +uvicorn==0.34.0 +uvloop==0.21.0 +watchfiles==1.0.3 +websockets==14.1 diff --git a/requirements/requirements-3.12.txt b/requirements/requirements-3.12.txt new file mode 100644 index 0000000..b5241dc --- /dev/null +++ b/requirements/requirements-3.12.txt @@ -0,0 +1,37 @@ +annotated-types==0.7.0 +anyio==4.7.0 +certifi==2024.12.14 +click==8.1.7 +dnspython==2.7.0 +email_validator==2.2.0 +fastapi==0.115.6 +fastapi-cli==0.0.7 +greenlet==3.1.1 +h11==0.14.0 +httpcore==1.0.7 +httptools==0.6.4 +httpx==0.28.1 +idna==3.10 +Jinja2==3.1.4 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +psycopg==3.2.3 +pydantic==2.10.3 +pydantic_core==2.27.1 +Pygments==2.18.0 +python-dotenv==1.0.1 +python-multipart==0.0.20 +PyYAML==6.0.2 +rich==13.9.4 +rich-toolkit==0.12.0 +shellingham==1.5.4 +sniffio==1.3.1 +SQLAlchemy==2.0.36 +starlette==0.41.3 +typer==0.15.1 +typing_extensions==4.12.2 +uvicorn==0.34.0 +uvloop==0.21.0 +watchfiles==1.0.3 +websockets==14.1 diff --git a/requirements/requirements-3.9.txt b/requirements/requirements-3.9.txt new file mode 100644 index 0000000..e093a82 --- /dev/null +++ b/requirements/requirements-3.9.txt @@ -0,0 +1,38 @@ +annotated-types==0.7.0 +anyio==4.7.0 +certifi==2024.12.14 +click==8.1.7 +dnspython==2.7.0 +email_validator==2.2.0 +exceptiongroup==1.2.2 +fastapi==0.115.6 +fastapi-cli==0.0.7 +greenlet==3.1.1 +h11==0.14.0 +httpcore==1.0.7 +httptools==0.6.4 +httpx==0.28.1 +idna==3.10 +Jinja2==3.1.4 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +mdurl==0.1.2 +psycopg==3.2.3 +pydantic==2.10.3 +pydantic_core==2.27.1 +Pygments==2.18.0 +python-dotenv==1.0.1 +python-multipart==0.0.20 +PyYAML==6.0.2 +rich==13.9.4 +rich-toolkit==0.12.0 +shellingham==1.5.4 +sniffio==1.3.1 +SQLAlchemy==2.0.36 +starlette==0.41.3 +typer==0.15.1 +typing_extensions==4.12.2 +uvicorn==0.34.0 +uvloop==0.21.0 +watchfiles==1.0.3 +websockets==14.1 diff --git a/requirements/requirements-dev-3.11.txt b/requirements/requirements-dev-3.11.txt new file mode 100644 index 0000000..226687d --- /dev/null +++ b/requirements/requirements-dev-3.11.txt @@ -0,0 +1,28 @@ +-r requirements-3.11.txt +-r requirements-test-3.11.txt +asttokens==3.0.0 +cfgv==3.4.0 +decorator==5.1.1 +distlib==0.3.9 +executing==2.1.0 +filelock==3.16.1 +identify==2.6.3 +ipdb==0.13.13 +ipython==8.30.0 +jedi==0.19.2 +matplotlib-inline==0.1.7 +nodeenv==1.9.1 +parso==0.8.4 +pexpect==4.9.0 +platformdirs==4.3.6 +pre_commit==4.0.1 +prompt_toolkit==3.0.48 +ptyprocess==0.7.0 +pure_eval==0.2.3 +Pygments==2.18.0 +PyYAML==6.0.2 +stack-data==0.6.3 +traitlets==5.14.3 +typing_extensions==4.12.2 +virtualenv==20.28.0 +wcwidth==0.2.13 diff --git a/requirements/requirements-dev-3.12.txt b/requirements/requirements-dev-3.12.txt new file mode 100644 index 0000000..5d637ec --- /dev/null +++ b/requirements/requirements-dev-3.12.txt @@ -0,0 +1,27 @@ +-r requirements-3.12.txt +-r requirements-test-3.12.txt +asttokens==3.0.0 +cfgv==3.4.0 +decorator==5.1.1 +distlib==0.3.9 +executing==2.1.0 +filelock==3.16.1 +identify==2.6.3 +ipdb==0.13.13 +ipython==8.30.0 +jedi==0.19.2 +matplotlib-inline==0.1.7 +nodeenv==1.9.1 +parso==0.8.4 +pexpect==4.9.0 +platformdirs==4.3.6 +pre_commit==4.0.1 +prompt_toolkit==3.0.48 +ptyprocess==0.7.0 +pure_eval==0.2.3 +Pygments==2.18.0 +PyYAML==6.0.2 +stack-data==0.6.3 +traitlets==5.14.3 +virtualenv==20.28.0 +wcwidth==0.2.13 diff --git a/requirements/requirements-dev-3.9.txt b/requirements/requirements-dev-3.9.txt new file mode 100644 index 0000000..02d3cdd --- /dev/null +++ b/requirements/requirements-dev-3.9.txt @@ -0,0 +1,30 @@ +-r requirements-3.9.txt +-r requirements-test-3.9.txt +asttokens==3.0.0 +cfgv==3.4.0 +decorator==5.1.1 +distlib==0.3.9 +exceptiongroup==1.2.2 +executing==2.1.0 +filelock==3.16.1 +identify==2.6.3 +ipdb==0.13.13 +ipython==8.18.1 +jedi==0.19.2 +matplotlib-inline==0.1.7 +nodeenv==1.9.1 +parso==0.8.4 +pexpect==4.9.0 +platformdirs==4.3.6 +pre_commit==4.0.1 +prompt_toolkit==3.0.48 +ptyprocess==0.7.0 +pure_eval==0.2.3 +Pygments==2.18.0 +PyYAML==6.0.2 +stack-data==0.6.3 +tomli==2.2.1 +traitlets==5.14.3 +typing_extensions==4.12.2 +virtualenv==20.28.0 +wcwidth==0.2.13 diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in new file mode 100644 index 0000000..aeeee77 --- /dev/null +++ b/requirements/requirements-dev.in @@ -0,0 +1,3 @@ +ipython +ipdb +pre-commit diff --git a/requirements/requirements-test-3.11.txt b/requirements/requirements-test-3.11.txt new file mode 100644 index 0000000..2c4d3a9 --- /dev/null +++ b/requirements/requirements-test-3.11.txt @@ -0,0 +1,7 @@ +coverage==7.6.9 +iniconfig==2.0.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +ruff==0.8.3 diff --git a/requirements/requirements-test-3.12.txt b/requirements/requirements-test-3.12.txt new file mode 100644 index 0000000..2c4d3a9 --- /dev/null +++ b/requirements/requirements-test-3.12.txt @@ -0,0 +1,7 @@ +coverage==7.6.9 +iniconfig==2.0.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +ruff==0.8.3 diff --git a/requirements/requirements-test-3.9.txt b/requirements/requirements-test-3.9.txt new file mode 100644 index 0000000..767dd6f --- /dev/null +++ b/requirements/requirements-test-3.9.txt @@ -0,0 +1,9 @@ +coverage==7.6.9 +exceptiongroup==1.2.2 +iniconfig==2.0.0 +packaging==24.2 +pluggy==1.5.0 +pytest==8.3.4 +pytest-cov==6.0.0 +ruff==0.8.3 +tomli==2.2.1 diff --git a/requirements-dev.txt b/requirements/requirements-test.in similarity index 57% rename from requirements-dev.txt rename to requirements/requirements-test.in index 11491d0..a61377f 100644 --- a/requirements-dev.txt +++ b/requirements/requirements-test.in @@ -1,5 +1,3 @@ ruff pytest pytest-cov -httpx -pre-commit diff --git a/requirements/requirements.in b/requirements/requirements.in new file mode 100644 index 0000000..6491224 --- /dev/null +++ b/requirements/requirements.in @@ -0,0 +1,4 @@ +fastapi[standard] +greenlet +sqlalchemy +psycopg diff --git a/scripts/freeze.py b/scripts/freeze.py new file mode 100755 index 0000000..2916415 --- /dev/null +++ b/scripts/freeze.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python + +import argparse +import shutil +import subprocess +from concurrent.futures import ThreadPoolExecutor, as_completed +from pathlib import Path + + +def freeze(python_version: str, requirement: Path) -> str: + print(f"🥶 Freezing Python {python_version} {requirement.stem}...", flush=True) + + python_bin = shutil.which(f"python{python_version}") + if python_bin is None: + return f"Unable to find Python{python_version}" + + python_bin = Path(python_bin) + + repo_root = requirement.parent.parent + venv_path = repo_root / ".venvs" / f"freezer-{requirement.stem}-{python_version}" + venv_python = venv_path / "bin" / "python" + constraints = repo_root / "requirements" / "constraints.txt" + freeze_file = repo_root / "requirements" / f"{requirement.stem}-{python_version}.txt" + + # Create a fresh virtual environment + subprocess.check_output([python_bin, "-m", "venv", "--clear", "--system-site-packages", venv_path]) + subprocess.check_output([venv_python, "-m", "pip", "install", "--upgrade", "pip"]) + + # Install requirements with constraints + subprocess.check_output( + [venv_python, "-m", "pip", "install", "--requirement", requirement, "--constraint", constraints] + ) + + # Generate a freeze file + result = subprocess.run([venv_python, "-m", "pip", "freeze"], check=True, capture_output=True) + header = b"" + if requirement.stem.endswith("-dev"): + reqs_header = f"-r requirements-{ python_version }.txt\n".encode("utf-8") + test_header = f"-r requirements-test-{ python_version }.txt\n".encode("utf-8") + header = reqs_header + test_header + + freeze_file.write_bytes(header + result.stdout) + + return f"✅ {requirement.stem}-{python_version} complete" + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--python-versions", default="3.9,3.11,3.12") + + return parser.parse_args() + + +def sort_versions(versions: list[str]) -> list[str]: + def list_of_parts(items): + return [int(n) for n in items.split(".")] + + stripped_versions = [version.strip() for version in versions.split(",")] + return sorted(stripped_versions, key=list_of_parts) + + +def main(): + args = parse_args() + file = Path(__file__) + repo_root = file.parent.parent + requirements = repo_root.joinpath("requirements").glob("*.in") + python_versions = sort_versions(args.python_versions) + with ThreadPoolExecutor(max_workers=4) as executor: + futures = [ + executor.submit(freeze, py_ver, req) + for req in requirements + for py_ver in python_versions # noformat + ] + for future in as_completed(futures): + print(future.result()) + + target_python_version = "3.9" + shutil.copy(repo_root / "requirements" / f"requirements-{target_python_version}.txt", "requirements.txt") + + +if __name__ == "__main__": + main()