[RSPEED-363] Add tooling and documentation for managing dependencies (#8

)

* Add script for generating freeze file
* Update requirements files
* Update ignored files

* Update Makefile
  - Change virtual env location
  - Separate requirements and dev requirements
  - Remove touch commands since the targets are phony and not real files
  - Change the run command to use fastapi directly

* Use correct requirements in CI
* Update requirements to include greenlet
* Remove top level dev requirements
* Use pre-commit for lint make target
* Add pytest config

* Update README
    - Add more complete developer instructions
    - Add instructions for testing
    - Add instructions for updating requirements

.github/workflows/unit-tests.yml

@@ -7,25 +7,31 @@ on:
 
 jobs:
   build:
+    name: Units - ${{ matrix.python-version }}
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: true
       matrix:
-        python-version: ["3.9"]
+        python-version:
+          - "3.9"
 
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements-dev.txt -r requirements.txt
-        continue-on-error: true
+          pip install -r requirements/requirements-test-${{ matrix.python-version }}.txt -r requirements/requirements-${{ matrix.python-version }}.txt
+
       - name: Test with pytest
         run: |
-          pytest -v --cov=app --cov-report=xml
+          pytest --cov-report=xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4

.gitignore

@@ -1,5 +1,5 @@
-venv
 __pycache__
-.ruff_cache
 .install
-.venv
+.ruff_cache
+/.venv*
+/venv*

Makefile

@@ -1,34 +1,44 @@
 # Variables
-VENV_DIR=venv
+VENV_DIR=.venvs/digital_roadmap
 PYTHON=$(VENV_DIR)/bin/python
-PIP=$(VENV_DIR)/bin/pip
-UVICORN=$(VENV_DIR)/bin/uvicorn
+PIP=$(VENV_DIR)/bin/python -m pip
 RUFF=$(VENV_DIR)/bin/ruff
+PRE_COMMIT=$(VENV_DIR)/bin/pre-commit
+PYTEST=$(VENV_DIR)/bin/pytest
 PROJECT_DIR=$(shell pwd)
+PYTHON_VERSION = $(shell python -V | cut -d ' ' -f 2 | cut -d '.' -f 1,2)
+export PIP_DISABLE_PIP_VERSION_CHECK = 1
 
 default: install
 
-.venv:
-	python3 -m venv $(VENV_DIR)
-	touch $@
+.PHONY: venv
+venv:
+	python3 -m venv --clear $(VENV_DIR)
 
-.install:
-	$(PIP) install -r requirements.txt
-	$(PIP) install -r requirements-dev.txt
-	touch $@
+.PHONY: install
+install: venv
+	$(PIP) install -r requirements/requirements-$(PYTHON_VERSION).txt
 
-install: .venv .install
+.PHONY: install-dev
+install-dev: venv
+	$(PIP) install -r requirements/requirements-dev-$(PYTHON_VERSION).txt
 
-run: install
-	$(UVICORN) app.main:app --reload --port 8081
+.PHONY: run
+run:
+	$(VENV_DIR)/bin/fastapi run app/main.py --reload --host 127.0.0.1 --port 8081
 
+.PHONY: clean
 clean:
-	rm -rf $(VENV_DIR)
-	rm -rf .install .venv
+	@rm -rf $(VENV_DIR)
 
+.PHONY: freeze
+freeze:
+	@$(PROJECT_DIR)/scripts/freeze.py
+
+.PHONY: lint
 lint:
-	@echo "Running lint checks..."
-	@$(RUFF) check $(PROJECT_DIR) --fix
-	@echo "Linting completed."
+	@$(PRE_COMMIT) run --all-files
 
-.PHONY: venv install run clean lint
+.PHONY: test
+test:
+	@$(PYTEST)

README.md

@@ -1,22 +1,74 @@
-# Digital roadmap backend PoC
+# Digital roadmap backend
+
+API server providing access to Red Hat Enterprise Linux roadmap information.
 
-FastAPI application using `Uvicorn` as the ASGI server.
 
 ## Prerequisites
 
-Before you begin, ensure you have the following installed:
+Python 3.9 or later.
+A container runtime such as `docker` or `podman`.
 
-- Python 3.9 or later
 
 ## Setup Instructions
 
-1. Clone this repository
-2. Create a virtual environment (`make venv`), activate it and install requirements `make install`
-3. Run server - `make run`
-4. Take a look at the docs endpoint at `http://127.0.0.1:8000/docs`
+Create a virtual environment, install the requirements, and run the server.
+
+```shell
+make install
+make run
+```
+
+This runs a server using the default virtual environment. Documentation can be found at  `http://127.0.0.1:8081/docs`.
+
+
+## Developer Guide
+Install the developer tools and run the server.
+
+```shell
+make install-dev
+make run
+```
+
+Alternatively you may create your own virtual environment, install the requirements, and run the server manually.
+```
+# After creating and activating a virtual environment
+pip install -r requirements/requirements-dev-{Python version}.txt
+fastapi run app/main.py --reload --host 127.0.0.1 --port 8081
+```
+
+
+### Testing
+
+Lint and run tests.
+
+```shell
+make lint
+make test
+```
+
+All `make` targets use the default virtual environment. If you want to use your own virtual environment, run the commands directly.
+
+```shell
+ruff check --fix
+ruff format
+pytest
+pre-commit run --all-files
+```
+
+
+### Updating requirements
+
+Python 3.9, 3.11, and 3.12 must be available in order to generate requirements files.
+
+The following files are used for updating requiremetns:
+
+- `requiremetns.in` - Direct project dependencies
+- `requiremetns-dev.in` - Requirements for development
+- `requiremetns-test.in` - Requirements for running tests
+- `constraints.txt` - Indirect project dependencies
 
-## TODO
+```
+make freeze
+```
 
-- [ ] Contributing guide
-- [ ] Tests
-- [ ] CI/CD pipeline
+Commit the changes.

pyproject.toml

@@ -16,3 +16,19 @@ target-version = "py39"
 lint.ignore = [
     "E501",  # Ignore line length (similar to flake8's max-line-length)
 ]
+
+[tool.pytest.ini_options]
+addopts ="""
+    -r a
+    --verbose
+    --strict-markers
+    --cov app
+    --cov-report html
+    --durations-min 1
+    --durations 10
+    --color yes
+    --showlocals
+"""
+testpaths = [
+    "app",
+]

requirements.txt

@@ -1,6 +1,38 @@
-fastapi
-uvicorn
-beautifulsoup4
-pydantic
-sqlalchemy
-psycopg[binary]
+annotated-types==0.7.0
+anyio==4.7.0
+certifi==2024.12.14
+click==8.1.7
+dnspython==2.7.0
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.115.6
+fastapi-cli==0.0.7
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+idna==3.10
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+psycopg==3.2.3
+pydantic==2.10.3
+pydantic_core==2.27.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+PyYAML==6.0.2
+rich==13.9.4
+rich-toolkit==0.12.0
+shellingham==1.5.4
+sniffio==1.3.1
+SQLAlchemy==2.0.36
+starlette==0.41.3
+typer==0.15.1
+typing_extensions==4.12.2
+uvicorn==0.34.0
+uvloop==0.21.0
+watchfiles==1.0.3
+websockets==14.1

requirements/constraints.txt

@@ -0,0 +1 @@
+# Constraints on indirect dependencies

requirements/requirements-3.11.txt

@@ -0,0 +1,37 @@
+annotated-types==0.7.0
+anyio==4.7.0
+certifi==2024.12.14
+click==8.1.7
+dnspython==2.7.0
+email_validator==2.2.0
+fastapi==0.115.6
+fastapi-cli==0.0.7
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+idna==3.10
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+psycopg==3.2.3
+pydantic==2.10.3
+pydantic_core==2.27.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+PyYAML==6.0.2
+rich==13.9.4
+rich-toolkit==0.12.0
+shellingham==1.5.4
+sniffio==1.3.1
+SQLAlchemy==2.0.36
+starlette==0.41.3
+typer==0.15.1
+typing_extensions==4.12.2
+uvicorn==0.34.0
+uvloop==0.21.0
+watchfiles==1.0.3
+websockets==14.1

requirements/requirements-3.12.txt

@@ -0,0 +1,37 @@
+annotated-types==0.7.0
+anyio==4.7.0
+certifi==2024.12.14
+click==8.1.7
+dnspython==2.7.0
+email_validator==2.2.0
+fastapi==0.115.6
+fastapi-cli==0.0.7
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+idna==3.10
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+psycopg==3.2.3
+pydantic==2.10.3
+pydantic_core==2.27.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+PyYAML==6.0.2
+rich==13.9.4
+rich-toolkit==0.12.0
+shellingham==1.5.4
+sniffio==1.3.1
+SQLAlchemy==2.0.36
+starlette==0.41.3
+typer==0.15.1
+typing_extensions==4.12.2
+uvicorn==0.34.0
+uvloop==0.21.0
+watchfiles==1.0.3
+websockets==14.1

requirements/requirements-3.9.txt

@@ -0,0 +1,38 @@
+annotated-types==0.7.0
+anyio==4.7.0
+certifi==2024.12.14
+click==8.1.7
+dnspython==2.7.0
+email_validator==2.2.0
+exceptiongroup==1.2.2
+fastapi==0.115.6
+fastapi-cli==0.0.7
+greenlet==3.1.1
+h11==0.14.0
+httpcore==1.0.7
+httptools==0.6.4
+httpx==0.28.1
+idna==3.10
+Jinja2==3.1.4
+markdown-it-py==3.0.0
+MarkupSafe==3.0.2
+mdurl==0.1.2
+psycopg==3.2.3
+pydantic==2.10.3
+pydantic_core==2.27.1
+Pygments==2.18.0
+python-dotenv==1.0.1
+python-multipart==0.0.20
+PyYAML==6.0.2
+rich==13.9.4
+rich-toolkit==0.12.0
+shellingham==1.5.4
+sniffio==1.3.1
+SQLAlchemy==2.0.36
+starlette==0.41.3
+typer==0.15.1
+typing_extensions==4.12.2
+uvicorn==0.34.0
+uvloop==0.21.0
+watchfiles==1.0.3
+websockets==14.1