Merge pull request #1 from grip-on-software/module-tests

Module setup, tests and GitHub Actions workflow

.coveragerc

@@ -0,0 +1,9 @@
+[run]
+branch = True
+[report]
+exclude_also =
+	if TYPE_CHECKING:
+	except ImportError:
+	raise NotImplementedError
+	raise AssertionError
+	\.\.\.

.github/workflows/status-tests.yml

@@ -0,0 +1,54 @@
+name: grip-on-software/status-dashboard
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+      - uses: actions/[email protected]
+        with:
+          python-version: "${{ matrix.python }}"
+      - name: Install development packages for some dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install libldap-dev libsasl2-dev
+      - name: Install dependencies for test
+        run: make setup_test
+      - name: Unit test and coverage
+        run: make coverage
+        env:
+          WEBTEST_INTERACTIVE: "False"
+      - name: Mypy typing coverage
+        run: |
+          make setup_analysis
+          make mypy
+      - name: Adjust source paths in coverage for Sonar
+        run: |
+          sed -i "s/<source>\/home\/runner\/work\/status-dashboard\/status-dashboard/<source>\/github\/workspace/g" \
+            test-reports/cobertura.xml mypy-report/cobertura.xml
+      - name: SonarCloud Scan
+        uses: sonarsource/[email protected]
+        if: "${{ matrix.python == '3.8.18' }}"
+        env:
+          SONAR_TOKEN: "${{ secrets.SONAR_TOKEN }}"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+      - name: Coveralls upload
+        run: |
+          pip install coveralls
+          coveralls
+        if: "${{ success() }}"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+    strategy:
+      matrix:
+        python:
+        - '3.8.18'
+        - '3.12.3'

.gitignore

@@ -5,7 +5,7 @@
 *.pyc
 *.swp
 *.log
-mypy-report/
+!test/sample/**/*.log
 
 # Pip installation
 src/*/
@@ -21,6 +21,7 @@ downloads/
 eggs/
 .eggs/
 lib/
+!typeshed/*/lib/
 lib64/
 parts/
 sdist/
@@ -29,3 +30,15 @@ wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+
+# Typing coverage
+.mypy_cache/
+mypy-report/
+
+# Pylint
+pylint-report.txt
+
+# Unit tests and coverage
+.coverage
+htmlcov/
+test-reports/

MANIFEST.in

@@ -0,0 +1,2 @@
+include requirements.txt
+include status/py.typed

Makefile

@@ -0,0 +1,133 @@
+COVERAGE=coverage
+MYPY=mypy
+PIP=python -m pip
+PYLINT=pylint
+RM=rm -rf
+SOURCES_ANALYSIS=status test
+SOURCES_COVERAGE=status,test
+TEST=-m pytest -s test
+TEST_OUTPUT=--junit-xml=test-reports/TEST-pytest.xml
+TWINE=twine
+
+.PHONY: all
+all: coverage mypy pylint
+
+.PHONY: release
+release: test mypy pylint clean build tag push upload
+
+.PHONY: setup
+setup:
+	$(PIP) install -r requirements.txt
+
+.PHONY: setup_release
+setup_release:
+	$(PIP) install -r requirements-release.txt
+
+.PHONY: setup_analysis
+setup_analysis:
+	$(PIP) install -r requirements-analysis.txt
+
+.PHONY: setup_test
+setup_test:
+	$(PIP) install -r requirements-test.txt
+
+.PHONY: install
+install:
+	$(PIP) install .
+
+.PHONY: pylint
+pylint:
+	$(PYLINT) $(SOURCES_ANALYSIS) --exit-zero --reports=n \
+		--msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" \
+		-d duplicate-code
+
+.PHONY: mypy
+mypy:
+	$(MYPY) $(SOURCES_ANALYSIS) \
+		--cobertura-xml-report mypy-report \
+		--junit-xml mypy-report/TEST-junit.xml \
+		--no-incremental --show-traceback
+
+.PHONY: mypy_html
+mypy_html:
+	$(MYPY) $(SOURCES_ANALYSIS) \
+		--html-report mypy-report \
+		--cobertura-xml-report mypy-report \
+		--junit-xml mypy-report/TEST-junit.xml \
+		--no-incremental --show-traceback
+
+.PHONY: test
+test:
+	python $(TEST) $(TEST_OUTPUT)
+
+.PHONY: coverage
+coverage:
+	$(COVERAGE) run --source=$(SOURCES_COVERAGE) $(TEST) $(TEST_OUTPUT)
+	$(COVERAGE) report -m
+	$(COVERAGE) xml -i -o test-reports/cobertura.xml
+
+# Version of the coverage target that does not write JUnit/cobertura XML output
+.PHONY: cover
+cover:
+	$(COVERAGE) run --source=$(SOURCES_COVERAGE) $(TEST)
+	$(COVERAGE) report -m
+
+.PHONY: get_version
+get_version: get_toml_version get_init_version get_sonar_version get_citation_version get_changelog_version
+	if [ "${TOML_VERSION}" != "${INIT_VERSION}" ] || [ "${TOML_VERSION}" != "${SONAR_VERSION}" ] || [ "${TOML_VERSION}" != "${CITATION_VERSION}" ] || [ "${TOML_VERSION}" != "${CHANGELOG_VERSION}" ]; then \
+		echo "Version mismatch"; \
+		exit 1; \
+	fi
+	$(eval VERSION=$(TOML_VERSION))
+
+.PHONY: get_init_version
+get_init_version:
+	$(eval INIT_VERSION=v$(shell grep __version__ status/__init__.py | sed -E "s/__version__ = .([0-9.]+)./\\1/"))
+	$(info Version in __init__.py: $(INIT_VERSION))
+	if [ -z "${INIT_VERSION}" ]; then \
+		echo "Could not parse version"; \
+		exit 1; \
+	fi
+
+.PHONY: get_toml_version
+get_toml_version:
+	$(eval TOML_VERSION=v$(shell grep "^version" pyproject.toml | sed -E "s/version = .([0-9.]+)./\\1/"))
+	$(info Version in pyproject.toml: $(TOML_VERSION))
+
+.PHONY: get_sonar_version
+get_sonar_version:
+	$(eval SONAR_VERSION=v$(shell grep projectVersion sonar-project.properties | cut -d= -f2))
+	$(info Version in sonar-project.properties: $(SONAR_VERSION))
+
+.PHONY: get_citation_version
+get_citation_version:
+	$(eval CITATION_VERSION=v$(shell grep "^version:" CITATION.cff | cut -d' ' -f2))
+	$(info Version in CITATION.cff: $(CITATION_VERSION))
+
+.PHONY: get_changelog_version
+get_changelog_version:
+	$(eval CHANGELOG_VERSION=v$(shell grep "^## \[[0-9]\+\.[0-9]\+\.[0-9]\+\]" CHANGELOG.md | head -n 1 | sed -E "s/## \[([0-9]+\.[0-9]+\.[0-9]+)\].*/\1/"))
+	$(info Version in CHANGELOG.md: $(CHANGELOG_VERSION))
+
+.PHONY: tag
+tag: get_version
+	git tag $(VERSION)
+
+.PHONY: build
+build:
+	python -m build
+
+.PHONY: push
+push: get_version
+	git push origin $(VERSION)
+
+.PHONY: upload
+upload:
+	$(TWINE) upload dist/*
+
+.PHONY: clean
+clean:
+	# Typing coverage and Pylint
+	$(RM) .mypy_cache mypy-report/ pylint-report.txt
+	# Pip and distribution
+	$(RM) src/ build/ dist/ gros-status.egg-info/

README.md

@@ -1,24 +1,75 @@
 GROS data gathering agent status
 ================================
 
+[![PyPI](https://img.shields.io/pypi/v/gros-status.svg)](https://pypi.python.org/pypi/gros-status)
+[![Build 
+status](https://github.com/grip-on-software/status-dashboard/actions/workflows/status-tests.yml/badge.svg)](https://github.com/grip-on-software/status-dashboard/actions/workflows/status-tests.yml)
+[![Coverage 
+Status](https://coveralls.io/repos/github/grip-on-software/status-dashboard/badge.svg?branch=master)](https://coveralls.io/github/grip-on-software/status-dashboard?branch=master)
+[![Quality Gate
+Status](https://sonarcloud.io/api/project_badges/measure?project=grip-on-software_status-dashboard&metric=alert_status)](https://sonarcloud.io/project/overview?id=grip-on-software_status-dashboard)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12533335.svg)](https://doi.org/10.5281/zenodo.12533335)
+
 This repository contains a Web application that provides an overview of the 
 data gathering agents and importer jobs, based on log parsing.
 
 ## Installation
 
-Run `pip install -r requirements.txt` to install the dependencies. Add `--user` 
-if you do not have access to the system libraries, or make use of `virtualenv`.
-You may need to add additional parameters, such as `--extra-index-url` for 
-a private repository.
+The latest version of GROS status dashboard and its dependencies can be 
+installed using `pip install gros-status`.
+
+Another option is to build the module from this repository, which allows using 
+the most recent development code. Run `make setup` to install the dependencies. 
+The status dashboard itself may then be installed with `make install`, which 
+places the package in your current environment. We recommend using a virtual 
+environment during development.
 
 ## Running
 
-Simply start the application using `python status.py`. Use command-line 
-arguments (displayed with `python status.py --help`) and/or a data-gathering 
-`settings.cfg` file (specifically the sections `ldap`, `deploy`, `jenkins` and 
-`schedule` influence this application's behavior - see the gros-gatherer 
-documentation for details).
+Simply start the application using `gros-status`. Use command-line arguments 
+(displayed with `gros-status --help`) and/or a data-gathering `settings.cfg` 
+file (specifically the sections `ldap`, `deploy`, `jenkins` and `schedule` 
+influence this application's behavior - see the [gros-gatherer documentation on 
+configuration](https://gros.liacs.nl/data-gathering/configuration.html) for 
+details).
 
 You can also configure the application as a systemd service such that it can 
-run headless under a separate user, using a virtualenv setup. See 
-`gros-status.service` for details.
+run headless under a separate user, using a virtualenv setup. See the 
+`gros-status.service` file in this repository for a possible setup.
+
+## Development and testing
+
+To run tests, first install the test dependencies with `make setup_test` which 
+also installs all dependencies for the server framework. Then `make coverage` 
+provides test results in the output and in XML versions compatible with, e.g., 
+JUnit and SonarQube available in the `test-reports/` directory. If you do not 
+need XML outputs, then run `make test` to just report on test successes and 
+failures or `make cover` to also have the terminal report on hits and misses in 
+statements and branches.
+
+[GitHub Actions](https://github.com/grip-on-software/status-dashboard/actions) 
+is used to run the unit tests and report on coverage on commits and pull 
+requests. This includes quality gate scans tracked by 
+[SonarCloud](https://sonarcloud.io/project/overview?id=grip-on-software_status-dashboard) 
+and [Coveralls](https://coveralls.io/github/grip-on-software/status-dashboard) 
+for coverage history.
+
+The Python module conforms to code style and typing standards which can be 
+checked using Pylint with `make pylint` and mypy with `make mypy`, after 
+installing the pylint and mypy dependencies using `make setup_analysis`; typing 
+reports are XML formats compatible with JUnit and SonarQube placed in the 
+`mypy-report/` directory. To also receive the HTML report, use `make mypy_html` 
+instead.
+
+We publish releases to [PyPI](https://pypi.org/project/gros-status/) using 
+`make setup_release` to install dependencies and `make release` which performs 
+multiple checks: unit tests, typing, lint and version number consistency. The 
+release files are also published on 
+[GitHub](https://github.com/grip-on-software/status-dashboard/releases) and 
+from there are archived on 
+[Zenodo](https://zenodo.org/doi/10.5281/zenodo.12533334). Noteworthy changes to 
+the module are added to the [changelog](CHANGELOG.md).
+
+## License
+
+GROS status dashboard is licensed under the Apache 2.0 License.

gros-status.service

@@ -4,7 +4,7 @@ After=network.target
 
 [Service]
 Type=forking
-ExecStart=/usr/local/bin/virtualenv.sh /usr/local/envs/controller /usr/local/bin/status.py --scgi --port 8116 --daemonize --pidfile /var/log/statusboard/statusboard.pid --log-path /var/log/statusboard --agent-path /agents --controller-path /controller --log INFO
+ExecStart=/usr/local/bin/virtualenv.sh /usr/local/envs/controller /usr/local/bin/gros-status --scgi --port 8116 --daemonize --pidfile /var/log/statusboard/statusboard.pid --log-path /var/log/statusboard --agent-path /agents --controller-path /controller --log INFO
 ExecReload=/bin/kill -s HUP $MAINPID
 WorkingDirectory=/srv/data-gathering
 KillSignal=SIGTERM

pyproject.toml

@@ -0,0 +1,62 @@
+[project]
+name = "gros-status"
+version = "0.0.3"
+description = "Grip on Software status dashboard"
+readme = "README.md"
+authors = [{name = "Leon Helwerda", email = "[email protected]"}]
+license = {text = "Apache 2.0"}
+requires-python = ">=3.8"
+dependencies = [
+    "gros-gatherer==1.0.0",
+    "gros-server==1.0.0",
+    "Pyro4==4.82"
+]
+classifiers=[
+    "Development Status :: 3 - Alpha",
+    "Environment :: Web Environment",
+    "Intended Audience :: Developers",
+    "Topic :: Internet :: WWW/HTTP :: HTTP Servers",
+    "License :: OSI Approved :: Apache Software License",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+keywords = ["status dashboard", "agent-based", "log parser"]
+
+[project.scripts]
+gros-status = "status.__main__:main"
+
+[project.urls]
+"Homepage" = "https://gros.liacs.nl"
+"PyPI" = "https://pypi.python.org/pypi/gros-status"
+"Source Code" = "https://github.com/grip-on-software/status-dashboard"
+"Issues" = "https://github.com/grip-on-software/status-dashboard/issues"
+"Pull Requests" = "https://github.com/grip-on-software/status-dashboard/pulls"
+"CI: GitHub Actions" = "https://github.com/grip-on-software/status-dashboard/actions"
+"CI: Coveralls" = "https://coveralls.io/github/grip-on-software/status-dashboard?branch=master"
+"CI: SonarCloud" = "https://sonarcloud.io/project/overview?id=grip-on-software_status-dashboard"
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[tool.setuptools]
+packages = ["status"]
+
+[tool.setuptools.package-data]
+"status" = ["py.typed"]
+
+[tool.mypy]
+mypy_path = "typeshed"
+
+[[tool.mypy.overrides]]
+module = ["Pyro4"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = "test"
+python_files = "*.py"