Refactor using python-common-utility

.github/workflows/python-release.yml → .github/workflows/test_and_release.yml

@@ -1,28 +1,59 @@
-name: Python release
+name: Test and Release
 
 on:
   push:
-    tags:
-      - "v*.*.*"
+    branches: main
+    tags: v*.*.*
+
+  pull_request:
+    branches: [ "main" ]
+    types:
+      - synchronize
+      - opened
+      - reopened
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.sha }}
+  cancel-in-progress: true
 
 jobs:
-  publish-and-release:
-    name: Publish and release distributions
+  test:
+    name: Build and test
 
     runs-on: ubuntu-latest
 
     permissions:
+      # Gives the action the necessary permissions for publishing new
+      # comments in pull requests.
+      pull-requests: write
       contents: write
-      discussions: write
+      statuses: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Verify changes
+        uses: EffectiveRange/python-verify-github-action@v1
+        with:
+          coverage-threshold: '95'
+
+  release:
+    if: startsWith(github.ref, 'refs/tags/')
+    needs: test
+
+    name: Publish and release
+
+    runs-on: ubuntu-latest
 
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
       - name: Package and publish
-        uses: EffectiveRange/python-package-github-action@v1
+        uses: EffectiveRange/python-package-github-action@v2
         with:
-          debian-dist-type: 'library'
-          post-build-command: 'make service TAG=${GITHUB_REF#refs/tags/}'
+          debian-dist-type: 'fpm-deb'
+      - name: Create systemd service unit file
+        run: make service TAG=${GITHUB_REF#refs/tags/}
       - name: Set up QEMU for multi-architecture builds
         uses: docker/setup-qemu-action@v3
       - name: Setup Docker buildx for multi-architecture builds

.gitignore

@@ -37,7 +37,6 @@ pip-log.txt
 pip-delete-this-directory.txt
 
 # Unit test / coverage reports
-htmlcov/
 .tox/
 .nox/
 .coverage

.gitmodules

@@ -0,0 +1,3 @@
+[submodule ".devcontainer"]
+	path = .devcontainer
+	url = https://github.com/EffectiveRange/devcontainer-defs

README.md

@@ -1,3 +1,7 @@
+
+[![Test and Release](https://github.com/EffectiveRange/debian-package-collector/actions/workflows/test_and_release.yml/badge.svg)](https://github.com/EffectiveRange/debian-package-collector/actions/workflows/test_and_release.yml)
+[![Coverage badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/EffectiveRange/debian-package-collector/python-coverage-comment-action-data/endpoint.json)](https://htmlpreview.github.io/?https://github.com/EffectiveRange/debian-package-collector/blob/python-coverage-comment-action-data/htmlcov/index.html)
+
 # debian-package-collector
 
 Debian package collector to download .deb packages from new releases
@@ -43,9 +47,9 @@ pip install .
 
 ### Command line reference
 
-```commandline
+```bash
 $ bin/debian-package-collector.py --help
-usage: debian-package-collector.py [-h] [-f LOG_FILE] [-l LOG_LEVEL] [-d DOWNLOAD] [-i INTERVAL] [-p PORT] [-s SECRET] [-t TOKEN] [--initial | --no-initial] [--monitor | --no-monitor] [--webhook | --no-webhook] release_config
+usage: debian-package-collector.py [-h] [-f LOG_FILE] [-l LOG_LEVEL] [-d DOWNLOAD] [-i INTERVAL] [-p PORT] [-s SECRET] [-t TOKEN] [-D DELAY] [--initial | --no-initial] [--monitor | --no-monitor] [--webhook | --no-webhook] release_config
 
 positional arguments:
   release_config        release config JSON file path or URL
@@ -64,7 +68,9 @@ options:
   -s SECRET, --secret SECRET
                         webhook secret to verify requests, supports environment variables with $ (default: None)
   -t TOKEN, --token TOKEN
-                        GitHub token to use if not specified in config, supports environment variables with $ (default: None)
+                        global token to use if not specified in config, supports environment variables with $ (default: None)
+  -D DELAY, --delay DELAY
+                        download delay in seconds after webhook request (default: 10)
   --initial, --no-initial
                         enable initial collection (default: True)
   --monitor, --no-monitor
@@ -75,7 +81,7 @@ options:
 
 ### Example
 
-```commandline
+```bash
 $ bin/debian-package-collector.py ~/config/release-config.json
 ```
 
@@ -96,7 +102,7 @@ Example configuration (example `release-config.json` config file content):
 
 Output:
 
-```commandline
+```bash
 2024-07-16T06:09:01.076743Z [info     ] Starting package collector     [PackageCollectorApp] app_version=1.0.3 application=debian-package-collector arguments={'log_file': '/var/log/effective-range/debian-package-collector/debian-package-collector.log', 'log_level': 'info', 'download': '/tmp/packages', 'interval': 600, 'port': 8080, 'secret': None, 'token': None, 'initial': True, 'monitor': True, 'webhook': True, 'release_config': 'build/release-config.json'} hostname=Legion7iPro
 2024-07-16T06:09:01.080167Z [info     ] Local file path provided, skipping download [FileDownloader] app_version=1.0.3 application=debian-package-collector file=/home/attilagombos/EffectiveRange/debian-package-collector/build/release-config.json hostname=Legion7iPro
 2024-07-16T06:09:01.081313Z [info     ] Registered release source for repository [SourceRegistry] app_version=1.0.3 application=debian-package-collector config=ReleaseConfig(EffectiveRange/wifi-manager.git, matcher=*.deb, has_token=False) hostname=Legion7iPro repo=EffectiveRange/wifi-manager

bin/debian-package-collector.py

@@ -9,11 +9,18 @@
 from signal import signal, SIGINT, SIGTERM
 from typing import Any
 
+from common_utility import SessionProvider, FileDownloader, JsonLoader, ReusableTimer
 from context_logger import get_logger, setup_logging
-from package_downloader import SessionProvider, RepositoryProvider, FileDownloader, AssetDownloader, JsonLoader
+from package_downloader import RepositoryProvider, AssetDownloader
 
-from package_collector import PackageCollector, SourceRegistry, ReleaseMonitor, ReusableTimer, WebhookServer, \
-    PackageCollectorConfig
+from package_collector import (
+    PackageCollector,
+    SourceRegistry,
+    ReleaseMonitor,
+    WebhookServer,
+    PackageCollectorConfig,
+    WebhookServerConfig,
+)
 
 log = get_logger('PackageCollectorApp')
 
@@ -34,7 +41,8 @@ def main() -> None:
 
     reusable_timer = ReusableTimer()
     release_monitor = ReleaseMonitor(source_registry, asset_downloader, reusable_timer, arguments.interval)
-    webhook_server = WebhookServer(source_registry, file_downloader, arguments.port, arguments.secret)
+    server_config = WebhookServerConfig(arguments.port, arguments.secret, arguments.delay)
+    webhook_server = WebhookServer(source_registry, file_downloader, asset_downloader, reusable_timer, server_config)
     config_path = file_downloader.download(arguments.release_config, skip_if_exists=False)
     config = PackageCollectorConfig(config_path, arguments.initial, arguments.monitor, arguments.webhook)
     json_loader = JsonLoader()
@@ -53,16 +61,24 @@ def handler(signum: int, frame: Any) -> None:
 
 def _get_arguments() -> Namespace:
     parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
-    parser.add_argument('-f', '--log-file', help='log file path',
-                        default='/var/log/effective-range/debian-package-collector/debian-package-collector.log')
+    parser.add_argument(
+        '-f',
+        '--log-file',
+        help='log file path',
+        default='/var/log/effective-range/debian-package-collector/debian-package-collector.log',
+    )
     parser.add_argument('-l', '--log-level', help='logging level', default='info')
     parser.add_argument('-d', '--download', help='package download location', default='/tmp/packages')
     parser.add_argument('-i', '--interval', help='release monitor interval in seconds', type=int, default=600)
     parser.add_argument('-p', '--port', help='webhook server port to listen on', type=int, default=8080)
-    parser.add_argument('-s', '--secret',
-                        help='webhook secret to verify requests, supports environment variables with $')
-    parser.add_argument('-t', '--token',
-                        help='global token to use if not specified in config, supports environment variables with $')
+    parser.add_argument(
+        '-s', '--secret', help='webhook secret to verify requests, supports environment variables with $'
+    )
+    parser.add_argument(
+        '-t', '--token', help='global token to use if not specified in config, supports environment variables with $'
+    )
+    parser.add_argument('-D', '--delay', help='download delay in seconds after webhook request', type=int, default=10)
+
     parser.add_argument('--initial', help='enable initial collection', action=BooleanOptionalAction, default=True)
     parser.add_argument('--monitor', help='enable periodic monitoring', action=BooleanOptionalAction, default=True)
     parser.add_argument('--webhook', help='enable the webhook server', action=BooleanOptionalAction, default=True)

package_collector/__init__.py

@@ -1,4 +1,3 @@
-from .reusableTimer import *
 from .releaseSource import *
 from .sourceRegistry import *
 from .releaseMonitor import *

package_collector/packageCollector.py

@@ -5,8 +5,9 @@
 from dataclasses import dataclass
 from typing import Any
 
+from common_utility import IJsonLoader
 from context_logger import get_logger
-from package_downloader import IJsonLoader, ReleaseConfig
+from package_downloader import ReleaseConfig
 
 from package_collector import IReleaseMonitor, IWebhookServer, ISourceRegistry
 
@@ -23,8 +24,14 @@ class PackageCollectorConfig:
 
 class PackageCollector(object):
 
-    def __init__(self, config: PackageCollectorConfig, json_loader: IJsonLoader, source_registry: ISourceRegistry,
-                 release_monitor: IReleaseMonitor, webhook_server: IWebhookServer):
+    def __init__(
+        self,
+        config: PackageCollectorConfig,
+        json_loader: IJsonLoader,
+        source_registry: ISourceRegistry,
+        release_monitor: IReleaseMonitor,
+        webhook_server: IWebhookServer,
+    ):
         self._config = config
         self._json_loader = json_loader
         self._source_registry = source_registry

package_collector/releaseMonitor.py

@@ -2,10 +2,11 @@
 # SPDX-FileCopyrightText: 2024 Attila Gombos <[email protected]>
 # SPDX-License-Identifier: MIT
 
+from common_utility import IReusableTimer
 from context_logger import get_logger
 from package_downloader import IAssetDownloader
 
-from package_collector import IReusableTimer, ISourceRegistry, IReleaseSource
+from package_collector import ISourceRegistry, IReleaseSource
 
 log = get_logger('ReleaseMonitor')
 
@@ -27,8 +28,13 @@ def check(self, package: str) -> None:
 
 class ReleaseMonitor(IReleaseMonitor):
 
-    def __init__(self, source_registry: ISourceRegistry, asset_downloader: IAssetDownloader,
-                 monitor_timer: IReusableTimer, monitor_interval: int = 600) -> None:
+    def __init__(
+        self,
+        source_registry: ISourceRegistry,
+        asset_downloader: IAssetDownloader,
+        monitor_timer: IReusableTimer,
+        monitor_interval: int = 600,
+    ) -> None:
         self._source_registry = source_registry
         self._asset_downloader = asset_downloader
         self._monitor_timer = monitor_timer