diff --git a/.devcontainer b/.devcontainer new file mode 160000 index 0000000..190f80b --- /dev/null +++ b/.devcontainer @@ -0,0 +1 @@ +Subproject commit 190f80babcba5a15b7ea3a0129c2f72cbec2ca0f diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml deleted file mode 100755 index f9d4e17..0000000 --- a/.github/workflows/python-test.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Python test - -on: - pull_request: - branches: [ "main" ] - -jobs: - build: - name: Build and test - - runs-on: ubuntu-latest - - permissions: - # Gives the action the necessary permissions for publishing new - # comments in pull requests. - pull-requests: write - contents: write - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Verify changes - uses: EffectiveRange/python-verify-github-action@v1 - with: - coverage-threshold: '95' diff --git a/.github/workflows/python-release.yml b/.github/workflows/test_and_release.yml old mode 100755 new mode 100644 similarity index 53% rename from .github/workflows/python-release.yml rename to .github/workflows/test_and_release.yml index 235560e..f7095db --- a/.github/workflows/python-release.yml +++ b/.github/workflows/test_and_release.yml @@ -1,28 +1,59 @@ -name: Python release +name: Test and Release on: push: - tags: - - "v*.*.*" + branches: main + tags: v*.*.* + + pull_request: + branches: [ "main" ] + types: + - synchronize + - opened + - reopened + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }} + cancel-in-progress: true jobs: - publish-and-release: - name: Publish and release distributions + test: + name: Build and test runs-on: ubuntu-latest permissions: + # Gives the action the necessary permissions for publishing new + # comments in pull requests. + pull-requests: write contents: write - discussions: write + statuses: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Verify changes + uses: EffectiveRange/python-verify-github-action@v1 + with: + coverage-threshold: '95' + + release: + if: startsWith(github.ref, 'refs/tags/') + needs: test + + name: Publish and release + + runs-on: ubuntu-latest steps: - name: Checkout repository uses: actions/checkout@v4 - name: Package and publish - uses: EffectiveRange/python-package-github-action@v1 + uses: EffectiveRange/python-package-github-action@v2 with: - debian-dist-type: 'library' - post-build-command: 'make service TAG=${GITHUB_REF#refs/tags/}' + debian-dist-type: 'fpm-deb' + - name: Create systemd service unit file + run: make service TAG=${GITHUB_REF#refs/tags/} - name: Set up QEMU for multi-architecture builds uses: docker/setup-qemu-action@v3 - name: Setup Docker buildx for multi-architecture builds diff --git a/.gitignore b/.gitignore index 82f9275..cef5098 100644 --- a/.gitignore +++ b/.gitignore @@ -37,7 +37,6 @@ pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports -htmlcov/ .tox/ .nox/ .coverage diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..f1b0753 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule ".devcontainer"] + path = .devcontainer + url = https://github.com/EffectiveRange/devcontainer-defs diff --git a/README.md b/README.md index 164bee7..cbb4227 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ + +[![Test and Release](https://github.com/EffectiveRange/debian-package-collector/actions/workflows/test_and_release.yml/badge.svg)](https://github.com/EffectiveRange/debian-package-collector/actions/workflows/test_and_release.yml) +[![Coverage badge](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/EffectiveRange/debian-package-collector/python-coverage-comment-action-data/endpoint.json)](https://htmlpreview.github.io/?https://github.com/EffectiveRange/debian-package-collector/blob/python-coverage-comment-action-data/htmlcov/index.html) + # debian-package-collector Debian package collector to download .deb packages from new releases @@ -43,9 +47,9 @@ pip install . ### Command line reference -```commandline +```bash $ bin/debian-package-collector.py --help -usage: debian-package-collector.py [-h] [-f LOG_FILE] [-l LOG_LEVEL] [-d DOWNLOAD] [-i INTERVAL] [-p PORT] [-s SECRET] [-t TOKEN] [--initial | --no-initial] [--monitor | --no-monitor] [--webhook | --no-webhook] release_config +usage: debian-package-collector.py [-h] [-f LOG_FILE] [-l LOG_LEVEL] [-d DOWNLOAD] [-i INTERVAL] [-p PORT] [-s SECRET] [-t TOKEN] [-D DELAY] [--initial | --no-initial] [--monitor | --no-monitor] [--webhook | --no-webhook] release_config positional arguments: release_config release config JSON file path or URL @@ -64,7 +68,9 @@ options: -s SECRET, --secret SECRET webhook secret to verify requests, supports environment variables with $ (default: None) -t TOKEN, --token TOKEN - GitHub token to use if not specified in config, supports environment variables with $ (default: None) + global token to use if not specified in config, supports environment variables with $ (default: None) + -D DELAY, --delay DELAY + download delay in seconds after webhook request (default: 10) --initial, --no-initial enable initial collection (default: True) --monitor, --no-monitor @@ -75,7 +81,7 @@ options: ### Example -```commandline +```bash $ bin/debian-package-collector.py ~/config/release-config.json ``` @@ -96,7 +102,7 @@ Example configuration (example `release-config.json` config file content): Output: -```commandline +```bash 2024-07-16T06:09:01.076743Z [info ] Starting package collector [PackageCollectorApp] app_version=1.0.3 application=debian-package-collector arguments={'log_file': '/var/log/effective-range/debian-package-collector/debian-package-collector.log', 'log_level': 'info', 'download': '/tmp/packages', 'interval': 600, 'port': 8080, 'secret': None, 'token': None, 'initial': True, 'monitor': True, 'webhook': True, 'release_config': 'build/release-config.json'} hostname=Legion7iPro 2024-07-16T06:09:01.080167Z [info ] Local file path provided, skipping download [FileDownloader] app_version=1.0.3 application=debian-package-collector file=/home/attilagombos/EffectiveRange/debian-package-collector/build/release-config.json hostname=Legion7iPro 2024-07-16T06:09:01.081313Z [info ] Registered release source for repository [SourceRegistry] app_version=1.0.3 application=debian-package-collector config=ReleaseConfig(EffectiveRange/wifi-manager.git, matcher=*.deb, has_token=False) hostname=Legion7iPro repo=EffectiveRange/wifi-manager diff --git a/bin/debian-package-collector.py b/bin/debian-package-collector.py index cbc2e8b..4ee8024 100755 --- a/bin/debian-package-collector.py +++ b/bin/debian-package-collector.py @@ -9,11 +9,18 @@ from signal import signal, SIGINT, SIGTERM from typing import Any +from common_utility import SessionProvider, FileDownloader, JsonLoader, ReusableTimer from context_logger import get_logger, setup_logging -from package_downloader import SessionProvider, RepositoryProvider, FileDownloader, AssetDownloader, JsonLoader +from package_downloader import RepositoryProvider, AssetDownloader -from package_collector import PackageCollector, SourceRegistry, ReleaseMonitor, ReusableTimer, WebhookServer, \ - PackageCollectorConfig +from package_collector import ( + PackageCollector, + SourceRegistry, + ReleaseMonitor, + WebhookServer, + PackageCollectorConfig, + WebhookServerConfig, +) log = get_logger('PackageCollectorApp') @@ -34,7 +41,8 @@ def main() -> None: reusable_timer = ReusableTimer() release_monitor = ReleaseMonitor(source_registry, asset_downloader, reusable_timer, arguments.interval) - webhook_server = WebhookServer(source_registry, file_downloader, arguments.port, arguments.secret) + server_config = WebhookServerConfig(arguments.port, arguments.secret, arguments.delay) + webhook_server = WebhookServer(source_registry, file_downloader, asset_downloader, reusable_timer, server_config) config_path = file_downloader.download(arguments.release_config, skip_if_exists=False) config = PackageCollectorConfig(config_path, arguments.initial, arguments.monitor, arguments.webhook) json_loader = JsonLoader() @@ -53,16 +61,24 @@ def handler(signum: int, frame: Any) -> None: def _get_arguments() -> Namespace: parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) - parser.add_argument('-f', '--log-file', help='log file path', - default='/var/log/effective-range/debian-package-collector/debian-package-collector.log') + parser.add_argument( + '-f', + '--log-file', + help='log file path', + default='/var/log/effective-range/debian-package-collector/debian-package-collector.log', + ) parser.add_argument('-l', '--log-level', help='logging level', default='info') parser.add_argument('-d', '--download', help='package download location', default='/tmp/packages') parser.add_argument('-i', '--interval', help='release monitor interval in seconds', type=int, default=600) parser.add_argument('-p', '--port', help='webhook server port to listen on', type=int, default=8080) - parser.add_argument('-s', '--secret', - help='webhook secret to verify requests, supports environment variables with $') - parser.add_argument('-t', '--token', - help='global token to use if not specified in config, supports environment variables with $') + parser.add_argument( + '-s', '--secret', help='webhook secret to verify requests, supports environment variables with $' + ) + parser.add_argument( + '-t', '--token', help='global token to use if not specified in config, supports environment variables with $' + ) + parser.add_argument('-D', '--delay', help='download delay in seconds after webhook request', type=int, default=10) + parser.add_argument('--initial', help='enable initial collection', action=BooleanOptionalAction, default=True) parser.add_argument('--monitor', help='enable periodic monitoring', action=BooleanOptionalAction, default=True) parser.add_argument('--webhook', help='enable the webhook server', action=BooleanOptionalAction, default=True) diff --git a/package_collector/__init__.py b/package_collector/__init__.py index 7a89961..1a7d12c 100644 --- a/package_collector/__init__.py +++ b/package_collector/__init__.py @@ -1,4 +1,3 @@ -from .reusableTimer import * from .releaseSource import * from .sourceRegistry import * from .releaseMonitor import * diff --git a/package_collector/packageCollector.py b/package_collector/packageCollector.py index 36fc94f..53ebb3f 100644 --- a/package_collector/packageCollector.py +++ b/package_collector/packageCollector.py @@ -5,8 +5,9 @@ from dataclasses import dataclass from typing import Any +from common_utility import IJsonLoader from context_logger import get_logger -from package_downloader import IJsonLoader, ReleaseConfig +from package_downloader import ReleaseConfig from package_collector import IReleaseMonitor, IWebhookServer, ISourceRegistry @@ -23,8 +24,14 @@ class PackageCollectorConfig: class PackageCollector(object): - def __init__(self, config: PackageCollectorConfig, json_loader: IJsonLoader, source_registry: ISourceRegistry, - release_monitor: IReleaseMonitor, webhook_server: IWebhookServer): + def __init__( + self, + config: PackageCollectorConfig, + json_loader: IJsonLoader, + source_registry: ISourceRegistry, + release_monitor: IReleaseMonitor, + webhook_server: IWebhookServer, + ): self._config = config self._json_loader = json_loader self._source_registry = source_registry diff --git a/package_collector/releaseMonitor.py b/package_collector/releaseMonitor.py index 0cd9dce..81d1006 100644 --- a/package_collector/releaseMonitor.py +++ b/package_collector/releaseMonitor.py @@ -2,10 +2,11 @@ # SPDX-FileCopyrightText: 2024 Attila Gombos # SPDX-License-Identifier: MIT +from common_utility import IReusableTimer from context_logger import get_logger from package_downloader import IAssetDownloader -from package_collector import IReusableTimer, ISourceRegistry, IReleaseSource +from package_collector import ISourceRegistry, IReleaseSource log = get_logger('ReleaseMonitor') @@ -27,8 +28,13 @@ def check(self, package: str) -> None: class ReleaseMonitor(IReleaseMonitor): - def __init__(self, source_registry: ISourceRegistry, asset_downloader: IAssetDownloader, - monitor_timer: IReusableTimer, monitor_interval: int = 600) -> None: + def __init__( + self, + source_registry: ISourceRegistry, + asset_downloader: IAssetDownloader, + monitor_timer: IReusableTimer, + monitor_interval: int = 600, + ) -> None: self._source_registry = source_registry self._asset_downloader = asset_downloader self._monitor_timer = monitor_timer diff --git a/package_collector/reusableTimer.py b/package_collector/reusableTimer.py deleted file mode 100755 index 7c0c544..0000000 --- a/package_collector/reusableTimer.py +++ /dev/null @@ -1,55 +0,0 @@ -# SPDX-FileCopyrightText: 2024 Ferenc Nandor Janky -# SPDX-FileCopyrightText: 2024 Attila Gombos -# SPDX-License-Identifier: MIT - -from threading import Timer, Lock -from typing import Any, Iterable, Optional, Mapping - - -class IReusableTimer(object): - - def start(self, interval_sec: int, function: Any, *args: Optional[Iterable[Any]], - **kwargs: Optional[Mapping[str, Any]]) -> 'IReusableTimer': - raise NotImplementedError() - - def restart(self) -> None: - raise NotImplementedError() - - def cancel(self) -> None: - raise NotImplementedError() - - def is_alive(self) -> bool: - raise NotImplementedError() - - -class ReusableTimer(IReusableTimer): - - def __init__(self) -> None: - self._timer_lock = Lock() - self._timer: Optional[Timer] = None - - def start(self, interval_sec: int, function: Any, *args: Optional[Iterable[Any]], - **kwargs: Optional[Mapping[str, Any]]) -> IReusableTimer: - with self._timer_lock: - if self._timer: - self._timer.cancel() - self._timer = Timer(interval_sec, function, args, kwargs) - self._timer.start() - return self - - def restart(self) -> None: - with self._timer_lock: - if self._timer: - self._timer.cancel() - self._timer = Timer(self._timer.interval, self._timer.function, self._timer.args, self._timer.kwargs) - self._timer.start() - - def cancel(self) -> None: - with self._timer_lock: - if self._timer: - self._timer.cancel() - self._timer = None - - def is_alive(self) -> bool: - with self._timer_lock: - return self._timer is not None and self._timer.is_alive() diff --git a/package_collector/webhookServer.py b/package_collector/webhookServer.py index 950a3e4..783248c 100644 --- a/package_collector/webhookServer.py +++ b/package_collector/webhookServer.py @@ -7,19 +7,28 @@ import hmac import json import os +from dataclasses import dataclass from threading import Thread from typing import Any, Optional +from common_utility import IFileDownloader, IReusableTimer from context_logger import get_logger from flask import Flask, request, Response, abort -from package_downloader import IFileDownloader, ReleaseConfig +from package_downloader import IAssetDownloader from waitress.server import create_server -from package_collector import ISourceRegistry +from package_collector import ISourceRegistry, IReleaseSource log = get_logger('WebhookServer') +@dataclass +class WebhookServerConfig: + port: int + secret: str + delay: int + + class IWebhookServer(object): def start(self) -> None: @@ -34,12 +43,21 @@ def is_running(self) -> bool: class WebhookServer(IWebhookServer): - def __init__(self, source_registry: ISourceRegistry, file_downloader: IFileDownloader, - port: int, secret: str) -> None: + def __init__( + self, + source_registry: ISourceRegistry, + file_downloader: IFileDownloader, + asset_downloader: IAssetDownloader, + delay_timer: IReusableTimer, + config: WebhookServerConfig, + ) -> None: self._source_registry = source_registry self._file_downloader = file_downloader - self._port = port - self._secret = self._get_secret(secret) + self._asset_downloader = asset_downloader + self._delay_timer = delay_timer + self._port = config.port + self._secret = self._get_secret(config.secret) + self._delay = config.delay self._app = Flask(__name__) self._server = create_server(self._app, listen=f'*:{self._port}') self._thread = Thread(target=self._start_server) @@ -59,6 +77,7 @@ def start(self) -> None: def shutdown(self) -> None: log.info('Shutting down') + self._delay_timer.cancel() self._server.close() self._thread.join() self._is_running = False @@ -115,33 +134,46 @@ def _process_release(self, payload: dict[str, Any]) -> Response: repo_name = payload['repository']['full_name'] release = payload['release'] action = payload['action'] + tag = release['tag_name'] - log.info('Processing release', repo=repo_name, action=action, tag=release['tag_name']) + log.info('Processing release', repo=repo_name, action=action, tag=tag) if self._source_registry.is_registered(repo_name): source = self._source_registry.get(repo_name) - return self._download_assets(source.get_config(), release['assets']) + assets = release['assets'] + + log.info('Available assets', assets=[asset['name'] for asset in assets]) + + if assets: + self._delay_timer.start(0, self._download_asset_from_url, args=[source, assets]) + return Response(status=200) + else: + log.warn( + 'No assets found in request, retrying using the API with a delay', + repo=repo_name, + tag=tag, + delay=self._delay, + ) + self._delay_timer.start(self._delay, self._download_asset_from_api, args=[source]) + return Response(status=200) else: log.warn('Repository not registered, skipping', repo=repo_name) return Response(status=204) - def _download_assets(self, config: ReleaseConfig, assets: list[dict[str, Any]]) -> Response: - log.info('Available assets', assets=[asset['name'] for asset in assets]) - + def _download_asset_from_url(self, source: IReleaseSource, assets: list[dict[str, Any]]) -> None: + config = source.get_config() any_match = False for asset in assets: if fnmatch.fnmatch(asset['name'], config.matcher): any_match = True log.info('Found matching asset', release=config, asset=asset['name']) - Thread(target=self._download_asset, args=[asset, config.raw_token]).start() + + self._download_asset(asset, config.raw_token) if not any_match: log.warn('No matching assets found', release=config) - return Response(status=204) - else: - return Response(status=200) def _download_asset(self, asset: dict[str, Any], token: Optional[str] = None) -> None: log.debug('Downloading asset', asset=asset['name']) @@ -152,3 +184,9 @@ def _download_asset(self, asset: dict[str, Any], token: Optional[str] = None) -> headers['Authorization'] = f'token {token}' self._file_downloader.download(asset['url'], asset['name'], headers) + + def _download_asset_from_api(self, source: IReleaseSource) -> None: + config = source.get_config() + release = source.get_release() + if release: + self._asset_downloader.download(config, release) diff --git a/service/debian-package-collector.docker.service b/service/debian-package-collector.docker.service index 9d1b055..ad2ee53 100644 --- a/service/debian-package-collector.docker.service +++ b/service/debian-package-collector.docker.service @@ -15,7 +15,7 @@ ExecStart=/usr/bin/docker run --net=host --name=debian-package-collector \ -v /opt/debs:/opt/debs \ -v /var/log/effective-range/debian-package-collector:/var/log/effective-range/debian-package-collector \ effectiverange/debian-package-collector:${TAG} \ - -- https://raw.githubusercontent.com/EffectiveRange/infrastructure-configuration/main/aptrepo/collector/release-config.json \ + -- https://raw.githubusercontent.com/EffectiveRange/infrastructure-configuration/main/aptrepo/collector/config/release-config.json \ --download /opt/debs \ --interval 3600 \ --token ${GITHUB_TOKEN} \ diff --git a/setup.cfg b/setup.cfg old mode 100755 new mode 100644 index 9e8cf0a..0a73ae9 --- a/setup.cfg +++ b/setup.cfg @@ -1,3 +1,8 @@ +[pack-python] +packaging = + wheel + fpm-deb + [mypy] packages = bin,package_collector strict = True diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index ad66fe4..4410c0d --- a/setup.py +++ b/setup.py @@ -8,8 +8,10 @@ author_email='info@effective-range.com', packages=['package_collector'], scripts=['bin/debian-package-collector.py'], - install_requires=['flask', 'waitress', - 'python-context-logger@git+https://github.com/EffectiveRange/python-context-logger.git@latest', - 'debian-package-downloader' - '@git+https://github.com/EffectiveRange/debian-package-downloader.git@latest'] + install_requires=[ + 'flask', + 'waitress', + 'python-context-logger@git+https://github.com/EffectiveRange/python-context-logger.git@latest', + 'debian-package-downloader@git+https://github.com/EffectiveRange/debian-package-downloader.git@latest', + ], ) diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 16281fe..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .utils import * diff --git a/tests/packageCollectorTest.py b/tests/packageCollectorTest.py index 04bcf86..bec61eb 100644 --- a/tests/packageCollectorTest.py +++ b/tests/packageCollectorTest.py @@ -5,9 +5,9 @@ from context_logger import setup_logging from package_downloader import IJsonLoader, ReleaseConfig +from test_utility import wait_for_assertion from package_collector import PackageCollector, PackageCollectorConfig, ISourceRegistry, IReleaseMonitor, IWebhookServer -from tests import wait_for_assertion class PackageCollectorTest(TestCase): @@ -24,20 +24,19 @@ def test_run_and_shutdown(self): release_config1 = ReleaseConfig(owner='owner1', repo='repo1') release_config2 = ReleaseConfig(owner='owner2', repo='repo2') config, json_loader, source_registry, release_monitor, webhook_server = create_components( - [release_config1, release_config2]) + [release_config1, release_config2] + ) # When - with PackageCollector(config, json_loader, source_registry, release_monitor, - webhook_server) as package_collector: + with PackageCollector( + config, json_loader, source_registry, release_monitor, webhook_server + ) as package_collector: Thread(target=package_collector.run).start() # Then wait_for_assertion(1, release_monitor.check_all.assert_called_once) - source_registry.register.assert_has_calls([ - mock.call(release_config1), - mock.call(release_config2) - ]) + source_registry.register.assert_has_calls([mock.call(release_config1), mock.call(release_config2)]) release_monitor.start.assert_called_once() webhook_server.start.assert_called_once() @@ -50,8 +49,9 @@ def test_run_and_shutdown_when_no_webhook_server(self): config, json_loader, source_registry, release_monitor, webhook_server = create_components(enable_webhook=False) # When - with PackageCollector(config, json_loader, source_registry, release_monitor, - webhook_server) as package_collector: + with PackageCollector( + config, json_loader, source_registry, release_monitor, webhook_server + ) as package_collector: Thread(target=package_collector.run).start() # Then @@ -65,12 +65,14 @@ def test_run_and_shutdown_when_no_webhook_server(self): def test_run_and_shutdown_when_no_initial_collection_and_no_monitoring(self): # Given - config, json_loader, source_registry, release_monitor, webhook_server = create_components(initial_collect=False, - enable_monitor=False) + config, json_loader, source_registry, release_monitor, webhook_server = create_components( + initial_collect=False, enable_monitor=False + ) # When - with PackageCollector(config, json_loader, source_registry, release_monitor, - webhook_server) as package_collector: + with PackageCollector( + config, json_loader, source_registry, release_monitor, webhook_server + ) as package_collector: Thread(target=package_collector.run).start() # Then @@ -83,12 +85,14 @@ def test_run_and_shutdown_when_no_initial_collection_and_no_monitoring(self): def test_run_and_shutdown_when_no_webhook_server_and_no_monitoring(self): # Given - config, json_loader, source_registry, release_monitor, webhook_server = create_components(enable_webhook=False, - enable_monitor=False) + config, json_loader, source_registry, release_monitor, webhook_server = create_components( + enable_webhook=False, enable_monitor=False + ) # When - with PackageCollector(config, json_loader, source_registry, release_monitor, - webhook_server) as package_collector: + with PackageCollector( + config, json_loader, source_registry, release_monitor, webhook_server + ) as package_collector: package_collector.run() # Then @@ -101,8 +105,12 @@ def test_run_and_shutdown_when_no_webhook_server_and_no_monitoring(self): webhook_server.shutdown.assert_not_called() -def create_components(config_list: list[ReleaseConfig] = None, - initial_collect: bool = True, enable_monitor: bool = True, enable_webhook: bool = True): +def create_components( + config_list: list[ReleaseConfig] = None, + initial_collect: bool = True, + enable_monitor: bool = True, + enable_webhook: bool = True, +): if config_list is None: config_list = [] config = PackageCollectorConfig('', initial_collect, enable_monitor, enable_webhook) diff --git a/tests/releaseMonitorTest.py b/tests/releaseMonitorTest.py index c7be0c7..2993419 100644 --- a/tests/releaseMonitorTest.py +++ b/tests/releaseMonitorTest.py @@ -2,11 +2,12 @@ from unittest import TestCase, mock from unittest.mock import MagicMock +from common_utility import IReusableTimer from context_logger import setup_logging from github.GitRelease import GitRelease from package_downloader import IAssetDownloader, ReleaseConfig -from package_collector import ReleaseMonitor, ReleaseSource, IReusableTimer, ISourceRegistry, IReleaseSource +from package_collector import ReleaseMonitor, ReleaseSource, ISourceRegistry, IReleaseSource class ReleaseMonitorTest(TestCase): @@ -41,9 +42,7 @@ def test_restarts_release_monitoring_when_timer_triggers(self): # Then monitor_timer.restart.assert_called_once() - asset_downloader.download.assert_has_calls([ - mock.call(source2.config, source2.release) - ]) + asset_downloader.download.assert_has_calls([mock.call(source2.config, source2.release)]) def test_stops_release_monitoring(self): # Given @@ -68,10 +67,9 @@ def test_downloads_all_release_assets_when_new_releases_found(self): release_monitor.check_all() # Then - asset_downloader.download.assert_has_calls([ - mock.call(source1.config, source1.release), - mock.call(source3.config, source3.release) - ]) + asset_downloader.download.assert_has_calls( + [mock.call(source1.config, source1.release), mock.call(source3.config, source3.release)] + ) def test_downloads_release_asset_when_new_release_found(self): # Given diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100755 index d499174..0000000 --- a/tests/utils.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -import shutil -import time -from difflib import Differ -from pathlib import Path -from typing import Callable, Any - -TEST_RESOURCE_ROOT = str(Path(os.path.dirname(__file__)).absolute()) -TEST_FILE_SYSTEM_ROOT = str(Path(TEST_RESOURCE_ROOT).joinpath('test_root').absolute()) -RESOURCE_ROOT = str(Path(TEST_RESOURCE_ROOT).parent.absolute()) - - -def delete_directory(directory: str) -> None: - if os.path.isdir(directory): - shutil.rmtree(directory) - - -def create_directory(directory: str) -> None: - if not os.path.isdir(directory): - os.makedirs(directory, exist_ok=True) - - -def copy_file(source: str, destination: str) -> None: - create_directory(os.path.dirname(destination)) - shutil.copy(source, destination) - - -def create_file(file: str, content: str) -> None: - create_directory(os.path.dirname(file)) - with open(file, 'w') as f: - f.write(content) - - -def compare_files(file1: str, file2: str) -> bool: - with open(file1, 'r') as f1, open(file2, 'r') as f2: - lines1 = f1.readlines() - lines2 = f2.readlines() - - return compare_lines(lines1, lines2) - - -def compare_lines(lines1: list[str], lines2: list[str]) -> bool: - all_lines_match = True - - for line in Differ().compare(lines1, lines2): - if not line.startswith('?'): - print(line.strip('\n')) - if line.startswith(('-', '+', '?')): - all_lines_match = False - - return all_lines_match - - -def wait_for_condition(timeout: float, condition: Callable[..., bool], *args: Any, **kwargs: Any) -> None: - time_step = 0.01 - total_time = 0.0 - - while total_time <= timeout: - if args or kwargs: - if condition(*args, **kwargs): - return - else: - if condition(): - return - total_time += time_step - time.sleep(time_step) - else: - raise TimeoutError(f'Failed to meet condition in {timeout} seconds') - - -def wait_for_assertion(timeout: float, assertion: Callable[..., Any], *args: Any, **kwargs: Any) -> None: - time_step = 0.01 - total_time = 0.0 - error_message = '' - - while total_time <= timeout: - try: - if args or kwargs: - assertion(*args, **kwargs) - else: - assertion() - print(total_time) - return - except AssertionError as error: - error_message = str(error) - total_time += time_step - time.sleep(time_step) - else: - raise AssertionError(f'Failed to assert in {timeout} seconds: {error_message}') diff --git a/tests/webhookServerTest.py b/tests/webhookServerTest.py index 950b133..a1dbab9 100644 --- a/tests/webhookServerTest.py +++ b/tests/webhookServerTest.py @@ -7,12 +7,13 @@ from unittest import TestCase from unittest.mock import MagicMock +from common_utility import IFileDownloader, ReusableTimer from context_logger import setup_logging from github.GitRelease import GitRelease -from package_downloader import IFileDownloader, ReleaseConfig +from package_downloader import ReleaseConfig, IAssetDownloader +from test_utility import wait_for_assertion -from package_collector import WebhookServer, IReleaseSource, ISourceRegistry, ReleaseSource -from tests import wait_for_assertion +from package_collector import WebhookServer, IReleaseSource, ISourceRegistry, ReleaseSource, WebhookServerConfig class WebhookServerTest(TestCase): @@ -28,10 +29,10 @@ def setUp(self): def test_startup_and_shutdown(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() # When - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() # Then @@ -42,18 +43,15 @@ def test_startup_and_shutdown(self): def test_returns_403_when_no_signature(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() release = create_release() - headers = { - 'Content-Type': 'application/json', - 'X-GitHub-Event': 'release' - } + headers = {'Content-Type': 'application/json', 'X-GitHub-Event': 'release'} # When response = client.post('/webhook', json=release, headers=headers) @@ -63,9 +61,9 @@ def test_returns_403_when_no_signature(self): def test_returns_403_when_not_supported_algorithm(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -74,7 +72,7 @@ def test_returns_403_when_not_supported_algorithm(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': 'sha128=abcdef', - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -85,9 +83,9 @@ def test_returns_403_when_not_supported_algorithm(self): def test_returns_403_when_invalid_signature(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -96,7 +94,7 @@ def test_returns_403_when_invalid_signature(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('wrong_secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -107,9 +105,9 @@ def test_returns_403_when_invalid_signature(self): def test_returns_204_when_not_release(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -118,7 +116,7 @@ def test_returns_204_when_not_release(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('secret', release), - 'X-GitHub-Event': 'push' + 'X-GitHub-Event': 'push', } # When @@ -129,9 +127,9 @@ def test_returns_204_when_not_release(self): def test_returns_204_when_action_filtered_out(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -141,7 +139,7 @@ def test_returns_204_when_action_filtered_out(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -153,9 +151,10 @@ def test_returns_204_when_action_filtered_out(self): def test_returns_200_and_downloads_asset_when_release_published(self): # Given source = create_source() - source_registry, file_downloader = create_components(source) + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components(source) + config.secret = '$TEST_SECRET' - with WebhookServer(source_registry, file_downloader, 0, '$TEST_SECRET') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -164,25 +163,57 @@ def test_returns_200_and_downloads_asset_when_release_published(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('test_secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When response = client.post('/webhook', json=release, headers=headers) # Then - wait_for_assertion(1, file_downloader.download.assert_called_once_with, - 'https://example.com/file1.deb', 'file1.deb', - {'Accept': 'application/octet-stream', 'Authorization': 'token test_token'}) + wait_for_assertion( + 1, + file_downloader.download.assert_called_once_with, + 'https://example.com/file1.deb', + 'file1.deb', + {'Accept': 'application/octet-stream', 'Authorization': 'token test_token'}, + ) + + self.assertEqual(200, response.status_code) + + def test_returns_200_and_downloads_asset_using_api_when_no_assets_in_release(self): + # Given + source = create_source() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components(source) + config.secret = '$TEST_SECRET' + + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: + webhook_server.start() + + client = webhook_server._app.test_client() + release = create_release() + release['release']['assets'] = [] + + headers = { + 'Content-Type': 'application/json', + 'X-Hub-Signature-256': create_signature('test_secret', release), + 'X-GitHub-Event': 'release', + } + + # When + response = client.post('/webhook', json=release, headers=headers) + + # Then + wait_for_assertion(2, asset_downloader.download.assert_called_once_with, source.config, source.release) self.assertEqual(200, response.status_code) def test_returns_200_when_release_released(self): # Given source = create_source() - source_registry, file_downloader = create_components(source) + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components(source) + config.secret = '$TEST_SECRET' - with WebhookServer(source_registry, file_downloader, 0, '$TEST_SECRET') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -192,7 +223,7 @@ def test_returns_200_when_release_released(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('test_secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -204,9 +235,10 @@ def test_returns_200_when_release_released(self): def test_returns_200_when_release_edited(self): # Given source = create_source() - source_registry, file_downloader = create_components(source) + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components(source) + config.secret = '$TEST_SECRET' - with WebhookServer(source_registry, file_downloader, 0, '$TEST_SECRET') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -216,7 +248,7 @@ def test_returns_200_when_release_edited(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('test_secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -227,9 +259,9 @@ def test_returns_200_when_release_edited(self): def test_returns_204_and_skips_download_when_repo_not_registered(self): # Given - source_registry, file_downloader = create_components() + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components() - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -238,7 +270,7 @@ def test_returns_204_and_skips_download_when_repo_not_registered(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When @@ -247,13 +279,13 @@ def test_returns_204_and_skips_download_when_repo_not_registered(self): # Then self.assertEqual(204, response.status_code) - def test_returns_204_when_no_matching_asset_found(self): + def test_returns_200_when_no_matching_asset_found(self): # Given source = create_source() source.config.matcher = '*.rpm' - source_registry, file_downloader = create_components(source) + source_registry, file_downloader, asset_downloader, delay_timer, config = create_components(source) - with WebhookServer(source_registry, file_downloader, 0, 'secret') as webhook_server: + with WebhookServer(source_registry, file_downloader, asset_downloader, delay_timer, config) as webhook_server: webhook_server.start() client = webhook_server._app.test_client() @@ -262,31 +294,21 @@ def test_returns_204_when_no_matching_asset_found(self): headers = { 'Content-Type': 'application/json', 'X-Hub-Signature-256': create_signature('secret', release), - 'X-GitHub-Event': 'release' + 'X-GitHub-Event': 'release', } # When response = client.post('/webhook', json=release, headers=headers) # Then - self.assertEqual(204, response.status_code) + self.assertEqual(200, response.status_code) def create_release() -> dict[str, Any]: return { 'action': 'published', - 'release': { - 'tag_name': '1.0.0', - 'assets': [ - { - 'name': 'file1.deb', - 'url': 'https://example.com/file1.deb' - } - ] - }, - 'repository': { - 'full_name': 'owner1/repo1' - }, + 'release': {'tag_name': '1.0.0', 'assets': [{'name': 'file1.deb', 'url': 'https://example.com/file1.deb'}]}, + 'repository': {'full_name': 'owner1/repo1'}, } @@ -310,7 +332,10 @@ def create_components(source: IReleaseSource = None): source_registry.get.return_value = source source_registry.is_registered.return_value = source is not None file_downloader = MagicMock(spec=IFileDownloader) - return source_registry, file_downloader + asset_downloader = MagicMock(spec=IAssetDownloader) + delay_timer = ReusableTimer() + server_config = WebhookServerConfig(0, 'secret', 1) + return source_registry, file_downloader, asset_downloader, delay_timer, server_config if __name__ == '__main__':