From 80d157aa9307c79b0cef74bafca06439e974491a Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Tue, 19 Sep 2023 09:37:12 +0200 Subject: [PATCH 1/2] Rename yaml_file_type.py to yaml_file_types.py --- komodo/cli.py | 2 +- komodo/fetch.py | 2 +- komodo/lint_package_status.py | 2 +- komodo/show_version.py | 2 +- komodo/snyk_reporting.py | 2 +- komodo/{yaml_file_type.py => yaml_file_types.py} | 0 tests/test_show_version.py | 6 +++--- 7 files changed, 8 insertions(+), 8 deletions(-) rename komodo/{yaml_file_type.py => yaml_file_types.py} (100%) diff --git a/komodo/cli.py b/komodo/cli.py index 3c2b67a3f..502afad51 100755 --- a/komodo/cli.py +++ b/komodo/cli.py @@ -20,7 +20,7 @@ ) from komodo.shebang import fixup_python_shebangs from komodo.shell import pushd, shell -from komodo.yaml_file_type import YamlFile +from komodo.yaml_file_types import YamlFile def create_enable_scripts(komodo_prefix: str, komodo_release: str) -> None: diff --git a/komodo/fetch.py b/komodo/fetch.py index 530e3cfdb..67aceabfb 100644 --- a/komodo/fetch.py +++ b/komodo/fetch.py @@ -14,7 +14,7 @@ strip_version, ) from komodo.shell import pushd, shell -from komodo.yaml_file_type import YamlFile +from komodo.yaml_file_types import YamlFile def eprint(*args, **kwargs): diff --git a/komodo/lint_package_status.py b/komodo/lint_package_status.py index 3cb9d40ac..4e8593c49 100644 --- a/komodo/lint_package_status.py +++ b/komodo/lint_package_status.py @@ -2,7 +2,7 @@ import argparse -from komodo.yaml_file_type import YamlFile +from komodo.yaml_file_types import YamlFile VALID_VISIBILITY = ["public", "private"] VALID_IMPORTANCE = ["low", "medium", "high"] diff --git a/komodo/show_version.py b/komodo/show_version.py index e35ad73b0..f6c533757 100644 --- a/komodo/show_version.py +++ b/komodo/show_version.py @@ -10,7 +10,7 @@ import yaml -from komodo.yaml_file_type import ManifestFile +from komodo.yaml_file_types import ManifestFile def get_release() -> str: diff --git a/komodo/snyk_reporting.py b/komodo/snyk_reporting.py index ea66dbe55..bcc9f011a 100644 --- a/komodo/snyk_reporting.py +++ b/komodo/snyk_reporting.py @@ -8,7 +8,7 @@ from snyk.managers import OrganizationManager from snyk.models import Vulnerability -from komodo.yaml_file_type import ReleaseDir, ReleaseFile, YamlFile +from komodo.yaml_file_types import ReleaseDir, ReleaseFile, YamlFile _CONSOLE_VULNERABILITY_FORMAT = """\t{id} \t\tPackage: {package} diff --git a/komodo/yaml_file_type.py b/komodo/yaml_file_types.py similarity index 100% rename from komodo/yaml_file_type.py rename to komodo/yaml_file_types.py diff --git a/tests/test_show_version.py b/tests/test_show_version.py index 73f05908e..e86395067 100644 --- a/tests/test_show_version.py +++ b/tests/test_show_version.py @@ -73,13 +73,13 @@ def test_get_version_with_filepath(mock_version_manifest): the file directly. Note that the file loading _and validation_ is handled by - komodo.yaml_file_type.ManifestFile, hence the different args for `open()`. + komodo.yaml_file_types.ManifestFile, hence the different args for `open()`. """ fname = "/foo/bar/komodo-release-0.0.1-py38/komodo-release-0.0.1-py38" args = parse_args(["foo", "--manifest-file", fname]) assert get_version(args.package, manifest=args.manifest_file) == "1.2.3" - # Goes through argparse.FileType via komodo.yaml_file_type.ManifestFile. + # Goes through argparse.FileType via komodo.yaml_file_types.ManifestFile. mock_version_manifest.assert_called_once_with(fname, "r", -1, None, None) @@ -95,7 +95,7 @@ def test_get_version_fails_with_release_file(mock_release_file): _ = parse_args(["foo", "--manifest-file", fname]) assert "does not appear to be a manifest file" in str(exception_info.value) - # Goes through argparse.FileType via komodo.yaml_file_type.ManifestFile. + # Goes through argparse.FileType via komodo.yaml_file_types.ManifestFile. mock_release_file.assert_called_once_with(fname, "r", -1, None, None) From 1af27e69d8cdeed34e269ae69c98ea61cc9c375d Mon Sep 17 00:00:00 2001 From: Jonathan Karlsen Date: Mon, 11 Sep 2023 09:30:23 +0200 Subject: [PATCH 2/2] Add additional yaml file types for linting Added yaml files to mainly use with argsparser for releasefile, upgrade_proposals, repository_file, package_status_file, and package_maturity --- komodo/yaml_file_types.py | 660 ++++++++++++++++++++++++- tests/test_yaml_file_types.py | 900 ++++++++++++++++++++++++++++++++++ 2 files changed, 1546 insertions(+), 14 deletions(-) create mode 100644 tests/test_yaml_file_types.py diff --git a/komodo/yaml_file_types.py b/komodo/yaml_file_types.py index cb185ed8a..c910453f9 100644 --- a/komodo/yaml_file_types.py +++ b/komodo/yaml_file_types.py @@ -1,9 +1,58 @@ import argparse import os +from collections import namedtuple from pathlib import Path -from typing import Any, Dict +from typing import Dict, List -import yaml as yml +from ruamel.yaml import YAML +from ruamel.yaml.constructor import DuplicateKeyError + +komodo_error = namedtuple( + "KomodoError", ["package", "version", "maintainer", "depends", "err"] +) +report = namedtuple( + "LintReport", ["release_name", "maintainers", "dependencies", "versions"] +) + + +class KomodoException(Exception): + def __init__(self, error_message: komodo_error): + self.error = error_message + + +MISSING_PACKAGE = "missing package" +MISSING_VERSION = "missing version" +MISSING_DEPENDENCY = "missing dependency" +MISSING_MAINTAINER = "missing maintainer" +MISSING_MAKE = "missing make information" +MALFORMED_VERSION = "malformed version" +MAIN_VERSION = "dangerous version (main branch)" +MASTER_VERSION = "dangerous version (master branch)" +FLOAT_VERSION = "dangerous version (float interpretable)" + + +def _komodo_error(package=None, version=None, maintainer=None, depends=None, err=None): + return komodo_error( + package=package, + version=version, + maintainer=maintainer, + depends=depends, + err=err, + ) + + +def __reg_version_err(errs, package, version, maintainer, err=MALFORMED_VERSION): + return _komodo_error( + package=package, version=version, maintainer=maintainer, err=err + ) + + +def load_yaml_from_string(value: str) -> dict: + try: + yml = YAML().load(value) + return yml + except DuplicateKeyError as e: + raise SystemExit(e) class YamlFile(argparse.FileType): @@ -12,15 +61,70 @@ def __init__(self, *args, **kwargs): def __call__(self, value): file_handle = super().__call__(value) - yaml = yml.safe_load(file_handle) + yml = load_yaml_from_string(file_handle) file_handle.close() - return yaml + return yml class ReleaseFile(YamlFile): - def __call__(self, value: str) -> Dict[str, Dict[Any, Any]]: - yaml = super().__call__(value) - return {Path(value).stem: yaml} + """ + Return the data from 'release' YAML file, but validate it first. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.content: dict = None + + def __call__(self, value: str): + yml: dict = super().__call__(value) + self.validate_release_file(yml) + self.content: dict = yml + return self + + def from_yaml_string(self, value: bytes): + yml = load_yaml_from_string(value) + self.validate_release_file(yml) + self.content: dict = yml + return self + + @staticmethod + def validate_release_file(release_file_content: dict) -> None: + message = ( + "The file you provided does not appear to be a release file " + "produced by komodo. It may be a repository file. Release files " + "have a format like the following:\n\n" + 'python: 3.8.6-builtin\nsetuptools: 68.0.0\nwheel: 0.40.0\nzopfli: "0.3"' + ) + assert isinstance(release_file_content, dict), message + errors = [] + for package_name, package_version in release_file_content.items(): + error = Package.validate_package_entry_with_errors( + package_name, package_version + ) + errors.extend(error) + handle_validation_errors(errors, message) + + @staticmethod + def lint_release_name(packagefile_path: str) -> List[komodo_error]: + relname = os.path.basename(packagefile_path) + found = False + for py_suffix in "-py27", "-py36", "-py38", "-py310": + for rh_suffix in "", "-rhel6", "-rhel7", "-rhel8": + if relname.endswith(py_suffix + rh_suffix + ".yml"): + found = True + break + if not found: + return [ + _komodo_error( + package=packagefile_path, + err=( + "Invalid release name suffix. " + "Must be of the form -pyXX[X] or -pyXX[X]-rhelY" + ), + ) + ] + + return [] class ReleaseDir: @@ -28,8 +132,8 @@ def __call__(self, value: str) -> Dict[str, YamlFile]: if not os.path.isdir(value): raise NotADirectoryError(value) result = {} - for yml_file in Path(value).glob("*.yml"): - result.update(ReleaseFile()(yml_file)) + for yaml_file in Path(value).glob("*.yaml"): + result.update(ReleaseFile()(yaml_file)) return result @@ -38,8 +142,17 @@ class ManifestFile(YamlFile): Return the data from 'manifest' YAML, but validate it first. """ + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.content: dict = None + def __call__(self, value: str) -> Dict[str, Dict[str, str]]: - yaml = super().__call__(value) + yml = super().__call__(value) + self.validate_manifest_file(yml) + return yml + + @staticmethod + def validate_manifest_file(manifest_file_content: dict): message = ( "The file you provided does not appear to be a manifest file " "produced by komodo. It may be a release file. Manifest files " @@ -47,7 +160,526 @@ def __call__(self, value: str) -> Dict[str, Dict[str, str]]: "python:\n maintainer: foo@example.com\n version: 3-builtin\n" "treelib:\n maintainer: foo@example.com\n version: 1.6.1\n" ) - for _, metadata in yaml.items(): - assert isinstance(metadata, dict), message - assert isinstance(metadata["version"], str), message - return yaml + assert isinstance(manifest_file_content, dict), message + errors = [] + for package_name, metadata in manifest_file_content.items(): + if not isinstance(metadata, dict): + errors.append(f"Invalid metadata for package '{package_name}'") + continue + if not isinstance(metadata["version"], str): + errors.append( + f"Invalid version type in metadata for package '{package_name}'" + ) + handle_validation_errors(errors, message) + + +class RepositoryFile(YamlFile): + """ + Return the data from 'repository' YAML, but validate it first. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.content: dict = None + + def __call__(self, value: str): + self.content: dict = super().__call__(value) + self.validate_repository_file() + return self + + def from_yaml_string(self, value: bytes): + yml = load_yaml_from_string(value) + self.content: dict = yml + self.validate_repository_file() + return self + + def validate_package_entry( + self, package_name: str, package_version: str + ) -> komodo_error: + repository_entries = self.content + if package_name not in repository_entries: + raise KomodoException(f"Package '{package_name}' not found in repository") + if package_version not in repository_entries[package_name]: + raise KomodoException( + f"Version '{package_version}' of package '{package_name}' not found in" + " repository" + ) + + def lint_maintainer(self, package, version) -> komodo_error: + repository_entries = self.content + if package not in repository_entries: + raise KomodoException(_komodo_error(package=package, err=MISSING_PACKAGE)) + if version not in repository_entries[package]: + raise KomodoException( + _komodo_error(package=package, version=version, err=MISSING_VERSION) + ) + return _komodo_error( + package=package, + version=version, + maintainer=repository_entries[package][version]["maintainer"], + ) + + def validate_repository_file(self) -> None: + repository_file_content: dict = self.content + message = ( + "The file you provided does not appear to be a repository file " + "produced by komodo. It may be a release file. Repository files " + "have a format like the following:\n\n" + "pytest-runner:\n 6.0.0:\n make: pip\n " + "maintainer: scout\n depends:\n - wheel\n - " + """setuptools\n - python\n\npython:\n "3.8":\n ...""" + ) + assert isinstance(repository_file_content, dict), message + errors = [] + for package_name, versions in repository_file_content.items(): + try: + Package.validate_package_name(package_name) + if not isinstance(versions, dict): + errors.append( + f"Versions of package '{package_name}' is not formatted" + f" correctly ({versions})" + ) + continue + validation_errors = self.validate_versions(package_name, versions) + if validation_errors: + errors.extend(validation_errors) + except (ValueError, TypeError) as e: + errors.append(str(e)) + + handle_validation_errors(errors, message) + + def validate_versions(self, package_name: str, versions: dict) -> List[str]: + """ + Validates versions-dictionary of a package and returns a list of error messages + """ + errors = [] + for version, version_metadata in versions.items(): + Package.validate_package_version(package_name, version) + make_errors = Package.validate_package_make_with_errors( + package_name, version, version_metadata.get("make") + ) + errors.extend(make_errors) + maintainer_errors = Package.validate_package_maintainer_with_errors( + package_name, + version, + version_metadata.get("maintainer"), + ) + errors.extend(maintainer_errors) + for ( + package_property, + package_property_value, + ) in version_metadata.items(): + validation_errors = self.validate_package_properties( + package_name, + version, + package_property, + package_property_value, + ) + errors.extend(validation_errors) + return errors + + def validate_package_properties( + self, + package_name: str, + package_version: str, + package_property: str, + package_property_value: str, + ) -> List[str]: + """ + Validates package properties of the specified package + and returns a list of error messages + """ + pre_checked_properties = ["make", "maintainer"] + errors = [] + if package_property in pre_checked_properties: + return errors + if package_property == "depends": + if not isinstance(package_property_value, list): + errors.append( + f"Dependencies for package {package_name} have" + f" invalid type {package_property_value}" + ) + return errors + for dependency in package_property_value: + if not isinstance(dependency, str): + errors.append( + f"Package {package_name} version {package_version} has" + f" invalid dependency type({dependency})" + ) + continue + if dependency not in self.content.keys(): + errors.append( + f"Dependency '{dependency}' not found for" + f" package '{package_name}'" + ) + else: + try: + Package.validate_package_property_type( + package_name, + package_version, + package_property, + package_property_value, + ) + except (ValueError, TypeError) as e: + errors.append(str(e)) + return errors + + +class UpgradeProposalsFile(YamlFile): + """ + Return the data from 'upgrade_proposals' YAML, but validate it first. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.content: dict = None + + def __call__(self, value: str) -> Dict[str, Dict[str, str]]: + yml = super().__call__(value) + self.validate_upgrade_proposals_file(yml) + self.content: dict = yml + return self + + def from_yaml_string(self, value): + yml = load_yaml_from_string(value) + self.validate_upgrade_proposals_file(yml) + self.content: dict = yml + return self + + def validate_upgrade_key(self, upgrade_key: str) -> None: + assert ( + upgrade_key in self.content + ), f"No section for this release ({upgrade_key}) in upgrade_proposals.yml" + + @staticmethod + def validate_upgrade_proposals_file(upgrade_proposals_file_content: dict) -> None: + message = ( + "The file you provided does not appear to be an upgrade_proposals file" + " produced by komodo. It may be a release file. Upgrade_proposals files" + ' have a format like the following:\n2022-08:\n2022-09:\n python: "3.9"\n' + ' zopfli: "0.3"\n libecalc: 8.2.9' + ) + errors = [] + assert isinstance(upgrade_proposals_file_content, dict), message + for ( + release_version, + packages_to_upgrade, + ) in upgrade_proposals_file_content.items(): + if not isinstance(release_version, str): + errors.append( + f"Release version ({release_version}) is not of type string" + ) + continue + if packages_to_upgrade is None: + continue + if not isinstance(packages_to_upgrade, dict): + errors.append( + "New package upgrades have to be listed in dictionary format" + f" ({packages_to_upgrade})" + ) + continue + for package_name, package_version in packages_to_upgrade.items(): + errors.extend( + Package.validate_package_entry_with_errors( + package_name, package_version + ) + ) + handle_validation_errors(errors, message) + + +class PackageStatusFile(YamlFile): + """ + Return the data from 'package_status' YAML, but validate it first. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.content: dict = None + + def __call__(self, value: str): + yml = super().__call__(value) + self.content: dict = yml + self.validate_package_status_file() + return self + + def from_yaml_string(self, value: str): + yml = load_yaml_from_string(value) + self.content: dict = yml + self.validate_package_status_file() + return self + + def validate_package_status_file(self) -> None: + package_status = self.content + message = ( + "The file you provided does not appear to be a package_status file" + " produced by komodo. It may be a release file. Package_status files have" + " a format like the following:\n\nzopfli:\n visibility:" + " private\npython:\n visibility: public\n maturity: stable\n " + " importance: high" + ) + + assert isinstance(package_status, dict), message + + errors = [] + for package_name, status in package_status.items(): + try: + Package.validate_package_name(package_name) + if not isinstance(status, dict): + errors.append(f"Invalid package data for {package_name} - {status}") + continue + Package.validate_package_visibility( + package_name, status.get("visibility") + ) + except (ValueError, TypeError) as e: + errors.append(str(e)) + continue + visibility = status["visibility"] + if visibility == "public": + maturity_errors = Package.validate_package_maturity_with_errors( + package_name, status.get("maturity") + ) + errors.extend(maturity_errors) + + importance_errors = Package.validate_package_importance_with_errors( + package_name, status.get("importance") + ) + errors.extend(importance_errors) + + handle_validation_errors(errors, message) + + +class Package: + VALID_VISIBILITIES = ["public", "private"] + VALID_IMPORTANCES = ["low", "medium", "high"] + VALID_MATURITIES = ["experimental", "stable", "deprecated"] + VALID_MAKES = ["rpm", "cmake", "sh", "pip", "rsync", "noop", "download"] + + @staticmethod + def validate_package_name(package_name: str) -> None: + if isinstance(package_name, str): + if str.islower(package_name): + return + raise ValueError(f"Package name '{package_name}' should be lowercase.") + raise TypeError(f"Package name ({package_name}) should be of type string") + + @staticmethod + def validate_package_version(package_name: str, package_version: str) -> None: + if isinstance(package_version, str): + return + raise TypeError( + f"Package '{package_name}' has invalid version type ({package_version})" + ) + + @staticmethod + def validate_package_entry(package_name: str, package_version) -> None: + Package.validate_package_name(package_name) + Package.validate_package_version(package_name, package_version) + + @staticmethod + def validate_package_entry_with_errors( + package_name: str, package_version: str + ) -> List[str]: + """ + Validates package name and version, and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_entry(package_name, package_version) + except (ValueError, TypeError) as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_importance(package_name: str, package_importance: str) -> None: + if isinstance(package_importance, str): + if package_importance in Package.VALID_IMPORTANCES: + return + raise ValueError( + f"{package_name} has invalid importance value ({package_importance})" + ) + raise TypeError( + f"{package_name} has invalid importance type ({package_importance})" + ) + + @staticmethod + def validate_package_importance_with_errors( + package_name, package_importance: str + ) -> List[str]: + """ + Validates package importance of a package and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_importance(package_name, package_importance) + except (ValueError, TypeError) as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_visibility(package_name: str, package_visibility: str) -> None: + if isinstance(package_visibility, str): + if package_visibility in Package.VALID_VISIBILITIES: + return + raise ValueError( + f"Package '{package_name}' has invalid visibility value" + f" ({package_visibility})" + ) + raise TypeError( + f"Package '{package_name}' has invalid visibility type" + f" ({package_visibility})" + ) + + @staticmethod + def validate_package_maturity(package_name: str, package_maturity: str) -> None: + if isinstance(package_maturity, str): + if package_maturity in Package.VALID_MATURITIES: + return + raise ValueError( + f"Package '{package_name}' has invalid maturity value" + f" ({package_maturity})" + ) + raise TypeError( + f"Package '{package_name}' has invalid maturity type ({package_maturity})" + ) + + @staticmethod + def validate_package_maturity_with_errors( + package_name: str, package_maturity: str + ) -> List[str]: + """ + Validates package maturity of a package and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_maturity(package_name, package_maturity) + except (ValueError, TypeError) as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_make( + package_name: str, package_version: str, package_make: str + ) -> None: + if isinstance(package_make, str): + if package_make in Package.VALID_MAKES: + return + raise ValueError( + f"Package '{package_name}' version {package_version} has invalid make " + f"value ({package_make})" + ) + raise TypeError( + f"Package '{package_name}' version {package_version} has invalid make type" + f" ({package_make})" + ) + + @staticmethod + def validate_package_make_with_errors( + package_name: str, + package_version: str, + package_make: str, + ) -> List[str]: + """ + Validates make of a package and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_make(package_name, package_version, package_make) + except (ValueError, TypeError) as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_maintainer( + package_name: str, package_version: str, package_maintainer: str + ) -> None: + if isinstance(package_maintainer, str): + return + raise TypeError( + f"Package '{package_name}' version {package_version} has invalid" + f" maintainer type ({package_maintainer})" + ) + + @staticmethod + def validate_package_maintainer_with_errors( + package_name: str, + package_version: str, + package_maintainer: str, + ) -> List[str]: + """ + Validates maintainer of a package and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_maintainer( + package_name, package_version, package_maintainer + ) + except TypeError as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_source( + package_name: str, package_version: str, package_source: str + ) -> None: + if isinstance(package_source, (str, type(None))): + return + raise TypeError( + f"Package '{package_name}' version {package_version} has invalid source" + f" type ({package_source})" + ) + + @staticmethod + def validate_package_source_with_errors( + package_name: str, + package_version: str, + package_source: str, + ) -> List[str]: + """ + Validates source of a package and returns a list of error messages + """ + errors = [] + try: + Package.validate_package_source( + package_name, package_version, package_source + ) + except TypeError as e: + errors.append(str(e)) + return errors + + @staticmethod + def validate_package_property_type( + package_name: str, + package_version: str, + package_property: str, + package_property_value: str, + ): + if isinstance(package_property, str): + if not package_property.islower(): + raise ValueError( + f"Package '{package_name}' version '{package_version}' property" + f" should be lowercase ({package_property})" + ) + else: + raise TypeError( + f"Package '{package_name}' version has invalid property type" + f" ({package_property})" + ) + if not isinstance(package_property_value, str): + raise TypeError( + f"Package '{package_name}' version '{package_version}' property" + f" '{package_property}' has invalid property value type" + f" ({package_property_value})" + ) + + +def handle_validation_errors(errors: List[str], message: str): + if errors: + raise SystemExit("\n".join(errors + [message])) + + +def load_package_status_file(package_status_string: str): + return PackageStatusFile().from_yaml_string(package_status_string) + + +def load_repository_file(repository_file_string): + return RepositoryFile().from_yaml_string(repository_file_string) diff --git a/tests/test_yaml_file_types.py b/tests/test_yaml_file_types.py new file mode 100644 index 000000000..b7950c68f --- /dev/null +++ b/tests/test_yaml_file_types.py @@ -0,0 +1,900 @@ +from contextlib import contextmanager + +import pytest + +from komodo.yaml_file_types import ( + KomodoException, + PackageStatusFile, + ReleaseFile, + RepositoryFile, + _komodo_error, +) + + +@contextmanager +def does_not_raise(): + yield + + +@pytest.mark.parametrize( + "content, expectations", + [ + pytest.param( + 'zopfli: "0.3"\npytest: 0.40.1', does_not_raise(), id="valid_release_file" + ), + pytest.param( + "zopfli", + pytest.raises( + AssertionError, + match=r"The file you provided does not appear to be a release file", + ), + id="invalid_release_file_format", + ), + pytest.param( + 'zopfli: "0.3"\nzopfli: "0.3"', + pytest.raises(SystemExit, match='found duplicate key "zopfli"'), + id="invalid_release_file_duplicate_packages", + ), + pytest.param( + '1.2: "0.3"\nzopfli: "0.3"', + pytest.raises( + SystemExit, match=r"Package name .* should be of type string" + ), + id="invalid_release_file_float_package_name", + ), + pytest.param( + 'zopfli: 0.3\npytest: "0.3"', + pytest.raises(SystemExit, match=r"Package .* has invalid version.*\(0.3\)"), + id="invalid_release_file_float_package_version", + ), + pytest.param( + 'zopfli:\npytest: "0.3"', + pytest.raises( + SystemExit, match=r"Package .* has invalid version.*\(None\)" + ), + id="invalid_release_file_None_package_version", + ), + pytest.param( + 'zopfli: "0.3"\nPYTEST: "0.3"', + pytest.raises(SystemExit, match=r"Package name .* should be lowercase"), + id="invalid_release_file_uppercase_package_name", + ), + pytest.param( + 'zopfli: 0.3\nPYTEST: "0.3"', + pytest.raises( + SystemExit, + match=( + r"Package .* has invalid version.*\n.*Package name .* should be" + r" lowercase" + ), + ), + id="invalid_release_file_multiple_errors", + ), + ], +) +def test_release_file_yaml_type(content, expectations): + with expectations: + ReleaseFile().from_yaml_string(content) + + +@pytest.mark.parametrize( + "valid", + ( + "bleeding-py36.yml", + "/home/anyuser/komodo/2020.01.03-py36-rhel6.yml", + "myrelease-py36.yml", + "myrelease-py310.yml", + "myrelease-py310-rhel8.yml", + "myrelease-py36-rhel6.yml", + "myrelease-py36-rhel7.yml", + ), +) +def test_release_name_valid(valid): + assert ReleaseFile.lint_release_name(valid) == [] + + +@pytest.mark.parametrize( + "invalid", + ( + "bleeding", + "bleeding.yml", + "2020.01.01", + "2020.01.00.yml", + "/home/anyuser/komodo-releases/releases/2020.01.00.yml", + "bleeding-py36", + "bleeding-rhel6.yml", + ), +) +def test_release_name_invalid(invalid): + assert ReleaseFile.lint_release_name(invalid) != [] + + +VALID_REPOSITORY = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +VALID_REPOSITORY_ADDITIONAL_UNKNOWN_PROPERTIES = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout + added_for: performance +""" +INVALID_REPOSITORY_ADDITIONAL_UNKNOWN_UPPERCASE_PROPERTIES = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout + ADDED_FOR: performance + style: 2.2 +""" +INVALID_REPOSITORY_DUPLICATE_PACKAGES = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + python: + 3.9.1: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_DUPLICATE_PACKAGE_VERSIONS = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_FLOAT_PACKAGE_NAME = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + 1.2: + 1.20.0: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_FLOAT_PACKAGE_VERSION = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_NONE_PACKAGE_VERSION = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.1: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + : + make: pip + maintainer: scout + source: pypi + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_UPPERCASE_PACKAGE_NAME = """ + ZOPFLI: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.2: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MULTIPLE_ERRORS = """ + ZOPFLI: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MISSING_DEPENDENCY = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +VALID_REPOSITORY = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_SOURCE_TYPE = """ + zopfli: + "0.3": + source: 1.2 + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MAKE_TYPE = """ + zopfli: + "0.3": + source: pypi + make: 1.2 + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MAKE_VALUE = """ + zopfli: + "0.3": + source: pypi + make: cargo + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MAINTAINER_TYPE = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: 1.2 + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_DEPENDENCY_TYPE = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - 1.2 + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MISSING_MAKE = """ + zopfli: + "0.3": + source: pypi + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_REPOSITORY_MISSING_MAINTAINER = """ + zopfli: + "0.3": + source: pypi + make: pip + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + wheel: + 0.40.0: + make: pip + maintainer: scout +""" +INVALID_YAML = "zopfli" + + +@pytest.mark.parametrize( + "content, expectations", + [ + pytest.param(VALID_REPOSITORY, does_not_raise(), id="valid_repository_file"), + pytest.param( + INVALID_YAML, + pytest.raises( + AssertionError, + match=r"The file you provided does not appear to be a repository file", + ), + id="invalid_repository_file_format", + ), + pytest.param( + INVALID_REPOSITORY_DUPLICATE_PACKAGES, + pytest.raises(SystemExit, match='found duplicate key "python"'), + id="invalid_repository_file_duplicate_packages", + ), + pytest.param( + INVALID_REPOSITORY_DUPLICATE_PACKAGE_VERSIONS, + pytest.raises(SystemExit, match='found duplicate key "3.8.6"'), + id="invalid_repository_file_duplicate_versions", + ), + pytest.param( + INVALID_REPOSITORY_FLOAT_PACKAGE_NAME, + pytest.raises( + SystemExit, match=r"Package name .* should be of type string" + ), + id="invalid_repository_file_float_package_name", + ), + pytest.param( + INVALID_REPOSITORY_FLOAT_PACKAGE_VERSION, + pytest.raises(SystemExit, match=r"Package .* has invalid version.*\(3.8\)"), + id="invalid_repository_file_float_package_version", + ), + pytest.param( + INVALID_REPOSITORY_NONE_PACKAGE_VERSION, + pytest.raises(SystemExit, match=r"Package .* has invalid version type"), + id="invalid_repository_file_None_package_version", + ), + pytest.param( + INVALID_REPOSITORY_UPPERCASE_PACKAGE_NAME, + pytest.raises(SystemExit, match=r"Package name .* should be lowercase"), + id="invalid_repository_file_uppercase_package_name", + ), + pytest.param( + INVALID_REPOSITORY_MISSING_DEPENDENCY, + pytest.raises(SystemExit, match=r"Dependency .* not found for package"), + id="invalid_repository_file_dependency_missing", + ), + pytest.param( + INVALID_REPOSITORY_SOURCE_TYPE, + pytest.raises( + SystemExit, + match=r"property \'source\' has invalid property value type", + ), + id="invalid_repository_file_float_source", + ), + pytest.param( + INVALID_REPOSITORY_MAKE_TYPE, + pytest.raises(SystemExit, match=r"Package.*has invalid make type"), + id="invalid_repository_file_float_make", + ), + pytest.param( + INVALID_REPOSITORY_MAINTAINER_TYPE, + pytest.raises(SystemExit, match=r"Package.*has invalid maintainer type"), + id="invalid_repository_file_float_maintainer", + ), + pytest.param( + INVALID_REPOSITORY_DEPENDENCY_TYPE, + pytest.raises(SystemExit, match=r"Package .* has invalid dependency type"), + id="invalid_repository_file_float_dependency", + ), + pytest.param( + INVALID_REPOSITORY_MISSING_MAKE, + pytest.raises( + SystemExit, match=r"Package .* has invalid make type \(None\)" + ), + id="invalid_repository_file_missing_make", + ), + pytest.param( + INVALID_REPOSITORY_MISSING_MAINTAINER, + pytest.raises( + SystemExit, match=r"Package .* has invalid maintainer type \(None\)" + ), + id="invalid_repository_file_missing_maintainer", + ), + pytest.param( + INVALID_REPOSITORY_MAKE_VALUE, + pytest.raises( + SystemExit, match=r"Package.*has invalid make value \(cargo\)" + ), + id="invalid_repository_file_invalid_make_value", + ), + pytest.param( + INVALID_REPOSITORY_MULTIPLE_ERRORS, + pytest.raises( + SystemExit, + match=( + r"Package name .* should be lowercase.*\n.*Package .* has invalid" + r" version type" + ), + ), + id="invalid_repository_file_multiple_errors", + ), + pytest.param( + VALID_REPOSITORY_ADDITIONAL_UNKNOWN_PROPERTIES, + does_not_raise(), + id="valid_repository_file_additional_unknown_properties", + ), + pytest.param( + INVALID_REPOSITORY_ADDITIONAL_UNKNOWN_UPPERCASE_PROPERTIES, + pytest.raises( + SystemExit, + match=( + r"property should be lowercase.*\n.*invalid property value type" + r" \(2.2\).*" + ), + ), + id="invalid_repository_file_additional_unknown_uppercase_properties", + ), + ], +) +def test_repository_file_yaml_type(content, expectations): + with expectations: + RepositoryFile().from_yaml_string(content) + + +LINT_MAINTAINER_TEST_REPO = """ + zopfli: + "0.3": + source: pypi + make: pip + maintainer: scout + depends: + - setuptools + - python + setuptools: + 68.0.0: + source: pypi + make: pip + maintainer: scout + depends: + - wheel + - python + python: + 3.8.6: + make: sh + makefile: build__python-virtualenv.sh + maintainer: scout + '3.9': + maintainer: scout + makefile: build__python-virtualenv.sh + make: pip + wheel: + 0.40.0: + make: pip + maintainer: scout +""" + + +@pytest.mark.parametrize( + "package_name, package_version, expectation, return_object", + [ + pytest.param( + "python", + "3.8.6", + does_not_raise(), + _komodo_error("python", "3.8.6", "scout"), + id="lint_maintainer_returns_valid", + ), + pytest.param( + "pytest", + "3.9", + pytest.raises(KomodoException), + None, + id="lint_maintainer_throws_on_missing_package", + ), + pytest.param( + "python", + "4.0", + pytest.raises(KomodoException), + None, + id="lint_maintainer_throws_on_missing_package", + ), + ], +) +def test_repository_file_lint_maintainer( + package_name, package_version, expectation, return_object +): + repo_file = RepositoryFile().from_yaml_string(LINT_MAINTAINER_TEST_REPO) + with expectation: + result = repo_file.lint_maintainer(package_name, package_version) + if return_object: + assert result == return_object + + +@pytest.mark.parametrize( + "package_status_file_content, expectation", + [ + pytest.param( + "python:\n visibility: public\n maturity: experimental\n importance:" + " high", + does_not_raise(), + id="valid_package_status", + ), + pytest.param( + "0.2:\n visibility: public\n maturity: experimental\n importance: high", + pytest.raises(SystemExit, match=r"should be of type string"), + id="invalid_package_status__float_package_name", + ), + pytest.param( + "python:\n maturity: important", + pytest.raises(SystemExit, match=r"invalid visibility type"), + id="invalid_package_status__missing_visibility", + ), + pytest.param( + "python:\n visibility: private", + does_not_raise(), + id="valid_package_status__private_visibility", + ), + pytest.param( + "python:\n visibility: hidden\n maturity: experimental\n importance:" + " high", + pytest.raises(SystemExit, match=r"invalid visibility value"), + id="invalid_package_status__invalid_visibility_value", + ), + pytest.param( + "python:\n visibility: public\n importance: high", + pytest.raises(SystemExit, match=r"invalid maturity type"), + id="invalid_package_status__public_visibility_missing_maturity", + ), + pytest.param( + "python:\n visibility: public\n maturity: experimental\n", + pytest.raises(SystemExit, match=r"invalid importance type"), + id="invalid_package_status__public_visibility_missing_importance", + ), + pytest.param( + "python:\n visibility: public\n maturity: immature\n importance: high", + pytest.raises(SystemExit, match=r"invalid maturity value"), + id="invalid_package_status__public_visibility_invalid_maturity", + ), + pytest.param( + "python:\n visibility: public\n maturity: experimental\n importance:" + " extremely", + pytest.raises(SystemExit, match=r"invalid importance value"), + id="invalid_package_status__public_visibility_invalid_importance", + ), + pytest.param( + "python:\n visibility: public\n maturity: immature\n importance:" + " extremely", + pytest.raises( + SystemExit, + match=r"invalid maturity value.*\n.*invalid importance value", + ), + id="invalid_package_status__multiple_errors", + ), + ], +) +def test_package_status_file_type(package_status_file_content: str, expectation): + with expectation: + PackageStatusFile().from_yaml_string(package_status_file_content)