diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 75f8a63368d8..ea8eef40933d 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy branches: - 'backport/**' jobs: + PythonUnitTests: + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Python unit tests + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 -m unittest discover -s . -p '*_test.py' DockerHubPushAarch64: runs-on: [self-hosted, style-checker-aarch64] steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29e3d0c4358a..ea2e1ed33fb5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,8 +52,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type auto - python3 docker_server.py --release-type auto --no-ubuntu \ + python3 docker_server.py --release-type auto --version "${{ github.ref }}" + python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 789abc30b0c9..0ddafc4b5821 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -24,7 +24,7 @@ ClickHouseVersion, get_tagged_versions, get_version_from_repo, - get_version_from_string, + version_arg, ) TEMP_PATH = p.join(RUNNER_TEMP, "docker_images_check") @@ -49,7 +49,8 @@ def parse_args() -> argparse.Namespace: "--version", type=version_arg, default=get_version_from_repo().string, - help="a version to build", + help="a version to build, automaticaly got from version_helper, accepts either " + "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) parser.add_argument( "--release-type", @@ -111,13 +112,6 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def version_arg(version: str) -> ClickHouseVersion: - try: - return get_version_from_string(version) - except ValueError as e: - raise argparse.ArgumentTypeError(e) - - def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: if release_type != "auto": return release_type @@ -125,7 +119,7 @@ def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: git_versions = get_tagged_versions() reference_version = git_versions[0] for i in reversed(range(len(git_versions))): - if git_versions[i] < version: + if git_versions[i] <= version: if i == len(git_versions) - 1: return "latest" reference_version = git_versions[i + 1] @@ -209,7 +203,7 @@ def build_and_push_image( result = [] if os != "ubuntu": tag += f"-{os}" - init_args = ["docker", "buildx", "build"] + init_args = ["docker", "buildx", "build", "--build-arg BUILDKIT_INLINE_CACHE=1"] if push: init_args.append("--push") init_args.append("--output=type=image,push-by-digest=true") diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 0d1c554026f6..23a007ccac05 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -9,7 +9,7 @@ import docker_images_check as di with patch("git_helper.Git"): - from version_helper import get_version_from_string, get_tagged_versions + from version_helper import get_version_from_string import docker_server as ds # di.logging.basicConfig(level=di.logging.INFO) @@ -251,7 +251,8 @@ def test_auto_release_type(self, mock_tagged_versions: MagicMock): get_version_from_string("2.2.1.1"), get_version_from_string("2.2.2.1"), ] - cases = ( + + cases_less = ( (get_version_from_string("1.0.1.1"), "minor"), (get_version_from_string("1.1.2.1"), "minor"), (get_version_from_string("1.3.1.1"), "major"), @@ -260,8 +261,18 @@ def test_auto_release_type(self, mock_tagged_versions: MagicMock): (get_version_from_string("2.2.3.1"), "latest"), (get_version_from_string("2.3.1.1"), "latest"), ) - _ = get_tagged_versions() - for case in cases: + for case in cases_less: + release = ds.auto_release_type(case[0], "auto") + self.assertEqual(case[1], release) + + cases_equal = ( + (get_version_from_string("1.1.1.1"), "minor"), + (get_version_from_string("1.2.1.1"), "major"), + (get_version_from_string("2.1.1.1"), "minor"), + (get_version_from_string("2.2.1.1"), "patch"), + (get_version_from_string("2.2.2.1"), "latest"), + ) + for case in cases_equal: release = ds.auto_release_type(case[0], "auto") self.assertEqual(case[1], release) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 5c02ea53cf89..50414ffb470d 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -93,7 +93,7 @@ def check_tag(self, value: str): if value == "": return if not self._tag_pattern.match(value): - raise Exception(f"last tag {value} doesn't match the pattern") + raise ValueError(f"last tag {value} doesn't match the pattern") @property def latest_tag(self) -> str: diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index ccbf19186021..be977bdd907c 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -40,13 +40,12 @@ def __init__(self, version: str): "_".join((name, version, arch + ".deb")) for name, arch in self.packages ) - rev = "2" self.rpm = tuple( - "-".join((name, version, rev + "." + self.rpm_arch[arch] + ".rpm")) + "-".join((name, version + "." + self.rpm_arch[arch] + ".rpm")) for name, arch in self.packages ) - self.tgz = tuple(f"{name}-{version}.tgz" for name, _ in self.packages) + self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages) def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 7db96cfde7cf..9c67191e4c3c 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import logging import os.path as p -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError from typing import Dict, List, Tuple, Union from git_helper import Git, removeprefix @@ -150,6 +150,9 @@ def __lt__(self, other: "ClickHouseVersion") -> bool: return False + def __le__(self, other: "ClickHouseVersion") -> bool: + return self == other or self < other + class VersionType: LTS = "lts" @@ -217,6 +220,20 @@ def get_version_from_tag(tag: str) -> ClickHouseVersion: return get_version_from_string(tag) +def version_arg(version: str) -> ClickHouseVersion: + version = removeprefix(version, "refs/tags/") + try: + return get_version_from_string(version) + except ValueError: + pass + try: + return get_version_from_tag(version) + except ValueError: + pass + + raise ArgumentTypeError(f"version {version} does not match tag of plain version") + + def get_tagged_versions() -> List[ClickHouseVersion]: versions = [] for tag in git.get_tags(): diff --git a/tests/ci/version_test.py b/tests/ci/version_test.py new file mode 100644 index 000000000000..86a2d58c3c8b --- /dev/null +++ b/tests/ci/version_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import unittest +from argparse import ArgumentTypeError + +import version_helper as vh + + +class TestFunctions(unittest.TestCase): + def test_version_arg(self): + cases = ( + ("0.0.0.0", vh.get_version_from_string("0.0.0.0")), + ("1.1.1.2", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-lts", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-prestable", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-stable", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), + ("refs/tags/v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), + ) + for case in cases: + version = vh.version_arg(case[0]) + self.assertEqual(case[1], version) + error_cases = ( + "0.0.0", + "1.1.1.a", + "1.1.1.1.1", + "1.1.1.2-testing", + "v1.1.1.2-testin", + "refs/tags/v1.1.1.2-testin", + ) + for case in error_cases: + with self.assertRaises(ArgumentTypeError): + version = vh.version_arg(case[0])