From 84020f5bbccc5b711d0a533b472f35227f25878c Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 6 Apr 2022 18:32:54 +0200 Subject: [PATCH 1/6] Tiny improvements to git and version helpers --- tests/ci/git_helper.py | 2 +- tests/ci/version_helper.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ci/git_helper.py b/tests/ci/git_helper.py index 5c02ea53cf89..50414ffb470d 100644 --- a/tests/ci/git_helper.py +++ b/tests/ci/git_helper.py @@ -93,7 +93,7 @@ def check_tag(self, value: str): if value == "": return if not self._tag_pattern.match(value): - raise Exception(f"last tag {value} doesn't match the pattern") + raise ValueError(f"last tag {value} doesn't match the pattern") @property def latest_tag(self) -> str: diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index 7db96cfde7cf..ecccc250e333 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -150,6 +150,9 @@ def __lt__(self, other: "ClickHouseVersion") -> bool: return False + def __le__(self, other: "ClickHouseVersion") -> bool: + return self == other or self < other + class VersionType: LTS = "lts" From 0a43cfe56548d2d302079dcd529b9b1ad9b1e298 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Wed, 6 Apr 2022 18:41:18 +0200 Subject: [PATCH 2/6] Improve and fix edge cases for docker_server.py - Allow define version as file - Add inline cache - Fix auto_release_type function --- tests/ci/docker_server.py | 22 +++++++++++++++++----- tests/ci/docker_test.py | 19 +++++++++++++++---- 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index 789abc30b0c9..a274cbbf170a 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -16,6 +16,7 @@ from docker_images_check import DockerImage from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET from get_robot_token import get_best_robot_token, get_parameter_from_ssm +from git_helper import removeprefix from pr_info import PRInfo from s3_helper import S3Helper from stopwatch import Stopwatch @@ -25,6 +26,7 @@ get_tagged_versions, get_version_from_repo, get_version_from_string, + get_version_from_tag, ) TEMP_PATH = p.join(RUNNER_TEMP, "docker_images_check") @@ -49,7 +51,8 @@ def parse_args() -> argparse.Namespace: "--version", type=version_arg, default=get_version_from_repo().string, - help="a version to build", + help="a version to build, automaticaly got from version_helper, accepts either " + "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format", ) parser.add_argument( "--release-type", @@ -112,10 +115,19 @@ def parse_args() -> argparse.Namespace: def version_arg(version: str) -> ClickHouseVersion: + version = removeprefix(version, "refs/tags/") try: return get_version_from_string(version) - except ValueError as e: - raise argparse.ArgumentTypeError(e) + except ValueError: + pass + try: + return get_version_from_tag(version) + except ValueError: + pass + + raise argparse.ArgumentTypeError( + f"version {version} does not match tag of plain version" + ) def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: @@ -125,7 +137,7 @@ def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: git_versions = get_tagged_versions() reference_version = git_versions[0] for i in reversed(range(len(git_versions))): - if git_versions[i] < version: + if git_versions[i] <= version: if i == len(git_versions) - 1: return "latest" reference_version = git_versions[i + 1] @@ -209,7 +221,7 @@ def build_and_push_image( result = [] if os != "ubuntu": tag += f"-{os}" - init_args = ["docker", "buildx", "build"] + init_args = ["docker", "buildx", "build", "--build-arg BUILDKIT_INLINE_CACHE=1"] if push: init_args.append("--push") init_args.append("--output=type=image,push-by-digest=true") diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 0d1c554026f6..23a007ccac05 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -9,7 +9,7 @@ import docker_images_check as di with patch("git_helper.Git"): - from version_helper import get_version_from_string, get_tagged_versions + from version_helper import get_version_from_string import docker_server as ds # di.logging.basicConfig(level=di.logging.INFO) @@ -251,7 +251,8 @@ def test_auto_release_type(self, mock_tagged_versions: MagicMock): get_version_from_string("2.2.1.1"), get_version_from_string("2.2.2.1"), ] - cases = ( + + cases_less = ( (get_version_from_string("1.0.1.1"), "minor"), (get_version_from_string("1.1.2.1"), "minor"), (get_version_from_string("1.3.1.1"), "major"), @@ -260,8 +261,18 @@ def test_auto_release_type(self, mock_tagged_versions: MagicMock): (get_version_from_string("2.2.3.1"), "latest"), (get_version_from_string("2.3.1.1"), "latest"), ) - _ = get_tagged_versions() - for case in cases: + for case in cases_less: + release = ds.auto_release_type(case[0], "auto") + self.assertEqual(case[1], release) + + cases_equal = ( + (get_version_from_string("1.1.1.1"), "minor"), + (get_version_from_string("1.2.1.1"), "major"), + (get_version_from_string("2.1.1.1"), "minor"), + (get_version_from_string("2.2.1.1"), "patch"), + (get_version_from_string("2.2.2.1"), "latest"), + ) + for case in cases_equal: release = ds.auto_release_type(case[0], "auto") self.assertEqual(case[1], release) From ee1b577394dbee6fe8425bbba6a9051a4dd2a8bf Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 12:23:15 +0200 Subject: [PATCH 3/6] Fix action for docker images build --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29e3d0c4358a..ea2e1ed33fb5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,8 +52,8 @@ jobs: - name: Check docker clickhouse/clickhouse-server building run: | cd "$GITHUB_WORKSPACE/tests/ci" - python3 docker_server.py --release-type auto - python3 docker_server.py --release-type auto --no-ubuntu \ + python3 docker_server.py --release-type auto --version "${{ github.ref }}" + python3 docker_server.py --release-type auto --version "${{ github.ref }}" --no-ubuntu \ --image-repo clickhouse/clickhouse-keeper --image-path docker/keeper - name: Cleanup if: always() From 7d6fd3d065cd31792578cd0418cc63339900ad53 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 12:25:00 +0200 Subject: [PATCH 4/6] A temporary fix for artifactory push before multiple architectures --- tests/ci/push_to_artifactory.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/ci/push_to_artifactory.py b/tests/ci/push_to_artifactory.py index ccbf19186021..be977bdd907c 100755 --- a/tests/ci/push_to_artifactory.py +++ b/tests/ci/push_to_artifactory.py @@ -40,13 +40,12 @@ def __init__(self, version: str): "_".join((name, version, arch + ".deb")) for name, arch in self.packages ) - rev = "2" self.rpm = tuple( - "-".join((name, version, rev + "." + self.rpm_arch[arch] + ".rpm")) + "-".join((name, version + "." + self.rpm_arch[arch] + ".rpm")) for name, arch in self.packages ) - self.tgz = tuple(f"{name}-{version}.tgz" for name, _ in self.packages) + self.tgz = tuple(f"{name}-{version}-amd64.tgz" for name, _ in self.packages) def arch(self, deb_pkg: str) -> str: if deb_pkg not in self.deb: From e8168a1927ad54e2189a137fb87bc671fbb4386a Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 12:37:46 +0200 Subject: [PATCH 5/6] Add python unit tests to backport workflow --- .github/workflows/backport_branches.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 75f8a63368d8..ea8eef40933d 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -9,6 +9,18 @@ on: # yamllint disable-line rule:truthy branches: - 'backport/**' jobs: + PythonUnitTests: + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Python unit tests + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 -m unittest discover -s . -p '*_test.py' DockerHubPushAarch64: runs-on: [self-hosted, style-checker-aarch64] steps: From a365ef56cdbb320b00a455f24c08f9df9638c01e Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 7 Apr 2022 13:48:52 +0200 Subject: [PATCH 6/6] Move version_arg to version_helper, add tests --- tests/ci/docker_server.py | 20 +------------------- tests/ci/version_helper.py | 16 +++++++++++++++- tests/ci/version_test.py | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 20 deletions(-) create mode 100644 tests/ci/version_test.py diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py index a274cbbf170a..0ddafc4b5821 100644 --- a/tests/ci/docker_server.py +++ b/tests/ci/docker_server.py @@ -16,7 +16,6 @@ from docker_images_check import DockerImage from env_helper import CI, GITHUB_RUN_URL, RUNNER_TEMP, S3_BUILDS_BUCKET from get_robot_token import get_best_robot_token, get_parameter_from_ssm -from git_helper import removeprefix from pr_info import PRInfo from s3_helper import S3Helper from stopwatch import Stopwatch @@ -25,8 +24,7 @@ ClickHouseVersion, get_tagged_versions, get_version_from_repo, - get_version_from_string, - get_version_from_tag, + version_arg, ) TEMP_PATH = p.join(RUNNER_TEMP, "docker_images_check") @@ -114,22 +112,6 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def version_arg(version: str) -> ClickHouseVersion: - version = removeprefix(version, "refs/tags/") - try: - return get_version_from_string(version) - except ValueError: - pass - try: - return get_version_from_tag(version) - except ValueError: - pass - - raise argparse.ArgumentTypeError( - f"version {version} does not match tag of plain version" - ) - - def auto_release_type(version: ClickHouseVersion, release_type: str) -> str: if release_type != "auto": return release_type diff --git a/tests/ci/version_helper.py b/tests/ci/version_helper.py index ecccc250e333..9c67191e4c3c 100755 --- a/tests/ci/version_helper.py +++ b/tests/ci/version_helper.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 import logging import os.path as p -from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter +from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter, ArgumentTypeError from typing import Dict, List, Tuple, Union from git_helper import Git, removeprefix @@ -220,6 +220,20 @@ def get_version_from_tag(tag: str) -> ClickHouseVersion: return get_version_from_string(tag) +def version_arg(version: str) -> ClickHouseVersion: + version = removeprefix(version, "refs/tags/") + try: + return get_version_from_string(version) + except ValueError: + pass + try: + return get_version_from_tag(version) + except ValueError: + pass + + raise ArgumentTypeError(f"version {version} does not match tag of plain version") + + def get_tagged_versions() -> List[ClickHouseVersion]: versions = [] for tag in git.get_tags(): diff --git a/tests/ci/version_test.py b/tests/ci/version_test.py new file mode 100644 index 000000000000..86a2d58c3c8b --- /dev/null +++ b/tests/ci/version_test.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python + +import unittest +from argparse import ArgumentTypeError + +import version_helper as vh + + +class TestFunctions(unittest.TestCase): + def test_version_arg(self): + cases = ( + ("0.0.0.0", vh.get_version_from_string("0.0.0.0")), + ("1.1.1.2", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-lts", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-prestable", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-stable", vh.get_version_from_string("1.1.1.2")), + ("v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), + ("refs/tags/v1.1.1.2-testing", vh.get_version_from_string("1.1.1.2")), + ) + for case in cases: + version = vh.version_arg(case[0]) + self.assertEqual(case[1], version) + error_cases = ( + "0.0.0", + "1.1.1.a", + "1.1.1.1.1", + "1.1.1.2-testing", + "v1.1.1.2-testin", + "refs/tags/v1.1.1.2-testin", + ) + for case in error_cases: + with self.assertRaises(ArgumentTypeError): + version = vh.version_arg(case[0])