From 491edf413a713d2aa819d6e37fe888138883d5b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Fri, 6 Sep 2024 17:38:23 +0200 Subject: [PATCH] Switch from CircleCI runners to Github actions. As part of this change, the dev (build) and end-user test images names changed from `dangerzone.rocks/*` to `dangerzone/`, to allow using another container registry. For the needs of our CI, we use ghcr.io. --- .circleci/config.yml | 622 ------------------------------------ .github/workflows/build.yml | 103 ++++++ .github/workflows/ci.yml | 261 +++++++++++---- dev_scripts/containers.conf | 3 + dev_scripts/env.py | 101 +++++- 5 files changed, 386 insertions(+), 704 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/build.yml create mode 100644 dev_scripts/containers.conf diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 62e689144..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,622 +0,0 @@ -version: 2.1 - -aliases: - - &install-podman - name: Install Podman in Ubuntu Focal - command: ./install/linux/install-podman-ubuntu-focal.sh - - # FIXME: Remove the following step once we drop Ubuntu Focal support. The - # python-all dependency is an artificial requirement due to an stdeb bug - # prior to v0.9.1. See: - # - # * https://github.com/astraw/stdeb/issues/153 - # * https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1349967888 - - &install-python-all - name: Install python-all package - command: | - export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true - apt-get update - apt-get install -y python-all - - - &install-dependencies-deb - name: Install dependencies (deb) - command: | - export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true - apt-get update - apt-get install -y dh-python python3 python3-stdeb - - - &install-dependencies-rpm - name: Install dependencies (rpm) - command: | - dnf install -y rpm-build python3 python3-devel python3-poetry-core pipx - pipx install poetry - - - &build-deb - name: Build the .deb package - command: | - ./install/linux/build-deb.py - ls -lh deb_dist/ - - - &build-rpm - name: Build the .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py - ls -lh dist/ - - - &build-rpm-qubes - name: Build the Qubes .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py --qubes - ls -lh dist/ - - - &calculate-cache-key - name: Caculating container cache key - command: | - mkdir -p /caches/ - cd dangerzone/conversion/ - cat common.py doc_to_pixels.py pixels_to_pdf.py | sha1sum | cut -d' ' -f1 > /caches/cache-id.txt - cd ../../ - - - &restore-cache - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - - ©-image - name: Copy container image into package - command: | - cp /caches/container.tar.gz share/ - cp /caches/image-id.txt share/ - -jobs: - run-lint: - docker: - - image: debian:bookworm - resource_class: small - steps: - - checkout - - run: - name: Install dev. dependencies - # Install only the necessary packages to run our linters. - # - # We run poetry with --no-ansi, to sidestep a Poetry bug that - # currently exists in 1.3. See: - # https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1351368122 - command: | - apt-get update - apt-get install -y git make python3 python3-poetry --no-install-recommends - poetry install --no-ansi --only lint,test - - run: - name: Run linters to enforce code style - command: poetry run make lint - - run: - name: Check that the QA script is up to date with the docs - command: ./dev_scripts/qa.py --check-refs - - build-container-image: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - # setup_remote_docker - - run: - name: Build Dangerzone image - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - sudo pip3 install poetry - python3 ./install/common/build-image.py - fi - - run: - name: Save Dangerzone image and image-id.txt to cache - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - mkdir -p /caches - podman save -o /caches/container.tar dangerzone.rocks/dangerzone - gzip -f /caches/container.tar - podman image ls dangerzone.rocks/dangerzone | grep "dangerzone.rocks/dangerzone" | tr -s ' ' | cut -d' ' -f3 > /caches/image-id.txt - fi - - run: *calculate-cache-key - - save_cache: - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - convert-test-docs: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Install poetry dependencies - command: | - sudo pip3 install poetry - # This flag is important, due to an open upstream Poetry issue: - # https://github.com/python-poetry/poetry/issues/7184 - poetry install --no-ansi - - run: - name: Install test dependencies - command: | - sudo apt-get install -y libqt5gui5 libxcb-cursor0 --no-install-recommends - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - run: - name: run automated tests - command: | - poetry run make test - - ci-ubuntu-noble: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-mantic: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-jammy: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-focal: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-fedora-40: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro fedora --version 40 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro fedora --version 40 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-fedora-39: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro fedora --version 39 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro fedora --version 39 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-debian-trixie: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro debian --version trixie build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro debian --version trixie run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-debian-bookworm: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro debian --version bookworm build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro debian --version bookworm run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - # NOTE: Making CI tests work in Debian Bullseye requires some tip-toeing - # around certain Podman issues, as you'll see below. Read the following for - # more details: - # - # https://github.com/freedomofpress/dangerzone/issues/388 - ci-debian-bullseye: - machine: - image: ubuntu-2204:2023.04.2 - steps: - - checkout - - run: *install-podman - - run: - name: Configure Podman for Ubuntu 22.04 - command: | - # This config circumvents the following issues: - # * https://github.com/containers/podman/issues/6368 - # * https://github.com/containers/podman/issues/10987 - mkdir -p ~/.config/containers - cat > ~/.config/containers/containers.conf \< Dockerfile.bullseye \< containers.conf < Dockerfile.bullseye < str: + """Returns the hash value of a list of files using the sha256 hashing algorithm. + + If specified, also adds the current date in the hash. + """ + hash_obj = hashlib.new("sha256") + for path in file_paths: + with open(path, "rb") as file: + file_data = file.read() + hash_obj.update(file_data) + + if include_current_date: + current_date = date.today().strftime("%Y-%m-%d") + hash_obj.update(current_date.encode()) + + return hash_obj.hexdigest() + + +def get_files_in(*folders: list[str]) -> list[pathlib.Path]: + """Return the list of all files present in the given folders""" + files = [] + for folder in folders: + files.extend([p for p in (git_root() / folder).glob("**") if p.is_file()]) + return files + + class PySide6Manager: """Provision PySide6 RPMs in our Dangerzone environments. @@ -512,13 +558,13 @@ def run( run_cmd += [ "--hostname", "dangerzone-dev", - image_name_build(self.distro, self.version), + image_name_build_dev(self.distro, self.version), ] else: run_cmd += [ "--hostname", "dangerzone", - image_name_install(self.distro, self.version), + image_name_build_enduser(self.distro, self.version), ] run_cmd += cmd @@ -534,8 +580,26 @@ def run( (dist_state / ".bash_history").touch(exist_ok=True) self.runtime_run(*run_cmd) - def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): + def pull_image_from_registry(self, image): + process = subprocess.run(self.runtime_cmd + ["pull", IMAGES_REGISTRY + image]) + return process.returncode == 0 + + def push_image_to_registry(self, image): + process = subprocess.run( + self.runtime_cmd + ["push", image, IMAGES_REGISTRY + image] + ) + return process.returncode == 0 + + def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, sync=False): """Build a Linux environment and install tools for Dangerzone development.""" + image = image_name_build_dev(self.distro, self.version) + + if sync and self.pull_image_from_registry(image): + # The image has been pulled from the registry, no need to build it. + return + elif sync: + print("Image label not in registry, building it") + if self.distro == "fedora": install_deps = DOCKERFILE_BUILD_DEV_FEDORA_DEPS else: @@ -588,6 +652,7 @@ def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): shutil.copy(git_root() / "pyproject.toml", build_dir) shutil.copy(git_root() / "poetry.lock", build_dir) shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) + shutil.copy(git_root() / "dev_scripts" / "containers.conf", build_dir) if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) shutil.copy( @@ -596,9 +661,12 @@ def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_build(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) + if sync: + if not self.push_image_to_registry(image): + print("An error occured while trying to push to the container registry") + def build( self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, @@ -671,6 +739,7 @@ def build( # Populate the build context. shutil.copy(package_src, package_dst) shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) + shutil.copy(git_root() / "dev_scripts" / "containers.conf", build_dir) if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) shutil.copy( @@ -679,7 +748,7 @@ def build( with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_install(self.distro, self.version) + image = image_name_build_enduser(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) @@ -698,7 +767,7 @@ def env_run(args): def env_build_dev(args): """Invoke the 'build-dev' command based on the CLI args.""" env = Env.from_args(args) - return env.build_dev(show_dockerfile=args.show_dockerfile) + return env.build_dev(show_dockerfile=args.show_dockerfile, sync=args.sync) def env_build(args): @@ -784,6 +853,12 @@ def parse_args(): action="store_true", help="Do not build, only show the Dockerfile", ) + parser_build_dev.add_argument( + "--sync", + default=False, + action="store_true", + help="Attempt to pull the image, build it if not found and push it to the container registry", + ) # Build a development variant of a Dangerzone environment. parser_build = subparsers.add_parser(