From 025e5dda513abb213809faaf8e26a9e83fba1522 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexis=20M=C3=A9taireau?= Date: Fri, 6 Sep 2024 17:38:23 +0200 Subject: [PATCH 1/4] Switch from CircleCI runners to Github actions. As part of this change, the dev (build) and end-user test images names changed from `dangerzone.rocks/*` to `ghcr.io`. A new `--sync` option is provided in the `env.py` command, in order to retrieve the images from the registry, or build and upload otherwise. --- .circleci/config.yml | 628 ------------------------------------ .github/workflows/build.yml | 96 ++++++ .github/workflows/ci.yml | 283 ++++++++++++---- dev_scripts/containers.conf | 3 + dev_scripts/env.py | 129 ++++++-- 5 files changed, 421 insertions(+), 718 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/build.yml create mode 100644 dev_scripts/containers.conf diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 94bf93cfa..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,628 +0,0 @@ -version: 2.1 - -aliases: - - &install-podman - name: Install Podman in Ubuntu Focal - command: ./install/linux/install-podman-ubuntu-focal.sh - - - &install-dependencies-deb - name: Install dependencies (deb) - command: | - export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true - apt-get update - apt build-dep -y . - - - &install-dependencies-rpm - name: Install dependencies (rpm) - command: | - dnf install -y rpm-build python3 python3-devel python3-poetry-core pipx - pipx install poetry - - - &build-deb - name: Build the .deb package - command: | - ./install/linux/build-deb.py - ls -lh deb_dist/ - - - &build-rpm - name: Build the .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py - ls -lh dist/ - - - &build-rpm-qubes - name: Build the Qubes .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py --qubes - ls -lh dist/ - - - &calculate-cache-key - name: Caculating container cache key - command: | - mkdir -p /caches/ - cd dangerzone/conversion/ - cat common.py doc_to_pixels.py pixels_to_pdf.py | sha1sum | cut -d' ' -f1 > /caches/cache-id.txt - cd ../../ - - - &restore-cache - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - - ©-image - name: Copy container image into package - command: | - cp /caches/container.tar.gz share/ - cp /caches/image-id.txt share/ - -jobs: - run-lint: - docker: - - image: debian:bookworm - resource_class: small - steps: - - checkout - - run: - name: Install dev. dependencies - # Install only the necessary packages to run our linters. - # - # We run poetry with --no-ansi, to sidestep a Poetry bug that - # currently exists in 1.3. See: - # https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1351368122 - command: | - apt-get update - apt-get install -y git make python3 python3-poetry --no-install-recommends - poetry install --no-ansi --only lint,test - - run: - name: Run linters to enforce code style - command: poetry run make lint - - run: - name: Check that the QA script is up to date with the docs - command: ./dev_scripts/qa.py --check-refs - - build-container-image: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - # setup_remote_docker - - run: - name: Build Dangerzone image - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - sudo pip3 install poetry - python3 ./install/common/build-image.py - fi - - run: - name: Save Dangerzone image and image-id.txt to cache - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - mkdir -p /caches - podman save -o /caches/container.tar dangerzone.rocks/dangerzone - gzip -f /caches/container.tar - podman image ls dangerzone.rocks/dangerzone | grep "dangerzone.rocks/dangerzone" | tr -s ' ' | cut -d' ' -f3 > /caches/image-id.txt - fi - - run: *calculate-cache-key - - save_cache: - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - convert-test-docs: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Install poetry dependencies - command: | - sudo pip3 install poetry - # This flag is important, due to an open upstream Poetry issue: - # https://github.com/python-poetry/poetry/issues/7184 - poetry install --no-ansi - - run: - name: Install test dependencies - command: | - sudo apt-get install -y libqt5gui5 libxcb-cursor0 --no-install-recommends - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - run: - name: run automated tests - command: | - poetry run make test - - ci-ubuntu-noble: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-mantic: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-jammy: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-focal: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-fedora-40: - machine: - image: ubuntu-2204:current - steps: - - checkout - - run: *install-podman - - run: - name: Configure Podman for Ubuntu 22.04 - command: | - # This config circumvents the following issues: - # * https://github.com/containers/podman/issues/6368 - # * https://github.com/containers/podman/issues/10987 - mkdir -p ~/.config/containers - cat > ~/.config/containers/containers.conf \< ~/.config/containers/containers.conf \< ~/.config/containers/containers.conf \< Dockerfile.bullseye \< containers.conf < Dockerfile.bullseye < str: + """Returns the hash value of a list of files using the sha256 hashing algorithm.""" + hash_obj = hashlib.new("sha256") + for path in file_paths: + with open(path, "rb") as file: + file_data = file.read() + hash_obj.update(file_data) + + return hash_obj.hexdigest() + + +def get_files_in(*folders: list[str]) -> list[pathlib.Path]: + """Return the list of all files present in the given folders""" + files = [] + for folder in folders: + files.extend([p for p in (git_root() / folder).glob("**") if p.is_file()]) + return files + + class PySide6Manager: """Provision PySide6 RPMs in our Dangerzone environments. @@ -553,13 +610,13 @@ def run( run_cmd += [ "--hostname", "dangerzone-dev", - image_name_build(self.distro, self.version), + image_name_build_dev(self.distro, self.version), ] else: run_cmd += [ "--hostname", "dangerzone", - image_name_install(self.distro, self.version), + image_name_build_enduser(self.distro, self.version), ] run_cmd += cmd @@ -575,8 +632,33 @@ def run( (dist_state / ".bash_history").touch(exist_ok=True) self.runtime_run(*run_cmd) - def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): + def pull_image_from_registry(self, image): + try: + subprocess.run(self.runtime_cmd + ["pull", image], check=True) + return True + except subprocess.CalledProcessError: + # Do not log an error here, we are just checking if the image exists + # on the registry. + return False + + def push_image_to_registry(self, image): + try: + subprocess.run(self.runtime_cmd + ["push", image], check=True) + return True + except subprocess.CalledProcessError as e: + print("An error occured when pulling the image: ", e) + return False + + def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, sync=False): """Build a Linux environment and install tools for Dangerzone development.""" + image = image_name_build_dev(self.distro, self.version) + + if sync and self.pull_image_from_registry(image): + print("Image has been pulled from the registry, no need to build it.") + return + elif sync: + print("Image label not in registry, building it") + if self.distro == "fedora": install_deps = DOCKERFILE_BUILD_DEV_FEDORA_DEPS else: @@ -626,20 +708,18 @@ def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): os.makedirs(build_dir, exist_ok=True) # Populate the build context. - shutil.copy(git_root() / "pyproject.toml", build_dir) - shutil.copy(git_root() / "poetry.lock", build_dir) - shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) - if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): - shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) - shutil.copy( - git_root() / "dev_scripts" / "apt-tools-prod.sources", build_dir - ) + for source in get_build_dir_sources(self.distro, self.version): + shutil.copy(source, build_dir) + with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_build(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) + if sync: + if not self.push_image_to_registry(image): + print("An error occured while trying to push to the container registry") + def build( self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, @@ -715,6 +795,7 @@ def build( # Populate the build context. shutil.copy(package_src, package_dst) shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) + shutil.copy(git_root() / "dev_scripts" / "containers.conf", build_dir) if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) shutil.copy( @@ -723,7 +804,7 @@ def build( with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_install(self.distro, self.version) + image = image_name_build_enduser(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) @@ -742,7 +823,7 @@ def env_run(args): def env_build_dev(args): """Invoke the 'build-dev' command based on the CLI args.""" env = Env.from_args(args) - return env.build_dev(show_dockerfile=args.show_dockerfile) + return env.build_dev(show_dockerfile=args.show_dockerfile, sync=args.sync) def env_build(args): @@ -828,6 +909,12 @@ def parse_args(): action="store_true", help="Do not build, only show the Dockerfile", ) + parser_build_dev.add_argument( + "--sync", + default=False, + action="store_true", + help="Attempt to pull the image, build it if not found and push it to the container registry", + ) # Build a development variant of a Dangerzone environment. parser_build = subparsers.add_parser( From a32522f6c87c4d5502c5e8157c00a30daf662e96 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 1 Oct 2024 17:05:07 +0300 Subject: [PATCH 2/4] debian: Bump version to 0.7.1 Add a dummy entry in debian/changelog, to signal that the latest Dangerzone version is 0.7.1. --- debian/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/changelog b/debian/changelog index 86f59f6fd..1d4161811 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +dangerzone (0.7.1) unstable; urgency=low + + * Released Dangerzone 0.7.1 + + -- Freedom of the Press Foundation Tue, 1 Oct 2024 17:02:28 +0300 + dangerzone (0.7.0) unstable; urgency=low * Removed stdeb in favor of direct debian packaging tools From eb2d114ea75b8ade0bf25276c5a30e7ca9eee697 Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 1 Oct 2024 17:06:59 +0300 Subject: [PATCH 3/4] install: Catch version errors when building DEBs Make sure that the Debian package we build conforms to the expected naming scheme else, it's possible that something is off. A scenario we've encountered is bumping `share/version.txt`, but not `debian/changelog`, which would create a Debian package with an older version. --- install/linux/build-deb.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/install/linux/build-deb.py b/install/linux/build-deb.py index 10d73be33..82d87a605 100755 --- a/install/linux/build-deb.py +++ b/install/linux/build-deb.py @@ -64,11 +64,10 @@ def main(): # dpkg-buildpackage produces a .deb file in the parent folder # that needs to be copied to the `deb_dist` folder manually - for item in root.parent.glob(f"dangerzone_{version}_*.deb"): - arch = item.stem.split("_")[-1] - destination = root / "deb_dist" / f"dangerzone_{version}-{deb_ver}_{arch}.deb" - shutil.move(item, destination) - print(f"sudo dpkg -i {destination}") + src = root.parent / f"dangerzone_{version}_amd64.deb" + destination = root / "deb_dist" / f"dangerzone_{version}-{deb_ver}_amd64.deb" + shutil.move(src, destination) + print(f"sudo dpkg -i {destination}") if __name__ == "__main__": From a001b5497c8be7893b0e684f965d8f43665d432f Mon Sep 17 00:00:00 2001 From: Alex Pyrgiotis Date: Tue, 1 Oct 2024 17:27:37 +0300 Subject: [PATCH 4/4] Add release note for Debian packages --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index f3d1b78b0..301113cd0 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -13,6 +13,7 @@ Before making a release, all of these should be complete: - [ ] Update `version` in `pyproject.toml` - [ ] Update `share/version.txt` - [ ] Update the "Version" field in `install/linux/dangerzone.spec` +- [ ] Bump the Debian version by adding a new changelog entry in `debian/changelog` - [ ] Update screenshot in `README.md`, if necessary - [ ] CHANGELOG.md should be updated to include a list of all major changes since the last release