diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 94bf93cfa..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,628 +0,0 @@ -version: 2.1 - -aliases: - - &install-podman - name: Install Podman in Ubuntu Focal - command: ./install/linux/install-podman-ubuntu-focal.sh - - - &install-dependencies-deb - name: Install dependencies (deb) - command: | - export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true - apt-get update - apt build-dep -y . - - - &install-dependencies-rpm - name: Install dependencies (rpm) - command: | - dnf install -y rpm-build python3 python3-devel python3-poetry-core pipx - pipx install poetry - - - &build-deb - name: Build the .deb package - command: | - ./install/linux/build-deb.py - ls -lh deb_dist/ - - - &build-rpm - name: Build the .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py - ls -lh dist/ - - - &build-rpm-qubes - name: Build the Qubes .rpm package - command: | - PATH=/root/.local/bin:$PATH ./install/linux/build-rpm.py --qubes - ls -lh dist/ - - - &calculate-cache-key - name: Caculating container cache key - command: | - mkdir -p /caches/ - cd dangerzone/conversion/ - cat common.py doc_to_pixels.py pixels_to_pdf.py | sha1sum | cut -d' ' -f1 > /caches/cache-id.txt - cd ../../ - - - &restore-cache - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - - ©-image - name: Copy container image into package - command: | - cp /caches/container.tar.gz share/ - cp /caches/image-id.txt share/ - -jobs: - run-lint: - docker: - - image: debian:bookworm - resource_class: small - steps: - - checkout - - run: - name: Install dev. dependencies - # Install only the necessary packages to run our linters. - # - # We run poetry with --no-ansi, to sidestep a Poetry bug that - # currently exists in 1.3. See: - # https://github.com/freedomofpress/dangerzone/issues/292#issuecomment-1351368122 - command: | - apt-get update - apt-get install -y git make python3 python3-poetry --no-install-recommends - poetry install --no-ansi --only lint,test - - run: - name: Run linters to enforce code style - command: poetry run make lint - - run: - name: Check that the QA script is up to date with the docs - command: ./dev_scripts/qa.py --check-refs - - build-container-image: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - # setup_remote_docker - - run: - name: Build Dangerzone image - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - sudo pip3 install poetry - python3 ./install/common/build-image.py - fi - - run: - name: Save Dangerzone image and image-id.txt to cache - command: | - if [ -f "/caches/container.tar.gz" ]; then - echo "Already cached, skipping" - else - mkdir -p /caches - podman save -o /caches/container.tar dangerzone.rocks/dangerzone - gzip -f /caches/container.tar - podman image ls dangerzone.rocks/dangerzone | grep "dangerzone.rocks/dangerzone" | tr -s ' ' | cut -d' ' -f3 > /caches/image-id.txt - fi - - run: *calculate-cache-key - - save_cache: - key: v1-{{ checksum "Dockerfile" }}-{{ checksum "/caches/cache-id.txt" }} - paths: - - /caches/container.tar.gz - - /caches/image-id.txt - - convert-test-docs: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - run: - name: Install poetry dependencies - command: | - sudo pip3 install poetry - # This flag is important, due to an open upstream Poetry issue: - # https://github.com/python-poetry/poetry/issues/7184 - poetry install --no-ansi - - run: - name: Install test dependencies - command: | - sudo apt-get install -y libqt5gui5 libxcb-cursor0 --no-install-recommends - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - run: - name: run automated tests - command: | - poetry run make test - - ci-ubuntu-noble: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 24.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-mantic: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 23.10 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-jammy: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 22.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-ubuntu-focal: - machine: - image: ubuntu-2004:202111-01 - steps: - - checkout - - run: *install-podman - - - run: - name: Prepare cache directory - command: | - sudo mkdir -p /caches - sudo chown -R $USER:$USER /caches - - run: *calculate-cache-key - - restore_cache: *restore-cache - - run: *copy-image - - - run: - name: Prepare Dangerzone environment - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 build-dev - - - run: - name: Run CI tests - command: | - ./dev_scripts/env.py --distro ubuntu --version 20.04 run --dev \ - bash -c 'cd dangerzone; poetry run make test' - - ci-fedora-40: - machine: - image: ubuntu-2204:current - steps: - - checkout - - run: *install-podman - - run: - name: Configure Podman for Ubuntu 22.04 - command: | - # This config circumvents the following issues: - # * https://github.com/containers/podman/issues/6368 - # * https://github.com/containers/podman/issues/10987 - mkdir -p ~/.config/containers - cat > ~/.config/containers/containers.conf \< ~/.config/containers/containers.conf \< ~/.config/containers/containers.conf \< Dockerfile.bullseye \< containers.conf < Dockerfile.bullseye < Tue, 1 Oct 2024 17:02:28 +0300 + dangerzone (0.7.0) unstable; urgency=low * Removed stdeb in favor of direct debian packaging tools diff --git a/dev_scripts/containers.conf b/dev_scripts/containers.conf new file mode 100644 index 000000000..e638b650f --- /dev/null +++ b/dev_scripts/containers.conf @@ -0,0 +1,3 @@ +[engine] +cgroup_manager="cgroupfs" +events_logger="file" diff --git a/dev_scripts/env.py b/dev_scripts/env.py index 61ba88152..8f21ddacd 100755 --- a/dev_scripts/env.py +++ b/dev_scripts/env.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse +import hashlib import os import pathlib import platform @@ -8,6 +9,7 @@ import subprocess import sys import urllib.request +from datetime import date DEFAULT_GUI = True DEFAULT_USER = "user" @@ -56,8 +58,13 @@ # FIXME: Maybe create an enum for these values. DISTROS = ["debian", "fedora", "ubuntu"] CONTAINER_RUNTIMES = ["podman", "docker"] -IMAGE_NAME_BUILD_DEV_FMT = "dangerzone.rocks/build/{distro}:{version}" -IMAGE_NAME_BUILD_FMT = "dangerzone.rocks/{distro}:{version}" +IMAGES_REGISTRY = "ghcr.io/freedomofpress/" +IMAGE_NAME_BUILD_DEV_FMT = ( + IMAGES_REGISTRY + "v2/dangerzone/build-dev/{distro}-{version}:{date}-{hash}" +) +IMAGE_NAME_BUILD_ENDUSER_FMT = ( + IMAGES_REGISTRY + "v2/dangerzone/end-user/{distro}-{version}:{date}-{hash}" +) EPILOG = """\ Examples: @@ -306,14 +313,45 @@ def distro_build(distro, version): return distro_root(distro, version) / "build" -def image_name_build(distro, version): +def get_current_date(): + return date.today().strftime("%Y-%m-%d") + + +def get_build_dir_sources(distro, version): + """Return the files needed to build an image.""" + sources = [ + git_root() / "pyproject.toml", + git_root() / "poetry.lock", + git_root() / "dev_scripts" / "storage.conf", + git_root() / "dev_scripts" / "containers.conf", + ] + + if distro == "ubuntu" and version in ("22.04", "jammy"): + sources.extend( + [ + git_root() / "dev_scripts" / "apt-tools-prod.pref", + git_root() / "dev_scripts" / "apt-tools-prod.sources", + ] + ) + return sources + + +def image_name_build_dev(distro, version): """Get the container image for the dev variant of a Dangerzone environment.""" - return IMAGE_NAME_BUILD_DEV_FMT.format(distro=distro, version=version) + hash = hash_files(get_build_dir_sources(distro, version)) + + return IMAGE_NAME_BUILD_DEV_FMT.format( + distro=distro, version=version, hash=hash, date=get_current_date() + ) -def image_name_install(distro, version): - """Get the container image for the Dangerzone environment.""" - return IMAGE_NAME_BUILD_FMT.format(distro=distro, version=version) +def image_name_build_enduser(distro, version): + """Get the container image for the Dangerzone end-user environment.""" + + hash = hash_files(get_files_in("install/linux", "debian")) + return IMAGE_NAME_BUILD_ENDUSER_FMT.format( + distro=distro, version=version, hash=hash, date=get_current_date() + ) def dz_version(): @@ -322,6 +360,25 @@ def dz_version(): return f.read().strip() +def hash_files(file_paths: list[pathlib.Path]) -> str: + """Returns the hash value of a list of files using the sha256 hashing algorithm.""" + hash_obj = hashlib.new("sha256") + for path in file_paths: + with open(path, "rb") as file: + file_data = file.read() + hash_obj.update(file_data) + + return hash_obj.hexdigest() + + +def get_files_in(*folders: list[str]) -> list[pathlib.Path]: + """Return the list of all files present in the given folders""" + files = [] + for folder in folders: + files.extend([p for p in (git_root() / folder).glob("**") if p.is_file()]) + return files + + class PySide6Manager: """Provision PySide6 RPMs in our Dangerzone environments. @@ -553,13 +610,13 @@ def run( run_cmd += [ "--hostname", "dangerzone-dev", - image_name_build(self.distro, self.version), + image_name_build_dev(self.distro, self.version), ] else: run_cmd += [ "--hostname", "dangerzone", - image_name_install(self.distro, self.version), + image_name_build_enduser(self.distro, self.version), ] run_cmd += cmd @@ -575,8 +632,33 @@ def run( (dist_state / ".bash_history").touch(exist_ok=True) self.runtime_run(*run_cmd) - def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): + def pull_image_from_registry(self, image): + try: + subprocess.run(self.runtime_cmd + ["pull", image], check=True) + return True + except subprocess.CalledProcessError: + # Do not log an error here, we are just checking if the image exists + # on the registry. + return False + + def push_image_to_registry(self, image): + try: + subprocess.run(self.runtime_cmd + ["push", image], check=True) + return True + except subprocess.CalledProcessError as e: + print("An error occured when pulling the image: ", e) + return False + + def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, sync=False): """Build a Linux environment and install tools for Dangerzone development.""" + image = image_name_build_dev(self.distro, self.version) + + if sync and self.pull_image_from_registry(image): + print("Image has been pulled from the registry, no need to build it.") + return + elif sync: + print("Image label not in registry, building it") + if self.distro == "fedora": install_deps = DOCKERFILE_BUILD_DEV_FEDORA_DEPS else: @@ -626,20 +708,18 @@ def build_dev(self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE): os.makedirs(build_dir, exist_ok=True) # Populate the build context. - shutil.copy(git_root() / "pyproject.toml", build_dir) - shutil.copy(git_root() / "poetry.lock", build_dir) - shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) - if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): - shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) - shutil.copy( - git_root() / "dev_scripts" / "apt-tools-prod.sources", build_dir - ) + for source in get_build_dir_sources(self.distro, self.version): + shutil.copy(source, build_dir) + with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_build(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) + if sync: + if not self.push_image_to_registry(image): + print("An error occured while trying to push to the container registry") + def build( self, show_dockerfile=DEFAULT_SHOW_DOCKERFILE, @@ -715,6 +795,7 @@ def build( # Populate the build context. shutil.copy(package_src, package_dst) shutil.copy(git_root() / "dev_scripts" / "storage.conf", build_dir) + shutil.copy(git_root() / "dev_scripts" / "containers.conf", build_dir) if self.distro == "ubuntu" and self.version in ("22.04", "jammy"): shutil.copy(git_root() / "dev_scripts" / "apt-tools-prod.pref", build_dir) shutil.copy( @@ -723,7 +804,7 @@ def build( with open(build_dir / "Dockerfile", mode="w") as f: f.write(dockerfile) - image = image_name_install(self.distro, self.version) + image = image_name_build_enduser(self.distro, self.version) self.runtime_run("build", "-t", image, build_dir) @@ -742,7 +823,7 @@ def env_run(args): def env_build_dev(args): """Invoke the 'build-dev' command based on the CLI args.""" env = Env.from_args(args) - return env.build_dev(show_dockerfile=args.show_dockerfile) + return env.build_dev(show_dockerfile=args.show_dockerfile, sync=args.sync) def env_build(args): @@ -828,6 +909,12 @@ def parse_args(): action="store_true", help="Do not build, only show the Dockerfile", ) + parser_build_dev.add_argument( + "--sync", + default=False, + action="store_true", + help="Attempt to pull the image, build it if not found and push it to the container registry", + ) # Build a development variant of a Dangerzone environment. parser_build = subparsers.add_parser( diff --git a/install/linux/build-deb.py b/install/linux/build-deb.py index 10d73be33..82d87a605 100755 --- a/install/linux/build-deb.py +++ b/install/linux/build-deb.py @@ -64,11 +64,10 @@ def main(): # dpkg-buildpackage produces a .deb file in the parent folder # that needs to be copied to the `deb_dist` folder manually - for item in root.parent.glob(f"dangerzone_{version}_*.deb"): - arch = item.stem.split("_")[-1] - destination = root / "deb_dist" / f"dangerzone_{version}-{deb_ver}_{arch}.deb" - shutil.move(item, destination) - print(f"sudo dpkg -i {destination}") + src = root.parent / f"dangerzone_{version}_amd64.deb" + destination = root / "deb_dist" / f"dangerzone_{version}-{deb_ver}_amd64.deb" + shutil.move(src, destination) + print(f"sudo dpkg -i {destination}") if __name__ == "__main__":