Add aarch64 fdb build & Update lib features

.github/workflows/ci.yml

@@ -24,6 +24,134 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  fdb:
+    strategy:
+      fail-fast: false
+    name: Build FoundationDB image
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+      attestations: write
+      packages: write
+    needs: [multiarch]
+    if: github.event_name == 'push' || inputs.Docker
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: "arm64"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: |
+            network=host
+
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@v3
+
+      - name: Log In to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{github.repository_owner}}
+          password: ${{github.token}}
+
+      - name: Log In to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{secrets.DOCKERHUB_USERNAME}}
+          password: ${{secrets.DOCKERHUB_TOKEN}}
+
+      - name: Get musl tags
+        uses: actions/download-artifact@v4
+        with:
+          name: bake-meta-musl
+
+      - name: Get gnu digests
+        uses: actions/download-artifact@v4
+        with:
+          pattern: digests-*-gnu
+          merge-multiple: false
+
+      - name: Prepare fdb tags & files
+        run: |
+          sed -i 's#alpine#fdb#g' bake-meta.json
+          echo "arm64_digest=$(printf '%s' digests-gnu-aarch64-*/* | awk -F '/' '{print $2}')" | tee -a "${GITHUB_ENV}"
+          echo "amd64_digest=$(printf '%s' digests-gnu-x86_64-*/* | awk -F '/' '{print $2}')" | tee -a "${GITHUB_ENV}"
+
+      - name: Build FDB arm64
+        id: arm64
+        uses: docker/bake-action@v6
+        env:
+          DOCKER_BUILD_RECORD_UPLOAD: false
+          TARGET: aarch64-unknown-linux-gnu
+          GHCR_REPO: ghcr.io/${{github.repository}}
+          BUILD_ENV: "JEMALLOC_SYS_WITH_LG_PAGE=16"
+        with:
+          source: .
+          set: |
+            *.tags=
+            fdb.platform=linux/arm64
+            fdb.args.BASE_IMG=ghcr.io/${{github.repository}}@sha256:${{env.arm64_digest}}
+            fdb.output=type=image,"name=ghcr.io/${{github.repository}},index.docker.io/${{github.repository}}",push-by-digest=true,push=true,compression=zstd,compression-level=9,force-compression=true,oci-mediatypes=true
+          files: |
+            docker-bake.hcl
+            bake-meta.json
+          targets: fdb
+
+      - name: Build FDB amd64
+        id: amd64
+        uses: docker/bake-action@v6
+        env:
+          DOCKER_BUILD_RECORD_UPLOAD: false
+          TARGET: x86_64-unknown-linux-gnu
+          GHCR_REPO: ghcr.io/${{github.repository}}
+        with:
+          source: .
+          set: |
+            *.tags=
+            fdb.platform=linux/amd64
+            fdb.args.BASE_IMG=ghcr.io/${{github.repository}}@sha256:${{env.amd64_digest}}
+            fdb.output=type=image,"name=ghcr.io/${{github.repository}},index.docker.io/${{github.repository}}",push-by-digest=true,push=true,compression=zstd,compression-level=9,force-compression=true,oci-mediatypes=true
+          files: |
+            docker-bake.hcl
+            bake-meta.json
+          targets: fdb
+
+      - name: Generate manifest & apply cosign
+        run: |
+          docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("ghcr.io/${{github.repository}}")) | "-t " + .) | join(" ")' bake-meta.json) \
+            ghcr.io/${{github.repository}}@${{ fromJSON(steps.amd64.outputs.metadata).fdb['containerimage.digest'] }} \
+            ghcr.io/${{github.repository}}@${{ fromJSON(steps.arm64.outputs.metadata).fdb['containerimage.digest'] }}
+          docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | "-t " + .) | join(" ")' bake-meta.json) \
+            index.docker.io/${{github.repository}}@${{ fromJSON(steps.amd64.outputs.metadata).fdb['containerimage.digest'] }} \
+            index.docker.io/${{github.repository}}@${{ fromJSON(steps.arm64.outputs.metadata).fdb['containerimage.digest'] }}
+          docker buildx imagetools inspect --format '{{json .Manifest}}' ghcr.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' bake-meta.json) | jq -r '.digest' > GHCR_DIGEST_SHA
+          echo "GHCR_DIGEST_SHA=$(cat GHCR_DIGEST_SHA)" | tee -a "${GITHUB_ENV}"
+          docker buildx imagetools inspect --format '{{json .Manifest}}' index.docker.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' bake-meta.json) | jq -r '.digest' > DOCKERHUB_DIGEST_SHA
+          echo "DOCKERHUB_DIGEST_SHA=$(cat DOCKERHUB_DIGEST_SHA)" | tee -a "${GITHUB_ENV}"
+          cosign sign --yes $(jq --arg GHCR_DIGEST_SHA "$(cat GHCR_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("ghcr.io/${{github.repository}}")) | . + "@" + $GHCR_DIGEST_SHA) | join(" ")' bake-meta.json)
+          cosign sign --yes $(jq --arg DOCKERHUB_DIGEST_SHA "$(cat DOCKERHUB_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | . + "@" + $DOCKERHUB_DIGEST_SHA) | join(" ")' bake-meta.json)
+
+      - name: Attest GHCR
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: ghcr.io/${{github.repository}}
+          subject-digest: ${{ env.GHCR_DIGEST_SHA }}
+          push-to-registry: true
+
+      - name: Attest Dockerhub
+        uses: actions/attest-build-provenance@v2
+        with:
+          subject-name: index.docker.io/${{github.repository}}
+          subject-digest: ${{ env.DOCKERHUB_DIGEST_SHA }}
+          push-to-registry: true
+
   multiarch:
     strategy:
       fail-fast: false
@@ -41,6 +169,9 @@ jobs:
     needs: [linux]
     if: github.event_name == 'push' || inputs.Docker
     steps:
+      - name: Install Cosign
+        uses: sigstore/cosign-installer@v3
+
       - name: Log In to GitHub Container Registry
         uses: docker/login-action@v3
         with:
@@ -75,13 +206,15 @@ jobs:
           docker buildx imagetools create $(jq -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | "-t " + .) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) \
             $(printf 'index.docker.io/${{github.repository}}@sha256:%s ' *)
 
-      - name: Inspect ${{matrix.variant}} image
-        id: manifest-digest
+      - name: Sign ${{matrix.variant}} image with cosign
+        id: digest
         run: |
           docker buildx imagetools inspect --format '{{json .Manifest}}' ghcr.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) | jq -r '.digest' > GHCR_DIGEST_SHA
           echo "GHCR_DIGEST_SHA=$(cat GHCR_DIGEST_SHA)" | tee -a "${GITHUB_ENV}"
           docker buildx imagetools inspect --format '{{json .Manifest}}' index.docker.io/${{github.repository}}:$(jq -r '.target."docker-metadata-action".args.DOCKER_META_VERSION' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json) | jq -r '.digest' > DOCKERHUB_DIGEST_SHA
           echo "DOCKERHUB_DIGEST_SHA=$(cat DOCKERHUB_DIGEST_SHA)" | tee -a "${GITHUB_ENV}"
+          cosign sign --yes $(jq --arg GHCR_DIGEST_SHA "$(cat GHCR_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("ghcr.io/${{github.repository}}")) | . + "@" + $GHCR_DIGEST_SHA) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json)
+          cosign sign --yes $(jq --arg DOCKERHUB_DIGEST_SHA "$(cat DOCKERHUB_DIGEST_SHA)" -cr '.target."docker-metadata-action".tags | map(select(startswith("index.docker.io/${{github.repository}}")) | . + "@" + $DOCKERHUB_DIGEST_SHA) | join(" ")' ${{ runner.temp }}/${{matrix.variant}}/bake-meta.json)
 
       - name: Attest GHCR
         uses: actions/attest-build-provenance@v2
@@ -331,10 +464,12 @@ jobs:
           disable_annotations: true
 
       - name: Build FoundationDB Edition
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: |
           rustup target add ${{matrix.target}}
           # Get latest FoundationDB installer
-          curl -Lo foundationdb.pkg "https://glare.now.sh/apple/foundationdb/${{startsWith(matrix.target, 'x86') && 'x86_64' || 'arm64'}}.pkg"
+          curl --retry 5 -Lso foundationdb.pkg "$(gh api repos/apple/foundationdb/releases/latest --jq '.assets[] | select(.name | test("${{startsWith(matrix.target, 'x86') && 'x86_64' || 'arm64'}}.pkg$")) | .browser_download_url')"
           sudo installer -allowUntrusted -dumplog -pkg foundationdb.pkg -target /
           cargo build --release --target ${{matrix.target}} -p mail-server --no-default-features --features "foundationdb elastic s3 redis enterprise"
           mkdir -p artifacts
@@ -405,16 +540,25 @@ jobs:
             archive/**/*.tar.gz
             archive/**/*.zip
 
+      - name: Use cosign to sign existing artifacts
+        uses: sigstore/[email protected]
+        with:
+          inputs: |
+            archive/**/*.tar.gz
+            archive/**/*.zip
+
       - name: Release
         uses: softprops/action-gh-release@v2
         with:
           files: |
             archive/**/*.tar.gz
             archive/**/*.zip
+            archive/**/*.sigstore.json
           prerelease: ${{!startsWith(github.ref, 'refs/tags/') || null}}
           tag_name: ${{!startsWith(github.ref, 'refs/tags/') && 'nightly' || null}}
           append_body: true
+          # TODO add instructions about using cosign to verify binary artifact
           body: |
             <hr />
 
-            ## Check binary attestation at [here](${{ steps.attest.outputs.attestation-url }})
+            ### Check binary attestation at [here](${{ steps.attest.outputs.attestation-url }})

Dockerfile.build

@@ -1,6 +1,7 @@
 # syntax=docker/dockerfile:1
-# check=skip=FromPlatformFlagConstDisallowed,RedundantTargetPlatform
+# check=skip=FromPlatformFlagConstDisallowed,RedundantTargetPlatform,InvalidDefaultArgInFrom
 
+ARG BASE_IMG=scratch
 # *****************
 # Base image for planner & builder
 # *****************
@@ -14,24 +15,20 @@ ENV DEBIAN_FRONTEND="noninteractive" \
     TERM=xterm-256color
 # With zig, we only need libclang and make
 RUN \
-  --mount=type=cache,target=/var/cache/apt,sharing=locked \
-  --mount=type=cache,target=/var/lib/apt,sharing=locked \
-  rm -f /etc/apt/apt.conf.d/docker-clean && \
-  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
-  apt-get update && \
-  apt-get install -yq --no-install-recommends curl jq xz-utils make libclang-16-dev
+    --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    rm -f /etc/apt/apt.conf.d/docker-clean && \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
+    apt-get update && \
+    apt-get install -yq --no-install-recommends curl jq xz-utils make libclang-16-dev
 # Install zig
-RUN ZIG_VERSION=$(curl --retry 5 -sL "https://api.github.com/repos/ziglang/zig/releases/latest" | jq -r '.tag_name') && \
+RUN \
+    ZIG_VERSION=$(curl --retry 5 -sL "https://api.github.com/repos/ziglang/zig/releases/latest" | jq -r '.tag_name') && \
     [ ! -z "$ZIG_VERSION" ] && \
     curl --retry 5 -Ls "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-$(uname -m)-${ZIG_VERSION}.tar.xz" | tar -J -x -C /usr/local && \
     ln -s "/usr/local/zig-linux-$(uname -m)-${ZIG_VERSION}/zig" /usr/local/bin/zig
 # Install cargo-binstall
 RUN curl --retry 5 -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
-# Install FoundationDB
-# TODO According to https://github.com/apple/foundationdb/issues/11448#issuecomment-2417766293
-# Once FoundationDB v7.3.53 gets released, we should be able to build the aarch64-unknown-linux-gnu target.
-# The last command is for future build use, so if you are building on a native arm64 device, please use docker qemu.
-RUN curl --retry 5 -Lso /usr/lib/libfdb_c.so "$(curl --retry 5 -Ls 'https://api.github.com/repos/apple/foundationdb/releases' | jq --arg arch "$(uname -m)" -r '.[] | select(.prerelease == false) | .assets[] | select(.name | test("libfdb_c." + $arch + ".so")) | .browser_download_url' | head -n1)"
 # Install cargo-chef & sccache & cargo-zigbuild
 RUN cargo binstall --no-confirm cargo-chef sccache cargo-zigbuild
 
@@ -55,24 +52,31 @@ ARG BUILD_ENV
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 # Install toolchain and specify some env variables
 RUN \
-  rustup set profile minimal && \
-  rustup target add ${TARGET} && \
-  mkdir -p artifact && \
-  touch /env-cargo && \
-  if [ ! -z "${BUILD_ENV}" ]; then \
-      echo "export ${BUILD_ENV}" >> /env-cargo; \
-      echo "Setting up ${BUILD_ENV}"; \
-  fi
+    rustup set profile minimal && \
+    rustup target add ${TARGET} && \
+    mkdir -p artifact && \
+    touch /env-cargo && \
+    if [ ! -z "${BUILD_ENV}" ]; then \
+        echo "export ${BUILD_ENV}" >> /env-cargo; \
+        echo "Setting up ${BUILD_ENV}"; \
+    fi && \
+    if [[ "${TARGET}" == *gnu ]]; then \
+        echo "export FDB_ARCH=${TARGET%%-*}" >> /env-cargo; \
+    fi
+# Install FoundationDB
+RUN \
+    source /env-cargo && \
+    if [ ! -z "${FDB_ARCH}" ]; then \
+        curl --retry 5 -Lso /usr/lib/libfdb_c.so "$(curl --retry 5 -Ls 'https://api.github.com/repos/apple/foundationdb/releases' | jq --arg FDB_ARCH "$FDB_ARCH" -r '.[] | select(.prerelease == false) | .assets[] | select(.name | test("libfdb_c." + $FDB_ARCH + ".so")) | .browser_download_url' | head -n1)"; \
+    fi
 # Cargo-chef Cache layer
 RUN \
     --mount=type=secret,id=ACTIONS_CACHE_URL,env=ACTIONS_CACHE_URL \
     --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \
     --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
-    # TODO According to https://github.com/apple/foundationdb/issues/11448#issuecomment-2417766293
-    # Once FoundationDB v7.3.53 gets released, we should be able to build the aarch64-unknown-linux-gnu target.
     source /env-cargo && \
-    if [ "${TARGET}" = "x86_64-unknown-linux-gnu" ]; then \
+    if [ ! -z "${FDB_ARCH}" ]; then \
         RUSTFLAGS="-L /usr/lib" cargo chef cook --recipe-path recipe.json --zigbuild --release --target ${TARGET} -p mail-server --no-default-features --features "foundationdb elastic s3 redis enterprise"; \
     fi
 RUN \
@@ -87,16 +91,14 @@ RUN \
 COPY . .
 ENV RUSTC_WRAPPER="sccache" \
     SCCACHE_GHA_ENABLED=true
-# Build foundationdb version
+# Build FoundationDB version
 RUN \
     --mount=type=secret,id=ACTIONS_CACHE_URL,env=ACTIONS_CACHE_URL \
     --mount=type=secret,id=ACTIONS_RUNTIME_TOKEN,env=ACTIONS_RUNTIME_TOKEN \
     --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/usr/local/cargo/git \
-    # TODO According to https://github.com/apple/foundationdb/issues/11448#issuecomment-2417766293
-    # Once FoundationDB v7.3.53 gets released, we should be able to build the aarch64-unknown-linux-gnu target.
     source /env-cargo && \
-    if [ "${TARGET}" = "x86_64-unknown-linux-gnu" ]; then \
+    if [ ! -z "${FDB_ARCH}" ]; then \
         RUSTFLAGS="-L /usr/lib" cargo zigbuild --release --target ${TARGET} -p mail-server --no-default-features --features "foundationdb elastic s3 redis enterprise"; \
         mv /app/target/${TARGET}/release/stalwart-mail /app/artifact/stalwart-mail-foundationdb; \
     fi
@@ -149,3 +151,11 @@ CMD ["/usr/local/bin/stalwart-mail"]
 VOLUME [ "/opt/stalwart-mail" ]
 EXPOSE	443 25 110 587 465 143 993 995 4190 8080
 ENTRYPOINT ["/bin/sh", "/usr/local/bin/entrypoint.sh"]
+
+# *****************
+# Runtime image for fdb targets
+# *****************
+FROM ${BASE_IMG} AS fdb
+COPY --from=builder /app/artifact/stalwart-mail-foundationdb /usr/local/bin/stalwart-mail
+COPY --from=builder /usr/lib/libfdb_c.so /usr/lib/libfdb_c.so
+ENTRYPOINT ["/usr/local/bin/stalwart-mail", "--config", "/opt/stalwart-mail/etc/config.toml"]

crates/store/Cargo.toml

@@ -9,7 +9,7 @@ utils = { path = "../utils" }
 nlp = { path = "../nlp" }
 trc = { path = "../trc" }
 rocksdb = { version = "0.23", optional = true, features = ["multi-threaded-cf"] }
-foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_1"], optional = true }
+foundationdb = { version = "0.9.2", features = ["embedded-fdb-include", "fdb-7_3"], optional = true }
 rusqlite = { version = "0.32", features = ["bundled"], optional = true }
 rust-s3 = { version = "=0.35.0-alpha.2", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true }
 azure_core = { version = "0.21.0", optional = true }

docker-bake.hcl

@@ -48,3 +48,22 @@ target "image" {
     ""
   ]
 }
+target "fdb" {
+  inherits = ["docker-metadata-action"]
+  platforms = [
+    ""
+  ]
+  args = {
+    TARGET = "${TARGET}"
+    BUILD_ENV = equal("", "${BUILD_ENV}") ? null : "${BUILD_ENV}"
+  }
+  target = "fdb"
+  cache-from = [
+    "type=registry,ref=${GHCR_REPO}-buildcache:${TARGET}"
+  ]
+  context = "./"
+  dockerfile = "Dockerfile.build"
+  output = [
+    ""
+  ]
+}

resources/docker/Dockerfile.fdb

@@ -1,10 +1,10 @@
-FROM debian:buster-slim AS chef 
+FROM debian:buster-slim AS chef
 RUN apt-get update && \
     export DEBIAN_FRONTEND=noninteractive && \
     apt-get install -yq \
     build-essential \
     cmake \
-    clang \ 
+    clang \
     curl \
     protobuf-compiler
 ENV RUSTUP_HOME=/opt/rust/rustup \
@@ -28,7 +28,7 @@ COPY resources/ resources/
 COPY tests/ tests/
 RUN cargo chef prepare --recipe-path recipe.json
 
-FROM chef AS builder 
+FROM chef AS builder
 COPY --from=planner /app/recipe.json recipe.json
 RUN cargo chef cook --release --recipe-path recipe.json
 COPY Cargo.toml .