diff --git a/Dockerfile b/Dockerfile
index 25b314a1115..7a041e83498 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,7 +13,7 @@
 # limitations under the License.
 
 # Velero binary build section
-FROM --platform=$BUILDPLATFORM golang:1.22-bookworm AS velero-builder
+FROM --platform=$BUILDPLATFORM golang:1.22-bookworm AS velero-builder-base
 
 ARG GOPROXY
 ARG BIN
@@ -36,15 +36,20 @@ ENV CGO_ENABLED=0 \
 
 WORKDIR /go/src/github.com/vmware-tanzu/velero
 
+COPY go.mod go.sum /go/src/github.com/vmware-tanzu/velero/
+RUN go mod download
+
 COPY . /go/src/github.com/vmware-tanzu/velero
 
-RUN mkdir -p /output/usr/bin && \
-    export GOARM=$( echo "${GOARM}" | cut -c2-) && \
-    go build -o /output/${BIN} \
-    -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN} && \
-    go build -o /output/velero-helper \
-    -ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper && \
-    go clean -modcache -cache
+RUN mkdir -p /output/usr/bin
+
+FROM velero-builder-base AS velero-builder-helper
+RUN GOARM=$( echo "${GOARM}" | cut -c2-) go build -o /output/velero-helper \
+-ldflags "${LDFLAGS}" ${PKG}/cmd/velero-helper
+
+FROM velero-builder-base AS velero-builder
+RUN GOARM=$( echo "${GOARM}" | cut -c2-) go build -o /output/${BIN} \
+    -ldflags "${LDFLAGS}" ${PKG}/cmd/${BIN}
 
 # Restic binary build section
 FROM --platform=$BUILDPLATFORM golang:1.22-bookworm AS restic-builder
@@ -63,12 +68,33 @@ ENV CGO_ENABLED=0 \
     GOARCH=${TARGETARCH} \
     GOARM=${TARGETVARIANT}
 
-COPY . /go/src/github.com/vmware-tanzu/velero
-
-RUN mkdir -p /output/usr/bin && \
-    export GOARM=$(echo "${GOARM}" | cut -c2-) && \
-    /go/src/github.com/vmware-tanzu/velero/hack/build-restic.sh && \
-    go clean -modcache -cache
+# /output dir needed by last stage to copy even when BIN is not velero
+RUN mkdir -p /output/usr/bin
+
+# cache go mod download before applying patches
+RUN --mount=type=cache,target=/go/pkg/mod if [ "${BIN}" = "velero" ]; then \
+        mkdir -p /build/restic && \
+        cd /build/restic && \
+        git clone --single-branch -b v${RESTIC_VERSION} https://github.com/restic/restic.git . && \
+        go mod download; \
+    fi
+
+# invalidate cache if patch changes
+COPY hack/fix_restic_cve.txt /go/src/github.com/vmware-tanzu/velero/hack/
+
+# cache go mod download after applying patches
+RUN --mount=type=cache,target=/go/pkg/mod if [ "${BIN}" = "velero" ]; then \
+        cd /build/restic && \
+        git apply /go/src/github.com/vmware-tanzu/velero/hack/fix_restic_cve.txt && \
+        go mod download; \
+    fi
+
+# arch specific build layer
+RUN --mount=type=cache,target=/go/pkg/mod if [ "${BIN}" = "velero" ]; then \ 
+        cd /build/restic && \
+        GOARM=$(echo "${GOARM}" | cut -c2-) go run build.go --goos "${GOOS}" --goarch "${GOARCH}" --goarm "${GOARM}" -o /output/usr/bin/restic && \
+        chmod +x /output/usr/bin/restic; \
+    fi
 
 # Velero image packing section
 FROM paketobuildpacks/run-jammy-tiny:latest
@@ -77,6 +103,8 @@ LABEL maintainer="Xun Jiang <jxun@vmware.com>"
 
 COPY --from=velero-builder /output /
 
+COPY --from=velero-builder-helper /output /
+
 COPY --from=restic-builder /output /
 
 USER cnb:cnb
diff --git a/hack/build-restic.sh b/hack/build-restic.sh
deleted file mode 100755
index d6a233f4a5b..00000000000
--- a/hack/build-restic.sh
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/bash
-
-# Copyright 2020 the Velero contributors.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# Use /output/usr/bin/ as the default output directory as this
-# is the path expected by the Velero Dockerfile.
-output_dir=${OUTPUT_DIR:-/output/usr/bin}
-restic_bin=${output_dir}/restic
-build_path=$(dirname "$PWD")
-
-if [[ -z "${BIN}" ]]; then
-    echo "BIN must be set"
-    exit 1
-fi
-
-if [[ "${BIN}" != "velero" ]]; then
-    echo "${BIN} does not need the restic binary"
-    exit 0
-fi
-
-if [[ -z "${GOOS}" ]]; then
-    echo "GOOS must be set"
-    exit 1
-fi
-if [[ -z "${GOARCH}" ]]; then
-    echo "GOARCH must be set"
-    exit 1
-fi
-if [[ -z "${RESTIC_VERSION}" ]]; then
-    echo "RESTIC_VERSION must be set"
-    exit 1
-fi
-
-mkdir ${build_path}/restic
-git clone -b v${RESTIC_VERSION} https://github.com/restic/restic.git ${build_path}/restic
-pushd ${build_path}/restic
-git apply /go/src/github.com/vmware-tanzu/velero/hack/fix_restic_cve.txt
-go run build.go --goos "${GOOS}" --goarch "${GOARCH}" --goarm "${GOARM}" -o ${restic_bin}
-chmod +x ${restic_bin}
-popd