refactor(dockerfiles/cd): refactor multi stages building dockerfiles …

…for tiflow (#118) - include components: ticdc, dm - also fix build step for pd image Signed-off-by: wuhuizuo <[email protected]> --------- Signed-off-by: wuhuizuo <[email protected]>
PingCAP-QE · Nov 10, 2023 · a348d85 · a348d85
1 parent de1d1b6
commit a348d85
Show file tree

Hide file tree

Showing 6 changed files with 101 additions and 96 deletions.
diff --git a/Justfile b/Justfile
@@ -4,20 +4,23 @@ msb-tikv: (_msb "tikv" "https://github.com/tikv/tikv.git" "master")
 msb-tiflash: (_msb "tiflash" "https://github.com/pingcap/tiflash" "master")
     docker run --rm --entrypoint=/tiflash/tiflash tiflash:latest version
 
-msb-dm: (_msb "dm" "https://github.com/pingcap/tiflow.git" "master")
-    docker run --rm dm /dm-master -V
-    docker run --rm dm /dm-worker -V
-    docker run --rm dm /dm-syncer -V
-    docker run --rm dm /dmctl -V
+msb-dm: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master")
+    docker build -t localhost/dm:local-build -f dockerfiles/cd/pingcap/tiflow/Dockerfile --target final-dm ../tiflow
+    docker run --rm localhost/dm:local-build /dm-master -V
+    docker run --rm localhost/dm:local-build /dm-worker -V
+    docker run --rm localhost/dm:local-build /dm-syncer -V
+    docker run --rm localhost/dm:local-build /dmctl -V
 
-msb-ticdc: (_msb "ticdc" "https://github.com/pingcap/tiflow.git" "master")
-    docker run --rm ticdc /cdc version
+msb-ticdc: (_clone "tiflow" "https://github.com/pingcap/tiflow.git" "master")
+    docker build -t localhost/ticdc:local-build -f dockerfiles/cd/pingcap/tiflow/Dockerfile --target final-cdc ../tiflow
+    docker run --rm localhost/ticdc:local-build /cdc version
 
 msb-tidb: (_msb "tidb" "https://github.com/pingcap/tidb.git" "master")
     docker run --rm tidb -V
 
-msb-pd: (_msb "pd" "https://github.com/tikv/pd.git" "master")
-    docker run --rm pd -V
+msb-pd: (_clone "pd" "https://github.com/tikv/pd.git" "master")
+    docker build -t localhost/pd:local-build -f dockerfiles/cd/tikv/pd/Dockerfile ../pd
+    docker run --rm localhost/pd:local-build /cdc -V
 
 build_product_base_images: (_docker_build_prod_base_images "hub.pingcap.net/bases")
 
@@ -26,5 +29,9 @@ _msb component git_url git_branch:
     ([ -e ../{{component}}/.dockerignore ] && rm ../{{component}}/.dockerignore) || true # make step depended on git metadata.
     docker build -t {{component}} -f dockerfiles-multi-stages/{{component}}/Dockerfile ../{{component}}
 
+_clone component git_url git_branch:
+    [ -e ../{{component}} ] || git clone --recurse-submodules {{git_url}} --branch {{git_branch}} ../{{component}}
+    ([ -e ../{{component}}/.dockerignore ] && rm ../{{component}}/.dockerignore) || true # make step depended on git metadata.
+
 _docker_build_prod_base_images registry_prefix:
     cd dockerfiles/bases; skaffold build --profile local-docker --default-repo {{registry_prefix}}
diff --git a/dockerfiles-multi-stages/dm/Dockerfile b/dockerfiles-multi-stages/dm/Dockerfile
diff --git a/dockerfiles-multi-stages/ticdc/Dockerfile b/dockerfiles-multi-stages/ticdc/Dockerfile
diff --git a/dockerfiles/cd/pingcap/tiflow/Dockerfile b/dockerfiles/cd/pingcap/tiflow/Dockerfile
@@ -0,0 +1,80 @@
+# build requires:
+#   - docker >= v20.10
+#
+# build steps:
+#   - git clone --recurse-submodules https://github.com/pingcap/tiflow.git tiflow
+#   - docker build -t dm -f Dockerfile ./tiflow
+
+########### stage: Builder - cdc
+FROM centos:7.9.2009 as builder-cdc
+
+# install packages.
+RUN yum update --nogpgcheck -y && \
+    yum install --nogpgcheck -y epel-release centos-release-scl deltarpm && \
+    yum update --nogpgcheck -y && \
+    yum groupinstall --nogpgcheck -y "Development Tools" && \
+    yum clean all
+
+# install golang toolchain
+# renovate: datasource=docker depName=golang
+ARG GOLANG_VERSION=1.21.4
+RUN OS=linux; ARCH=$([ "$(arch)" = "x86_64" ] && echo amd64 || echo arm64); \
+    curl -fsSL https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz | tar -C /usr/local -xz
+ENV PATH /usr/local/go/bin/:$PATH
+LABEL go-version="${GOLANG_VERSION}"
+
+########### stage: Builder - dm
+FROM builder-cdc as builder-dm
+
+# install nodejs toolchain
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.3/install.sh | bash && \
+    source ~/.bashrc && \
+    nvm install 16 && \
+    npm install -g yarn
+
+########### stage: Builder - total
+FROM builder-dm as builder
+
+########### stage: Buiding cdc
+FROM builder-cdc as building-cdc
+
+COPY . /tiflow
+ARG GOPROXY
+RUN GOPROXY=${GOPROXY} make cdc -C /tiflow
+RUN /tiflow/bin/cdc version
+
+########### stage: Buiding dm
+FROM builder-dm as building-dm
+
+COPY . /tiflow
+ARG GOPROXY
+RUN GOPROXY=${GOPROXY} make dm-master-with-webui dm-worker dmctl dm-syncer -c /tiflow
+RUN /tiflow/bin/dm-master -V
+RUN /tiflow/bin/dm-worker -V
+RUN /tiflow/bin/dm-syncer -V
+RUN /tiflow/bin/dmctl -V
+
+########### stage: Final image - cdc
+FROM ghcr.io/pingcap-qe/bases/tools-base:v1.7.1 as final-cdc
+
+COPY --from=building-cdc /tiflow/bin/cdc /cdc
+EXPOSE 8300
+CMD ["/cdc"]
+
+COPY . /tiflow
+ARG GOPROXY
+RUN GOPROXY=${GOPROXY} make dm-master-with-webui dm-worker dmctl dm-syncer -c /tiflow
+RUN /tiflow/bin/dm-master -V
+RUN /tiflow/bin/dm-worker -V
+RUN /tiflow/bin/dm-syncer -V
+RUN /tiflow/bin/dmctl -V
+
+########### stage: Final image - dm
+FROM ghcr.io/pingcap-qe/bases/tools-base:v1.7.1 as final-dm
+
+COPY --from=building-dm /tiflow/bin/dm-master /dm-master
+COPY --from=building-dm /tiflow/bin/dm-worker /dm-worker
+COPY --from=building-dm /tiflow/bin/dm-syncer /dm-syncer
+COPY --from=building-dm /tiflow/bin/dmctl     /dmctl
+
+EXPOSE 8291 8261 8262
diff --git a/dockerfiles/cd/tikv/pd/Dockerfile b/dockerfiles/cd/tikv/pd/Dockerfile
@@ -28,16 +28,16 @@ LABEL go-version="${GOLANG_VERSION}"
 FROM builder as building
 COPY . /pd
 ARG GOPROXY
-RUN GOPROXY=${GOPROXY} make build tools -c /pd
+RUN GOPROXY=${GOPROXY} make build tools -C /pd
 RUN /pd/bin/pd-server -V
 RUN /pd/bin/pd-ctl -V
 RUN /pd-recover -V
 
 ########### stage: Final image
 FROM ghcr.io/pingcap-qe/bases/pd-base:v1.7.1
 
-COPY --from=building /pd/bin/pd-server /pd-server
-COPY --from=building /pd/bin/pd-ctl /pd-ctl
+COPY --from=building /pd/bin/pd-server  /pd-server
+COPY --from=building /pd/bin/pd-ctl     /pd-ctl
 COPY --from=building /pd/bin/pd-recover /pd-recover
 
 EXPOSE 2379 2380

diff --git a/packages/packages.yaml.tmpl b/packages/packages.yaml.tmpl
@@ -200,7 +200,7 @@ components:
             type: image
             artifactory:
               repo: hub.pingcap.net/pingcap/tiflow/images/cdc
-            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/tiflow/cdc.Dockerfile
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/cdc.Dockerfile
             files: # context files
               - name: cdc
                 src:
@@ -209,7 +209,7 @@ components:
             type: image
             artifactory:
               repo: hub.pingcap.net/pingcap/tiflow/images/dm
-            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/tiflow/dm.Dockerfile
+            dockerfile: https://github.com/PingCAP-QE/artifacts/raw/main/dockerfiles/dm.Dockerfile
             files:
               - name: dm-master
                 src: