From 9f6de86b072c02493c32471d4cc2e027e3d6aa81 Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 11 Jul 2024 10:14:22 +0530 Subject: [PATCH 1/5] using common-docker FIPS machine tag using temurin jdk --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index b1429ca8f..c5a269036 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -73,7 +73,7 @@ global_job_config: - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY - - export LATEST_TAG=$BRANCH_TAG-latest + - export LATEST_TAG=dev-7.0.x-12eb538f - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - export DOCKER_REPOS="confluentinc/cp-server-connect confluentinc/cp-server-connect-base confluentinc/cp-kafka-connect confluentinc/cp-kafka-connect-base confluentinc/cp-enterprise-kafka confluentinc/cp-kafka confluentinc/cp-server confluentinc/cp-zookeeper" From 9e176dde6873fc1218931f1f1fb6aacad2e3c3ef Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 11 Jul 2024 10:44:06 +0530 Subject: [PATCH 2/5] fixing upstream tag to have arch in the end also --- .semaphore/semaphore.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index c5a269036..357018de8 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -96,7 +96,7 @@ blocks: - export OS_PACKAGES_URL=$(echo "$PACKAGES_URL" | sed "s/PACKAGE_TYPE/rpm/g") - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY - -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG$AMD_ARCH -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - >- @@ -129,7 +129,7 @@ blocks: - export PACKAGING_BUILD_ARGS="$PACKAGING_BUILD_ARGS -DCONFLUENT_PACKAGES_REPO=$OS_PACKAGES_URL" - ci-tools ci-update-version - mvn -Dmaven.wagon.http.retryHandler.count=3 --batch-mode -P jenkins,docker clean package dependency:analyze validate -U -Ddocker.registry=$DOCKER_DEV_REGISTRY -Ddocker.upstream-registry=$DOCKER_UPSTREAM_REGISTRY - -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 + -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$ARM_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG$ARM_ARCH -Darch.type=$ARM_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - for image in $ARM_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done From 7dfa0c314bf95e3f5262b6347466004093d0468b Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 11 Jul 2024 10:54:42 +0530 Subject: [PATCH 3/5] using docker dev registry to fetch upstream image --- .semaphore/semaphore.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 357018de8..d54501ab7 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -72,7 +72,7 @@ global_job_config: fi - export DOCKER_DEV_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/dev/" - export DOCKER_PROD_REGISTRY="519856050701.dkr.ecr.us-west-2.amazonaws.com/docker/prod/" - - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_PROD_REGISTRY + - export DOCKER_UPSTREAM_REGISTRY=$DOCKER_DEV_REGISTRY - export LATEST_TAG=dev-7.0.x-12eb538f - export DOCKER_UPSTREAM_TAG="$LATEST_TAG" - export DOCKER_REPOS="confluentinc/cp-server-connect confluentinc/cp-server-connect-base confluentinc/cp-kafka-connect confluentinc/cp-kafka-connect-base confluentinc/cp-enterprise-kafka confluentinc/cp-kafka From eac164bebdd3a226321a6b6abd035764c081d87f Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 11 Jul 2024 11:14:49 +0530 Subject: [PATCH 4/5] using rhel9 machines to build the docker images now --- .semaphore/semaphore.yml | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index d54501ab7..98a6c1d2d 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -24,7 +24,7 @@ global_job_config: prologue: commands: - checkout - - sem-version java 8 + - sem-version java 8 || sem-version java 1.8 - . vault-setup - . cache-maven restore - pip install tox==3.28.0 @@ -80,12 +80,18 @@ global_job_config: - export DOCKER_DEV_TAG="dev-$BRANCH_TAG-$BUILD_NUMBER" - export AMD_ARCH=.amd64 - export ARM_ARCH=.arm64 + - sudo chown $(whoami) /var/run/docker.sock + - sudo chown -R $(whoami) /var/run/docker + - sudo chmod g+rw,u+rw /var/run/docker.sock blocks: - name: Build, Test, & Scan AMD dependencies: [] run: when: "pull_request =~ '.*'" task: + agent: + machine: + type: s1-prod-rhel-9-amd64-1 jobs: - name: Build, Test, & Scan ubi8 commands: @@ -99,11 +105,11 @@ blocks: -DBUILD_NUMBER=$BUILD_NUMBER -DGIT_COMMIT=$GIT_COMMIT -Ddocker.tag=$DOCKER_DEV_TAG$OS_TAG$AMD_ARCH -Ddocker.upstream-tag=$DOCKER_UPSTREAM_TAG$OS_TAG$AMD_ARCH -Darch.type=$AMD_ARCH -Ddocker.os_type=ubi8 $PACKAGING_BUILD_ARGS - . cache-maven store - - >- - for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; - do - cve-scan $dev_image - done +# - >- +# for dev_image in $AMD_DOCKER_DEV_FULL_IMAGES; +# do +# cve-scan $dev_image +# done - for image in $AMD_DOCKER_DEV_FULL_IMAGES; do echo "Pushing $image" && docker push $image; done epilogue: always: @@ -118,7 +124,7 @@ blocks: task: agent: machine: - type: s1-prod-ubuntu20-04-arm64-1 + type: s1-prod-rhel-9-arm64-1 jobs: - name: Build & Test ubi8 commands: From f60eb2c0cca4d0640919806dfbc73858b108098d Mon Sep 17 00:00:00 2001 From: Prince Raheja Date: Thu, 11 Jul 2024 11:26:16 +0530 Subject: [PATCH 5/5] assuming role to pull images from ECR --- .semaphore/semaphore.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 98a6c1d2d..fce290226 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -27,6 +27,7 @@ global_job_config: - sem-version java 8 || sem-version java 1.8 - . vault-setup - . cache-maven restore + - . assume-iam-role arn:aws:iam::519856050701:role/semaphore-oidc - pip install tox==3.28.0 - export GIT_COMMIT=$(git rev-parse --verify HEAD --short) - export BUILD_NUMBER=$(echo $SEMAPHORE_WORKFLOW_ID | cut -f1 -d"-")