Merge branch 'improvement/ZENKO-4773-end2end-pra-archive-tests' into …

…q/2.6
scality · Aug 13, 2024 · f4d1a3c · f4d1a3c
2 parents d8f4b29 + ebffb82
commit f4d1a3c
Show file tree

Hide file tree

Showing 27 changed files with 1,214 additions and 269 deletions.
diff --git a/.github/actions/deploy/action.yaml b/.github/actions/deploy/action.yaml
@@ -7,6 +7,10 @@ inputs:
     description: "The tag of the Zenko Operator image to use"
     required: false
     default: ""
+  extra_components:
+    description: "Extra components to add to zenkoversion"
+    required: false
+    default: ""
 runs:
   using: composite
   steps:
@@ -74,6 +78,8 @@ runs:
       shell: bash
       run: bash deploy-zenko.sh end2end default
       working-directory: ./.github/scripts/end2end
+      env:
+        EXTRA_COMPONENTS: ${{ inputs.extra_components }}
     - name: Add Keycloak user and assign StorageManager role
       shell: bash
       run: bash keycloak-helper.sh add-user default

diff --git a/.github/scripts/end2end/configs/keycloak_user.json b/.github/scripts/end2end/configs/keycloak_user.json
@@ -5,7 +5,7 @@
   "emailVerified": true,
   "firstName": "${OIDC_FIRST_NAME}",
   "lastName": "${OIDC_LAST_NAME}",
-  "email": "[email protected]",
+  "email": "${OIDC_EMAIL}",
   "attributes": {
     "instanceIds": [
         "${INSTANCE_ID}"

diff --git a/.github/scripts/end2end/configs/zenko.yaml b/.github/scripts/end2end/configs/zenko.yaml
@@ -24,11 +24,11 @@ spec:
     logging:
       logLevel: debug
   backbeat:
-    triggerExpirationsOneDayEarlierForTesting: true
+    triggerExpirationsOneDayEarlierForTesting: ${EXPIRE_ONE_DAY_EARLIER}
     lifecycleConductor:
       cronRule: "${BACKBEAT_LCC_CRON_RULE}"
     lifecycleBucketProcessor:
-      triggerTransitionsOneDayEarlierForTesting: true
+      triggerTransitionsOneDayEarlierForTesting: ${TRANSITION_ONE_DAY_EARLIER}
     logging:
       logLevel: trace
   mongodb:
@@ -101,6 +101,8 @@ spec:
       e2e-cold:
         debug: "true"
         command-timeout: "60s"
+        pending-job-poll-after-age: "10s"
+        pending-job-poll-check-interval: "10s"
   ingress:
     workloadPlaneClass: 'nginx'
     controlPlaneClass: 'nginx'

diff --git a/.github/scripts/end2end/configs/zenkoversion.yaml b/.github/scripts/end2end/configs/zenkoversion.yaml
@@ -5,8 +5,6 @@ kind: ZenkoVersion
 metadata:
   name: '${ZENKO_VERSION_NAME}'
 spec:
-  featureFlags:
-    backbeatGCVaultAuthorized: true
   dashboards:
     backbeat:
       image: '${BACKBEAT_DASHBOARD}'
@@ -129,3 +127,19 @@ spec:
         image: '${REDIS_EXPORTER_IMAGE}'
         tag: '${REDIS_EXPORTER_TAG}'
       kubedb: '${REDIS_KUBEDB_TAG}'
+    ${EXTRA_COMPONENTS}
+  defaults:
+    backbeatConcurrency:
+      lifecycleBucketProcessor: 30
+      lifecycleObjectProcessor: 20
+    replicaMultipliers:
+      cloudserver: 16
+      internalCloudserver: 4
+      vault: 4
+    kafkaResources:
+      broker:
+        limitCPU: 3
+      cruiseControl:
+        limitMemory: 3Gi
+  featureFlags:
+    backbeatGCVaultAuthorized: true
diff --git a/.github/scripts/end2end/configure-e2e.sh b/.github/scripts/end2end/configure-e2e.sh
@@ -51,7 +51,7 @@ KAFKA_REGISTRY_NAME=$(yq eval ".kafka.sourceRegistry" ../../../solution/deps.yam
 KAFKA_IMAGE_NAME=$(yq eval ".kafka.image" ../../../solution/deps.yaml)
 KAFKA_IMAGE_TAG=$(yq eval ".kafka.tag" ../../../solution/deps.yaml)
 KAFKA_IMAGE=$KAFKA_REGISTRY_NAME/$KAFKA_IMAGE_NAME:$KAFKA_IMAGE_TAG
-KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
+KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=${ZENKO_NAME} \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
 KAFKA_HOST_PORT=${KAFKA_HOST_PORT:1:-1}
 

diff --git a/.github/scripts/end2end/deploy-dr.sh b/.github/scripts/end2end/deploy-dr.sh
diff --git a/.github/scripts/end2end/deploy-zenko.sh b/.github/scripts/end2end/deploy-zenko.sh
@@ -38,7 +38,7 @@ fi
 
 # TODO: use kustomize
 ZENKO_MONGODB_SHARDED=${ZENKO_MONGODB_SHARDED:-'false'}
-if [ ${ZENKO_MONGODB_SHARDED} == 'true' ]; then
+if [ "${ZENKO_MONGODB_SHARDED}" = 'true' ]; then
     export ZENKO_ANNOTATIONS="annotations:
     zenko.io/x-backbeat-oneshard-replicaset: data-db-mongodb-sharded-shard-0
     zenko.io/x-backbeat-oneshard-replicaset-hosts: data-db-mongodb-sharded-shard0-data-0.data-db-mongodb-sharded-headless.default.svc.cluster.local:27017"
@@ -50,6 +50,11 @@ else
 fi
 export ZENKO_MONGODB_DATABASE="${ZENKO_MONGODB_DATABASE:-'datadb'}"
 
+if [ "${TIME_PROGRESSION_FACTOR}" -gt 1 ]; then
+    export ZENKO_ANNOTATIONS="${ZENKO_ANNOTATIONS:-annotations:}
+    zenko.io/time-progression-factor: \"${TIME_PROGRESSION_FACTOR}\""
+fi
+
 function dependencies_image_env()
 {
     yq eval '.[] | .envsubst + "=" + (.sourceRegistry // "docker.io") + "/" + .image' ${DEPS_PATH} |

diff --git a/.github/scripts/end2end/keycloak-helper.sh b/.github/scripts/end2end/keycloak-helper.sh
@@ -6,6 +6,7 @@ DIR=$(dirname "$0")
 
 COMMAND=${1:-''}
 NAMESPACE=${2:-default}
+ZENKO_NAME=${3:-end2end}
 
 KEYCLOAK_EXEC="kubectl -n ${NAMESPACE} exec -i keycloak-0 --"
 
@@ -28,7 +29,9 @@ case $COMMAND in
     "add-user")
         refresh_creds
 
-        export INSTANCE_ID=`kubectl -n ${NAMESPACE} get zenko -o jsonpath='{.items[0].status.instanceID}'`
+        export INSTANCE_ID=`kubectl -n ${NAMESPACE} get zenko ${ZENKO_NAME} -o jsonpath='{.status.instanceID}'`
+
+        export OIDC_EMAIL=${OIDC_EMAIL:-"[email protected]"}
 
         envsubst < $DIR/configs/keycloak_user.json | \
             ${KEYCLOAK_EXEC} /opt/jboss/keycloak/bin/kcadm.sh create users -r ${OIDC_REALM} -f -

diff --git a/.github/scripts/end2end/patch-coredns.sh b/.github/scripts/end2end/patch-coredns.sh
@@ -25,6 +25,13 @@ corefile="
     rewrite name exact sts.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact iam.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     rewrite name exact shell-ui.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact keycloak.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact ui.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact management.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact s3.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact sts.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact iam.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
+    rewrite name exact shell-ui.dr.zenko.local ingress-nginx-controller.ingress-nginx.svc.cluster.local
     kubernetes cluster.local in-addr.arpa ip6.arpa {
         pods insecure
         fallthrough in-addr.arpa ip6.arpa

diff --git a/.github/scripts/end2end/prepare-pra.sh b/.github/scripts/end2end/prepare-pra.sh
@@ -6,11 +6,14 @@ export MONGODB_PRA_DATABASE="${MONGODB_PRA_DATABASE:-'pradb'}"
 export ZENKO_MONGODB_DATABASE="${MONGODB_PRA_DATABASE}"
 export ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"
 
-export ZENKO_IAM_INGRESS="iam.zenko-pra.local"
-export ZENKO_STS_INGRESS="sts.zenko-pra.local"
-export ZENKO_MANAGEMENT_INGRESS="management.zenko-pra.local"
-export ZENKO_S3_INGRESS="s3.zenko-pra.local"
-export ZENKO_UI_INGRESS="ui.zenko-pra.local"
+echo 'ZENKO_MONGODB_DATABASE="pradb"' >> $GITHUB_ENV
+echo 'ZENKO_MONGODB_SECRET_NAME="mongodb-db-creds-pra"' >> $GITHUB_ENV
+
+echo 'ZENKO_IAM_INGRESS="iam.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_STS_INGRESS="sts.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_MANAGEMENT_INGRESS="management.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_S3_INGRESS="s3.dr.zenko.local"' >> $GITHUB_ENV
+echo 'ZENKO_UI_INGRESS="ui.dr.zenko.local"' >> $GITHUB_ENV
 
 MONGODB_ROOT_USERNAME="${MONGODB_ROOT_USERNAME:-'root'}"
 MONGODB_ROOT_PASSWORD="${MONGODB_ROOT_PASSWORD:-'rootpass'}"

diff --git a/.github/scripts/end2end/run-e2e-ctst.sh b/.github/scripts/end2end/run-e2e-ctst.sh
@@ -5,6 +5,9 @@ ZENKO_NAME=${1:-end2end}
 COMMAND=${2:-"premerge"}
 PARALLEL_RUNS=${PARALLEL_RUNS:-$(( ( $(nproc) + 1 ) / 2 ))}
 RETRIES=${4:-3}
+
+shift 4
+
 JUNIT_REPORT_PATH=${JUNIT_REPORT_PATH:-"ctst-junit.xml"}
 
 # Zenko Version
@@ -14,6 +17,8 @@ VERSION=$(cat ../../../VERSION | grep -Po 'VERSION="\K[^"]*')
 ZENKO_ACCOUNT_NAME="zenko-ctst"
 ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
 ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
+ADMIN_PRA_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
+ADMIN_PRA_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
 STORAGE_MANAGER_USER_NAME="ctst_storage_manager"
 STORAGE_ACCOUNT_OWNER_USER_NAME="ctst_storage_account_owner"
 DATA_CONSUMER_USER_NAME="ctst_data_consumer"
@@ -35,6 +40,9 @@ SORBET_FWD_2_ACCESSKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd
 SORBET_FWD_2_SECRETKEY=$(kubectl get secret -l app.kubernetes.io/name=sorbet-fwd-creds,app.kubernetes.io/instance=end2end -o jsonpath='{.items[0].data.secretKey}' | base64 -d)
 SERVICE_USERS_CREDENTIALS=$(echo '{"backbeat-lifecycle-bp-1":'${BACKBEAT_LCBP_1_CREDS}',"backbeat-lifecycle-conductor-1":'${BACKBEAT_LCC_1_CREDS}',"backbeat-lifecycle-op-1":'${BACKBEAT_LCOP_1_CREDS}',"backbeat-qp-1":'${BACKBEAT_QP_1_CREDS}',"sorbet-fwd-2":{"accessKey":"'${SORBET_FWD_2_ACCESSKEY}'","secretKey":"'${SORBET_FWD_2_SECRETKEY}'"}}' | jq -R)
 
+DR_ADMIN_ACCESS_KEY_ID=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
+DR_ADMIN_SECRET_ACCESS_KEY=$(kubectl get secret end2end-pra-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
+
 # Extracting kafka host from bacbeat's config
 KAFKA_HOST_PORT=$(kubectl get secret -l app.kubernetes.io/name=backbeat-config,app.kubernetes.io/instance=end2end \
     -o jsonpath='{.items[0].data.config\.json}' | base64 -di | jq .kafka.hosts)
@@ -48,6 +56,7 @@ KAFKA_CLEANER_INTERVAL=$(kubectl get zenko ${ZENKO_NAME} -o jsonpath='{.spec.kaf
 WORLD_PARAMETERS="$(jq -c <<EOF
 {
   "subdomain":"${SUBDOMAIN}",
+  "DRSubdomain":"${DR_SUBDOMAIN}",
   "ssl":false,
   "port":"${ZENKO_PORT}",
   "AccountName":"${ZENKO_ACCOUNT_NAME}",
@@ -70,7 +79,9 @@ WORLD_PARAMETERS="$(jq -c <<EOF
   "DataConsumerUsername":"${DATA_CONSUMER_USER_NAME}",
   "ServiceUsersCredentials":${SERVICE_USERS_CREDENTIALS},
   "InstanceID":"${INSTANCE_ID}",
-  "KafkaCleanerInterval":"${KAFKA_CLEANER_INTERVAL}"
+  "KafkaCleanerInterval":"${KAFKA_CLEANER_INTERVAL}",
+  "DRAdminAccessKey":"${DR_ADMIN_ACCESS_KEY_ID}",
+  "DRAdminSecretKey":"${DR_ADMIN_SECRET_ACCESS_KEY}"
 }
 EOF
 )"
@@ -85,8 +96,8 @@ kubectl set env cronjob end2end-ops-count-items PROMETHEUS_POLLING_PERIOD=1
 
 E2E_IMAGE=$E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG
 POD_NAME="${ZENKO_NAME}-ctst-tests"
-
 CTST_VERSION=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ../../../tests/ctst/package.json)
+
 # Configure keycloak
 docker run \
     --rm \
@@ -146,4 +157,4 @@ kubectl run $POD_NAME \
       }
     ]
   }
-}' -- ./run "$COMMAND" $WORLD_PARAMETERS --parallel $PARALLEL_RUNS --retry $RETRIES --retry-tag-filter @Flaky --format junit:$JUNIT_REPORT_PATH
+}' -- ./run "$COMMAND" $WORLD_PARAMETERS --parallel $PARALLEL_RUNS --retry $RETRIES --retry-tag-filter @Flaky --format junit:$JUNIT_REPORT_PATH "$@"
diff --git a/.github/workflows/end2end.yaml b/.github/workflows/end2end.yaml
@@ -81,6 +81,7 @@ env:
   NOTIF_ALT_DEST_NAME: "destination2"
   NOTIF_ALT_DEST_TOPIC: "destination-topic-2"
   SUBDOMAIN: "zenko.local"
+  DR_SUBDOMAIN: "dr.zenko.local"
   SKOPEO_PATH: "/tmp"
   HELM_VERSION: "v3.5.3"
   YQ_VERSION: "v4.27.5"
@@ -91,7 +92,11 @@ env:
   KUBECTL_VERSION: "1.30.0"
   TILT_VERSION: "0.23.4"
   KIND_VERSION: "v0.12.0"
+  DRCTL_TAG: "v1.0.2"
   ZENKO_ENABLE_SOSAPI: false
+  TIME_PROGRESSION_FACTOR: 1
+  EXPIRE_ONE_DAY_EARLIER: true
+  TRANSITION_ONE_DAY_EARLIER: true
   JUNIT_REPORT_PATH: /reports/ctst-junit.xml
   # DEBUG WAIT
   TMATE_SERVER_HOST: ${{ secrets.TMATE_SERVER_HOST }}
@@ -381,7 +386,9 @@ jobs:
         with:
           push: true
           context: ./tests/ctst
-          build-args: CTST_TAG=${{ env.CTST_TAG }}
+          build-args: |
+            CTST_TAG=${{ env.CTST_TAG }}
+            DRCTL_TAG=${{ env.DRCTL_TAG}}
           tags: "${{ env.E2E_CTST_IMAGE_NAME }}:${{ env.E2E_IMAGE_TAG }}"
           cache-from: type=gha,scope=end2end-ctst
           cache-to: type=gha,mode=max,scope=end2end-ctst
@@ -452,11 +459,8 @@ jobs:
         if: always()
 
   end2end-pra:
-    needs: [build-kafka, build-test-image, check-dashboard-versions]
-    runs-on:
-      - ubuntu
-      - focal
-      - xlarge
+    needs: [build-kafka, check-dashboard-versions, lint-and-build-ctst]
+    runs-on: ubuntu-22.04-16core
     env:
       GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
     steps:
@@ -477,46 +481,41 @@ jobs:
         env:
           ZENKO_MONGODB_SHARDED: "true"
         with:
-          zkop_tag: 1.6.0-preview.1
+          zkop_tag: 1.6.0-preview.2
+          extra_components: |
+            drctl:
+                  image: ghcr.io/scality/zenko-drctl
+                  tag: ${{ env.DRCTL_TAG }}
       - name: Prepare PRA environment
         run: bash prepare-pra.sh
         working-directory: ./.github/scripts/end2end
       - name: Deploy second Zenko for PRA
-        run: bash deploy-zenko.sh end2end-pra default
+        run: bash deploy-zenko.sh end2end-pra default './configs/zenko.yaml'
+        env:
+          ZENKO_MONGODB_SHARDED: "true"
+          ZENKO_MONGODB_DATABASE: "pradb"
+          EXTRA_COMPONENTS: |
+            drctl:
+                  image: 'ghcr.io/scality/zenko-drctl'
+                  tag: ${{ env.DRCTL_TAG }}
         working-directory: ./.github/scripts/end2end
-      - name: Deploy second Sorbet for PRA
-        run: bash deploy-sorbet-resources.sh end2end-pra
-        working-directory: ./.github/scripts/end2end/operator/tests/smoke
+      - name: Add Keycloak pra user and assign StorageManager role
+        shell: bash
+        run: bash keycloak-helper.sh add-user default end2end-pra
         env:
-          SORBETD_NAME: mock-sorbet-pra
-      - name: Extract environment
-        run: |-
-          sudo curl --fail -L https://github.com/mikefarah/yq/releases/download/${{ env.YQ_VERSION}}/${{ env.YQ_BINARY }} -o /usr/bin/yq
-          sudo chmod +x /usr/bin/yq
-
-          get_image_from_deps() {
-              local dep_name=$1
-              yq eval ".$dep_name | (.sourceRegistry // \"docker.io\") + \"/\" + .image" deps.yaml
-          }
-
-          cd solution/
-          cat <<EOF >> $GITHUB_ENV
-          KAFKA_IMAGE=$(get_image_from_deps kafka)
-          KAFKA_TAG=$(yq eval '.kafka.tag' deps.yaml)
-          KAFKA_CONNECT_IMAGE=$(get_image_from_deps kafka-connect)
-          KAFKA_CONNECT_TAG=$(yq eval '.kafka-connect.tag' deps.yaml)
-          JMX_JAVAAGENT_IMAGE=$(get_image_from_deps jmx-javaagent)
-          JMX_JAVAAGENT_TAG=$(yq eval '.jmx-javaagent.tag' deps.yaml)
-          MONGODB_CONNECTOR_TAG=$(yq eval '.mongodb-connector.tag' deps.yaml)
-          KAFKA_CLEANER_IMAGE=$(get_image_from_deps kafka-cleaner)
-          KAFKA_CLEANER_TAG=$(yq eval '.kafka-cleaner.tag' deps.yaml)
-          KAFKA_CRUISECONTROL_IMAGE=$(get_image_from_deps kafka-cruise-control)
-          KAFKA_CRUISECONTROL_TAG=$(yq eval '.kafka-cruise-control.tag' deps.yaml)
-          KAFKA_LAGEXPORTER_IMAGE=$(get_image_from_deps kafka-lag-exporter)
-          KAFKA_LAGEXPORTER_TAG=$(yq eval '.kafka-lag-exporter.tag' deps.yaml)
-          EOF
-      - name: Deploy zenko dr components
-        run: bash deploy-dr.sh
+          OIDC_USERNAME: 'zenko-end2end-pra'
+          OIDC_EMAIL: '[email protected]'
+        working-directory: ./.github/scripts/end2end
+      - name: Configure E2E PRA test environment
+        run: bash configure-e2e.sh end2end-pra ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG} default
+        working-directory: ./.github/scripts/end2end
+        env:
+          OIDC_USERNAME: 'zenko-end2end-pra'
+      - name: Configure E2E CTST test environment
+        run: bash configure-e2e-ctst.sh
+        working-directory: ./.github/scripts/end2end
+      - name: Run CTST end to end tests
+        run: bash run-e2e-ctst.sh "" "" "" "" --tags @PRA
         working-directory: ./.github/scripts/end2end
       - name: Archive artifact logs and data
         uses: ./.github/actions/archive-artifacts
@@ -685,11 +684,14 @@ jobs:
           GIT_ACCESS_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }}
           ZENKO_MONGODB_SHARDED: "true"
           ZENKO_ENABLE_SOSAPI: true
+          TIME_PROGRESSION_FACTOR: 86400
+          TRANSITION_ONE_DAY_EARLIER: false
+          EXPIRE_ONE_DAY_EARLIER: false
       - name: Configure E2E test environment
         run: bash configure-e2e-ctst.sh
         working-directory: ./.github/scripts/end2end
       - name: Run CTST end to end tests
-        run: bash run-e2e-ctst.sh
+        run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
         working-directory: ./.github/scripts/end2end
       - name: Upload results
         if: "!cancelled() && env.TRUNK_TOKEN"