Merge branches 'w/2.8/improvement/ZENKO-4799-Run-CTST-and-Zenko-in-a-…

…codespace' and 'q/2058/2.7/improvement/ZENKO-4799-Run-CTST-and-Zenko-in-a-codespace' into tmp/octopus/q/2.8

.devcontainer/Dockerfile

@@ -0,0 +1,14 @@
+FROM mcr.microsoft.com/devcontainers/base:jammy
+ENV YQ_VERSION=v4.27.5
+ENV TILT_VERSION=0.23.4
+ENV KUSTOMIZE_VERSION=v4.4.1
+
+RUN curl -L "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o /usr/local/bin/yq && \
+    chmod +x /usr/local/bin/yq
+
+RUN curl -L "https://github.com/tilt-dev/tilt/releases/download/v${TILT_VERSION}/tilt.${TILT_VERSION}.linux.x86_64.tar.gz" | tar xzvf - && \
+    mv tilt /usr/local/bin && \
+    chmod +x /usr/local/bin/tilt
+
+RUN curl -L "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/${KUSTOMIZE_VERSION}/kustomize_${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" | tar -xvz && \
+    install kustomize /usr/local/bin

.devcontainer/README.md

@@ -0,0 +1,50 @@
+# Zenko Codespaces
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/scality/Zenko)
+
+## Running CTST tests in the codespace
+
+To run the CTST tests in the codespace, simply head to `.github/script/end2end/` and run `run-e2e-ctst.sh` script.
+
+```bash
+    cd .github/script/end2end/
+    bash run-e2e-ctst.sh
+```
+
+## Accessing s3 service
+
+This devcontainer is a full Zenko development environment.
+Once you start a new Codespace, you will have a full Zenko stack running in Kubernetes after a few minutes.
+
+After deployment is done, which you can follow by opening another terminal, you will be able to access S3 service through a port-forward.
+First find a cloudserver connector using the following command:
+
+```bash
+    kubectl get pods
+```
+
+Then port-forward the connector
+
+```bash
+    kubectl port-forward pod/end2end-connector-cloudserver-XXXXXXX 8080
+```
+
+After that you will need Access Key and Secret Key which you can find with the following commands
+
+```bash
+    export ACCESS_KEY=$(kubectl get secret end2end-management-vault-admin-creds.v1 -o jsonpath='{.data.accessKey}' | base64 -d)
+    export SECRET_KEY=$(kubectl get secret end2end-management-vault-admin-creds.v1  -o jsonpath='{.data.secretKey}' | base64 -d)
+```
+
+Then configure aws cli with the following command
+
+```bash
+    aws configure set aws_access_key_id $ACCESS_KEY
+    aws configure set aws_secret_access_key $SECRET_KEY
+    aws configure set region us-east-1
+```
+
+Now you can use aws cli to interact with the S3 service
+
+```bash
+    aws s3 ls --endpoint http://localhost:8080
+```

.devcontainer/devcontainer.json

@@ -0,0 +1,82 @@
+{
+    "name": "Codespace",
+    "build": {
+        "context": ".",
+        "dockerfile": "Dockerfile"
+    },
+    "features": {
+        "ghcr.io/devcontainers/features/docker-in-docker": {},
+        "ghcr.io/devcontainers/features/github-cli:1": {},
+        "ghcr.io/devcontainers/features/node:1": {},
+        "ghcr.io/devcontainers/features/sshd:1": {},
+        "ghcr.io/devcontainers-contrib/features/kind:1": {},
+        "ghcr.io/devcontainers/features/aws-cli:1": {},
+        "ghcr.io/devcontainers/features/kubectl-helm-minikube:1": {
+            "version": "latest",
+            "minikube": "none",
+            "helm": "3.5.3"
+        }
+    },
+    "customizations": {
+        "codespaces": {
+            "repositories": {
+                "scality/cli-testing": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/zenko-operator": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/sorbet": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/backbeat": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/cloudserver": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/vault2": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/pensieve-api": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/s3utils": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/zenko-ui": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                },
+                "scality/kafka-cleaner": {
+                    "permissions": {
+                        "contents": "read"
+                    }
+                }
+            }
+        }
+    },
+    "containerEnv": {
+        "ZENKO_MONGODB_DATABASE": "zenko-database",
+        "ZENKO_MONGODB_SHARDED": "true" 
+        },
+    // Use 'postCreateCommand' to run commands after the container is created.
+    "postCreateCommand": "bash .devcontainer/setup.sh"
+}

.devcontainer/setup.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+env_variables=$(yq eval '.env | to_entries | .[] | .key + "=" + .value' .github/workflows/end2end.yaml | sed 's/\${{[^}]*}}//g') && export $env_variables
+export GIT_ACCESS_TOKEN=${GITHUB_TOKEN}
+export E2E_IMAGE_TAG=latest
+
+array_length=`yq ".runs.steps | length - 1" .github/actions/deploy/action.yaml`
+for i in $(seq 0 $array_length); do
+    step=`yq ".runs.steps[$i]" .github/actions/deploy/action.yaml`
+    working_dir=`yq ".runs.steps[$i].working-directory" .github/actions/deploy/action.yaml`
+    run_command=`yq ".runs.steps[$i].run" .github/actions/deploy/action.yaml`
+
+    # We don't want to run `run-e2e-test.sh` because it is used for linting here, user will run it manually if needed after deployment
+    # We can't run `configure-e2e.sh` here because it needs an image that is not yet built and sent to kind, will be run after
+    (
+        if [[ "$run_command" != "null" && "$run_command" != *"configure-e2e.sh"* && "$run_command" != *"run-e2e-test.sh"* ]]
+        then
+            if [ "$working_dir" != "null" ]
+            then
+                echo "Changing working dir: $working_dir"
+                cd $working_dir
+            fi
+            echo "Run command: $run_command"
+            while IFS= read -r line; do
+                eval $line
+            done <<< "$run_command";
+        fi
+    )
+done
+
+(
+    cd tests/zenko_tests
+
+    envsubst < 'e2e-config.yaml.template' > 'e2e-config.yaml'
+    if [[ "${ENABLE_RING_TESTS}" == "false" ]]; then
+    yq -i 'del(.locations[] | select(.locationType == "location-scality-ring-s3-v1"))' e2e-config.yaml
+    fi
+    docker build -t $E2E_IMAGE_NAME:$E2E_IMAGE_TAG .
+    kind load docker-image  ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG}
+    docker rmi ${E2E_IMAGE_NAME}:${E2E_IMAGE_TAG}
+)
+
+(
+    cd .github/scripts/end2end
+
+    bash configure-e2e-ctst.sh
+)
+
+docker image prune -af
+
+CTST_TAG=$(sed 's/.*"cli-testing": ".*#\(.*\)".*/\1/;t;d' ./tests/ctst/package.json)
+SORBET_TAG=$(yq eval '.sorbet.tag' solution/deps.yaml)
+docker build --build-arg CTST_TAG=$CTST_TAG --build-arg SORBET_TAG=$SORBET_TAG -t $E2E_CTST_IMAGE_NAME:$E2E_IMAGE_TAG ./tests/ctst
+kind load docker-image  ${E2E_CTST_IMAGE_NAME}:${E2E_IMAGE_TAG}

.github/scripts/end2end/configure-e2e.sh

@@ -54,7 +54,7 @@ kubectl run ${POD_NAME} \
   --restart=Never \
   --pod-running-timeout=5m \
   --namespace=${NAMESPACE} \
-  --image-pull-policy=Always \
+  --image-pull-policy=IfNotPresent \
   --overrides="{ \"spec\": { \"serviceAccount\": \"${SERVICE_ACCOUNT}\" }  }" \
   --env="TOKEN=${TOKEN}" \
   --env="UUID=${UUID}" \

.github/scripts/end2end/run-e2e-ctst.sh

@@ -125,7 +125,7 @@ kubectl run $POD_NAME \
         --restart=Never \
         --rm \
         --attach=True \
-        --image-pull-policy=Always \
+        --image-pull-policy=IfNotPresent \
         --env=TARGET_VERSION=$VERSION  \
         --env=AZURE_BLOB_URL=$AZURE_BACKEND_ENDPOINT  \
         --env=AZURE_QUEUE_URL=$AZURE_BACKEND_QUEUE_ENDPOINT \

README.md

@@ -3,6 +3,8 @@
 ![Zenko logo](res/zenko.io-logo-wide-bw.png)
 
 [![Documentation Status](https://readthedocs.org/projects/zenko/badge/?version=latest)](https://zenko.readthedocs.io/en/latest/?badge=latest)
+[![Open in GitHub Codespaces](https://github.com/codespaces/badge.svg)](https://codespaces.new/scality/Zenko)
+
 
 Zenko is  [Scality](http://www.scality.com/)’s open source multi-cloud data
 controller.