Merge pull request #4327 from wireapp/release_2024-11-04_13_21

Release 2024-11-04 - (expected chart version 5.7.0)

CHANGELOG.md

@@ -1,3 +1,21 @@
+# [2024-11-04] (Chart Release 5.7.0)
+
+## Bug fixes and other updates
+
+
+* galley: Use bulk query when getting all feature configs for a team user (#4325)
+
+
+## Internal changes
+
+
+* Block access to assets.*/minio/ path for public access. (#4297)
+
+* galley: Delete unused endpoint for getting feature status for multiple teams (#4326)
+
+* Fix shellcheck problems in all shell scripts (#4220)
+
+
 # [2024-10-30] (Chart Release 5.6.0)
 
 ## Release notes

Makefile

@@ -250,11 +250,11 @@ add-license:
 
 .PHONY: treefmt
 treefmt:
-	treefmt -u debug
-
+	treefmt -u debug --walk=git
+ 
 .PHONY: treefmt-check
 treefmt-check:
-	treefmt --fail-on-change -u debug
+	treefmt --fail-on-change -u debug --walk=git
 
 #################################
 ## docker targets

changelog.d/mk-changelog.sh

@@ -6,7 +6,7 @@ shopt -s nullglob
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 getPRNumber() {
-    git log --reverse --format=%s -- $1 | sed -rn '1 { /\((#.*)\)$/ s|^.*\((#.*)\)$|\1|p; }' | grep "" ||
+    git log --reverse --format=%s -- "$1" | sed -rn '1 { /\((#.*)\)$/ s|^.*\((#.*)\)$|\1|p; }' | grep "" ||
       echo "#PR_NOT_FOUND"
 }
 
@@ -18,10 +18,12 @@ for d in "$DIR"/*; do
     if [[ ${#entries[@]} -eq 0 ]]; then continue; fi
 
     echo -n "## "
+    # shellcheck disable=SC1003
     sed '$ a\' "$d/.title"
     echo ""
     for f in "${entries[@]}"; do
-        pr=$(getPRNumber $f)
+        pr=$(getPRNumber "$f")
+        # shellcheck disable=SC1003
         sed -r '
           # create a bullet point on the first line
           1 { s/^/\* /; }

charts/nginx-ingress-services/templates/ingress.yaml

@@ -51,9 +51,6 @@ spec:
 {{- if .Values.webapp.enabled }}
       - {{ .Values.config.dns.webapp }}
 {{- end }}
-{{- if .Values.fakeS3.enabled }}
-      - {{ .Values.config.dns.fakeS3 }}
-{{- end }}
 {{- if .Values.teamSettings.enabled }}
       - {{ .Values.config.dns.teamSettings }}
 {{- end }}
@@ -117,25 +114,6 @@ spec:
               servicePort: {{ .Values.service.webapp.externalPort }}
               {{- end }}
 {{- end }}
-{{- if .Values.fakeS3.enabled }}
-    - host: {{ .Values.config.dns.fakeS3 }}
-      http:
-        paths:
-          - path: /
-            {{- if $ingressSupportsPathType }}
-            pathType: Prefix
-            {{- end }}
-            backend:
-              {{- if $apiIsStable }}
-              service:
-                name: {{ .Values.service.s3.serviceName }}
-                port:
-                  number: {{ .Values.service.s3.externalPort }}
-              {{- else }}
-              serviceName: {{ .Values.service.s3.serviceName }}
-              servicePort: {{ .Values.service.s3.externalPort }}
-              {{- end }}
-{{- end }}
 {{- if .Values.teamSettings.enabled }}
     - host: {{ .Values.config.dns.teamSettings }}
       http:

charts/nginx-ingress-services/templates/ingress_minio.yaml

@@ -0,0 +1,45 @@
+{{- $apiIsStable := eq (include "ingress.isStable" .) "true" -}}
+{{- $ingressFieldNotAnnotation := eq (include "ingress.FieldNotAnnotation" .) "true" -}}
+{{- $ingressSupportsPathType := eq (include "ingress.supportsPathType" .) "true" -}}
+{{- if .Values.fakeS3.enabled }}
+# We use a separate ingress for minio because we want to restrict access to /minio/ path
+# for security reasons
+apiVersion: {{ include "ingress.apiVersion" . }}
+kind: Ingress
+metadata:
+  name: minio-ingress
+  annotations:
+    {{- if not $ingressFieldNotAnnotation }}
+    kubernetes.io/ingress.class: "{{ .Values.config.ingressClass }}"
+    {{- end }}
+    nginx.ingress.kubernetes.io/server-snippet: |
+      location /minio/ {
+        return 403;
+      }
+spec:
+  {{- if $ingressFieldNotAnnotation }}
+  ingressClassName: "{{ .Values.config.ingressClass }}"
+  {{- end }}
+  tls:
+  - hosts:
+      - {{ .Values.config.dns.fakeS3 }}
+    secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
+  rules:
+    - host: {{ .Values.config.dns.fakeS3 }}
+      http:
+        paths:
+          - path: /
+            {{- if $ingressSupportsPathType }}
+            pathType: Prefix
+            {{- end }}
+            backend:
+              {{- if $apiIsStable }}
+              service:
+                name: {{ .Values.service.s3.serviceName }}
+                port:
+                  number: {{ .Values.service.s3.externalPort }}
+              {{- else }}
+              serviceName: {{ .Values.service.s3.serviceName }}
+              servicePort: {{ .Values.service.s3.externalPort }}
+              {{- end }}
+{{- end }}

deploy/dockerephemeral/init.sh

@@ -12,7 +12,7 @@ aws configure set aws_secret_access_key dummysecret
 aws configure set region eu-west-1
 
 # Potentially delete pre-existing tables
-echo -n "waiting for dynamo: "
+echo "waiting for dynamo: "
 while (! aws --endpoint-url=http://dynamodb:8000 --cli-connect-timeout=1 dynamodb list-tables); do
     sleep 1;
 done

hack/bin/cabal-run-tests.sh

@@ -14,8 +14,8 @@ fi
 for cabal in $(find "$TOP_LEVEL" -name "$pattern" | grep -v dist-newstyle); do
   # This is required because some tests (e.g. golden tests) must be run from
   # the package root.
-  cd "$(dirname $cabal)"
-  package="$(basename ${cabal%.*})"
+  cd "$(dirname "$cabal")"
+  package="$(basename "${cabal%.*}")"
   for test_suite in $(cabal-plan list-bins "$package:test:*" | awk '{print $2}'); do
     $test_suite "${@:2}"
   done

hack/bin/copy-charts.sh

@@ -9,21 +9,21 @@ CHART=${1:?$USAGE}
 
 TOP_LEVEL="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../.." && pwd )"
 CHART_SOURCE=$TOP_LEVEL/charts
-CHART_DIST=$TOP_LEVEL/.local/charts
+CHART_DEST=$TOP_LEVEL/.local/charts
 
 # TODO sanity check folder must exist
 
 mkdir -p .local/charts
-rm -rf "$CHART_DIST/$CHART"
-cp -r "$CHART_SOURCE/$CHART" "$CHART_DIST/"
+rm -rf "${CHART_DEST:?}/$CHART"
+cp -r "$CHART_SOURCE/$CHART" "$CHART_DEST/"
 
 if [ -f "$CHART_SOURCE/$CHART/requirements.yaml" ]; then
   # very hacky bash, I'm sorry
   for subpath in $(grep "file://" "$CHART_SOURCE/$CHART/requirements.yaml" | awk '{ print $2 }' | xargs -n 1 | cut -c 8-)
   do
-    rm -rf "$CHART_DIST/$CHART/$subpath"
-    cp -r "$CHART_SOURCE/$CHART/$subpath" "$CHART_DIST/"
+    rm -rf "${CHART_DEST:?}/$CHART/$subpath"
+    cp -r "$CHART_SOURCE/$CHART/$subpath" "$CHART_DEST/"
   done
 fi
 
-echo "copied $CHART_SOURCE/$CHART (and its local dependencies) to $CHART_DIST/$CHART"
+echo "copied $CHART_SOURCE/$CHART (and its local dependencies) to $CHART_DEST/$CHART"

hack/bin/create_team_members.sh

@@ -33,7 +33,7 @@ $ grep code out.log | grep -v email-exists
 
 If you are in a hurry, you may want to change the sleep(1) at the end
 of the invite loop to less than a second.  If you want to give up on
-the first error, add an exit(1) where we check the $INVIDATION_ID.
+the first error, add an exit(1) where we check the INVITATION_ID.
 
 "
 

hack/bin/create_test_team_admins.sh

@@ -51,10 +51,11 @@ fi
 
 # Generate users
 
+#shellcheck disable=SC2034
 for i in $(seq 1 "$COUNT")
 do
-    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     CURL_OUT=$(curl -i -s --show-error \
         -XPOST "$BRIG_HOST/i/users" \

hack/bin/create_test_team_members.sh

@@ -84,7 +84,7 @@ END=$((COUNT + START - 1))
 for i in $(seq "$START" "$END")
 do
     EMAIL='w'$(printf "%03d" "$i")"@$TARGET_EMAIL_DOMAIN"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     # Generate the invitation
 
@@ -125,7 +125,7 @@ do
 
     if [ "$TEAM" != "$TEAM_UUID" ]; then
         echo "unexpected error: user got assigned to no / the wrong team?!"
-        echo ${CURL_OUT}
+        echo "${CURL_OUT}"
         exit 1
     fi
 

hack/bin/create_test_team_scim.sh

@@ -45,8 +45,8 @@ if [ "$#" -ne 0 ]; then
 fi
 
 
-ADMIN_EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-ADMIN_PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+ADMIN_EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+ADMIN_PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
 CURL_OUT=$(curl -i -s --show-error \
     -XPOST "$BRIG_HOST/i/users" \
@@ -61,23 +61,23 @@ BEARER=$(curl -X POST \
               --header 'Content-Type: application/json' \
               --header 'Accept: application/json' \
               -d '{"email":"'"$ADMIN_EMAIL"'","password":"'"$ADMIN_PASSWORD"'"}' \
-              $BRIG_HOST/login'?persist=false' | jq -r .access_token)
+              "$BRIG_HOST"/login'?persist=false' | jq -r .access_token)
 
 SCIM_TOKEN_FULL=$(curl -X POST \
                        --header "Authorization: Bearer $BEARER" \
                        --header 'Content-Type: application/json;charset=utf-8' \
                        --header 'Z-User: '"$ADMIN_UUID" \
-                       -d '{ "description": "test '"`date`"'", "password": "'"$ADMIN_PASSWORD"'" }' \
-                       $SPAR_HOST/scim/auth-tokens)
+                       -d '{ "description": "test '"$(date)"'", "password": "'"$ADMIN_PASSWORD"'" }' \
+                       "$SPAR_HOST/scim/auth-tokens")
 
-SCIM_TOKEN=$(echo $SCIM_TOKEN_FULL | jq -r .token)
-SCIM_TOKEN_ID=$(echo $SCIM_TOKEN_FULL | jq -r .info.id)
+SCIM_TOKEN=$(echo "$SCIM_TOKEN_FULL" | jq -r .token)
+SCIM_TOKEN_ID=$(echo "$SCIM_TOKEN_FULL" | jq -r .info.id)
 
 
 # Create regular user via team invitation
 
-REGULAR_USER_EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-REGULAR_USER_PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+REGULAR_USER_EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+REGULAR_USER_PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 CURL_OUT_INVITATION=$(curl -i -s --show-error \
                             -XPOST "$BRIG_HOST/teams/$TEAM_UUID/invitations" \
                             -H'Content-type: application/json' \
@@ -122,7 +122,7 @@ REGULAR_TEAM_MEMBER_UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z
 # Create user via SCIM invitation
 
 
-scimUserName=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+scimUserName=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 scimUserDisplayName="Display of $scimUserName"
 scimUserEmail="$scimUserName@example.com"
 scimUserExternalId="$scimUserEmail"
@@ -156,7 +156,7 @@ CURL_OUT_SCIM_POST=$(curl --location --request POST "$SPAR_HOST/scim/v2/Users" \
     --header "Authorization: Bearer $SCIM_TOKEN" \
     -d "$SCIM_USER")
 
-SCIM_USER_UUID=$(echo $CURL_OUT_SCIM_POST | jq -r .id)
+SCIM_USER_UUID=$(echo "$CURL_OUT_SCIM_POST" | jq -r .id)
 
 SCIM_USER_INVITATION_ID=$(curl --location -G "$BRIG_HOST/i/teams/invitations/by-email?" \
     --header 'Content-Type: application/json' \
@@ -170,17 +170,7 @@ SCIM_USER_INVITATION_CODE=$(curl --silent --show-error \
                             -XGET "$BRIG_HOST/i/teams/invitation-code?team=$TEAM_UUID&invitation_id=$SCIM_USER_INVITATION_ID" | jq -r .code
                             )
 
-scimUserPassword=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
-
-REGISTER_ACCEPT=$(cat <<EOF
-{
-   "name": "$scimUserDisplayName",
-   "email": "$scimUserEmail",
-   "password": "$scimUserPassword",
-   "team_code": "$SCIM_USER_INVITATION_CODE"
-}
-EOF
-)
+scimUserPassword=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
 # Create the user using that code
 CURL_OUT=$(curl \
@@ -192,7 +182,7 @@ SCIM_USER_REGISTER_TEAM=$(echo "$CURL_OUT" | jq -r .team)
 
 if [ "$SCIM_USER_REGISTER_TEAM" != "$TEAM_UUID" ]; then
     echo "unexpected error: user got assigned to no / the wrong team?!"
-    echo ${CURL_OUT}
+    echo "${CURL_OUT}"
     exit 1
 fi
 

hack/bin/create_test_user.sh

@@ -47,20 +47,21 @@ fi;
 
 # Generate users
 
-for i in `seq 1 $COUNT`
+# shellcheck disable=SC2034
+for i in $(seq 1 "$COUNT")
 do
-    EMAIL=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)"@example.com"
-    PASSWORD=$(cat /dev/urandom | env LC_CTYPE=C tr -dc a-zA-Z0-9 | head -c 8)
+    EMAIL=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)"@example.com"
+    PASSWORD=$(env LC_CTYPE=C tr -dc a-zA-Z0-9 < /dev/urandom | head -c 8)
 
     CURL_OUT=$(curl -i -s --show-error \
         -XPOST "$BRIG_HOST/i/users" \
         -H'Content-type: application/json' \
-        -d'{"email":"'$EMAIL'","password":"'$PASSWORD'","name":"demo"}')
+        -d'{"email":"'"$EMAIL"'","password":"'"$PASSWORD"'","name":"demo"}')
 
     UUID=$(echo "$CURL_OUT" | tail -1 | sed 's/.*\"id\":\"\([a-z0-9-]*\)\".*/\1/')
 
     if [ "$CSV" == "false" ]
-        then echo -e "Succesfully created a user with email: "$EMAIL" and password: "$PASSWORD
-        else echo -e $UUID","$EMAIL","$PASSWORD
+        then echo -e "Succesfully created a user with email: ""$EMAIL"" and password: ""$PASSWORD"
+        else echo -e "$UUID,$EMAIL,$PASSWORD"
     fi
 done

hack/bin/diff-failure.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 sed 's| =/= |\n|' | {
-    IFS= read first
-    IFS= read second
+    IFS= read -r first
+    IFS= read -r second
     exec wdiff -n -w $'\033[30;41m' -x $'\033[0m' \
          -y $'\033[30;42m' -z $'\033[0m' \
          <(echo "$first") <(echo "$second")

hack/bin/helm-template.sh

@@ -25,4 +25,4 @@ if [ -f "$certificatesfile" ]; then
 fi
 
 "$DIR/update.sh" "$CHARTS_DIR/$chart"
-helm template $"chart" "$CHARTS_DIR/$chart" ${options[*]}
+helm template "$chart" "$CHARTS_DIR/$chart" "${options[*]}"

hack/bin/integration-setup-federation.sh

@@ -10,8 +10,9 @@ HELMFILE_ENV=${HELMFILE_ENV:-default}
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 HELM_PARALLELISM=${HELM_PARALLELISM:-1}
 
+# shellcheck disable=SC1091
 . "$DIR/helm_overrides.sh"
-${DIR}/integration-cleanup.sh
+"${DIR}"/integration-cleanup.sh
 
 # FUTUREWORK explore: have helmfile do the interpolation (and skip the "make charts" step) https://wearezeta.atlassian.net/browse/SQPIT-722
 #

hack/bin/integration-spring-cleaning.sh

@@ -1,13 +1,9 @@
 #!/usr/bin/env bash
 
-DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/.."
-
 set -x
 
 IFS=$'\n'
 for NAMESPACE in $(kubectl get namespaces | grep "^test-" | awk '{print $1}'); do
-
     echo "$NAMESPACE"
     kubectl delete namespace "$NAMESPACE" &
-
 done

hack/bin/integration-teardown-federation.sh

@@ -24,6 +24,7 @@ else
     export INGRESS_CHART="nginx-ingress-controller"
 fi
 
+# shellcheck disable=SC1091
 . "$DIR/helm_overrides.sh"
 helmfile --environment "$HELMFILE_ENV" --file "${TOP_LEVEL}/hack/helmfile.yaml" destroy --skip-deps --skip-charts --concurrency 0 || echo "Failed to delete helm deployments, ignoring this failure as next steps will the destroy namespaces anyway."