Configurable s3 endpoint for download

.github/integration/scripts/charts/dependencies.sh

@@ -31,7 +31,7 @@ else
 fi
 
 # secret for the crypt4gh keypair
-C4GHPASSPHRASE="$(random-string)"
+C4GHPASSPHRASE="ZB8ko3g2P9wEgagpgqCS7vJhJoKrSUfJ"
 export C4GHPASSPHRASE
 dir=$PWD
 if [ -n "$1" ]; then

.github/integration/tests/sda/11_api_test.sh

@@ -2,7 +2,7 @@
 set -e
 
 # Test the API files endpoint
-token="$(curl http://oidc:8080/tokens | jq -r '.[0]')"
+token="$(curl -s http://oidc:8080/tokens | jq -r '.[0]')"
 response="$(curl -s -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')"
 if [ "$response" != "uploaded" ]; then
 	echo "API returned incorrect value, expected ready got: $response"
@@ -52,6 +52,12 @@ if [ "$resp" != "200" ]; then
 	exit 1
 fi
 
+ts=$(date +"%F %T")
+depr="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq -r .[1].deprecated_at)"
+if [ "$depr" != "$ts" ]; then
+	echo "Error when listing key hash, expected $ts got: $depr"
+	exit 1
+fi
 
 # list key hashes
 resp="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq '. | length')"
@@ -66,11 +72,5 @@ if [ "$resp" != "$manual_hash" ]; then
 	echo "Error when listing key hash, expected $manual_hash got: $resp"
 	exit 1
 fi
-ts=$(date +"%F %T")
-depr="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq -r .[1].deprecated_at)"
-if [ "$depr" != "$ts" ]; then
-	echo "Error when listing key hash, expected $ts got: $depr"
-	exit 1
-fi
 
 echo "api test completed successfully"

charts/sda-svc/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: sda-svc
-version: 0.29.0
+version: 0.30.0
 appVersion: v0.3.170
 kubeVersion: '>= 1.26.0'
 description: Components for Sensitive Data Archive (SDA) installation

charts/sda-svc/README.md

@@ -123,7 +123,8 @@ Parameter | Description | Default
 `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null`
 `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null`
 `global.download.enabled` | Deploy the download service | `true`
-`global.download.serveUnencryptedData` | Whether the download service serves unencrypted data | `false`
+`global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""`
+`global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key | `""`
 `global.download.sessionExpiration` | Session key expiration time in seconds | `28800`
 `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss`
 `global.download.trusted.configFile` | Name of ISS config file | `iss.json`

charts/sda-svc/templates/download-deploy.yaml

@@ -140,9 +140,14 @@ spec:
         - name: LOG_LEVEL
           value: {{ .Values.global.log.level | quote }}
       {{- end }}
-      {{- if .Values.global.download.serveUnencryptedData }}
-        - name: APP_SERVEUNENCRYPTEDDATA
-          value: {{ .Values.global.download.serveUnencryptedData }}
+      {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }}
+        - name: APP_C4GH_PRIVATEKEYPATH
+          value: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }}
+        - name: APP_C4GH_PASSPHRASE
+          valueFrom:
+            secretKeyRef:
+              name: {{ required "A secret for the transient c4gh key is required" .Values.global.download.serveDecrypted.secretName }}
+              key: passphrase
       {{- end }}
       {{- if .Values.global.tls.enabled }}
         - name: APP_PORT
@@ -224,8 +229,10 @@ spec:
 {{ toYaml .Values.download.resources | trim | indent 10 }}
         volumeMounts:
         {{- if not .Values.global.vaultSecrets }}
-        - name: c4gh
+        {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }}
+        - name: c4gh-transient
           mountPath: {{ template "c4ghPath" . }}
+        {{- end }}
         - name: iss
           mountPath: {{ template "trustedIssPath" . }}
         {{- end }}
@@ -251,13 +258,15 @@ spec:
         {{- end }}
       {{- end }}
       {{- if not .Values.global.vaultSecrets }}
-        - name: c4gh
+      {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }}
+        - name: c4gh-transient
           secret:
             defaultMode: 0440
-            secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }}
+            secretName: {{ required "A secret for the transient c4gh key is required" .Values.global.download.serveDecrypted.secretName }}
             items:
-            - key: {{ .Values.global.c4gh.keyFile }}
-              path: {{ .Values.global.c4gh.keyFile }}
+            - key: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }}
+              path: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }}
+      {{- end }}
         - name: iss
           secret:
             defaultMode: 0440

charts/sda-svc/values.yaml

@@ -239,7 +239,9 @@ global:
       iss:
         - iss: "https://login.elixir-czech.org/oidc"
           jku: "https://login.elixir-czech.org/oidc/jwk"
-    serveUnencryptedData: false
+    serveDecrypted:
+      c4ghKeyFile: ""
+      secretName: ""
 
   oidc:
     provider:  "https://login.elixir-czech.org/oidc/"

sda-download/.github/integration/setup/common/1_keys.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/common/20_tools.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+set -e
+
+C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')"
+curl --retry 100 -sL https://github.com/neicnordic/crypt4gh/releases/download/"${C4GH_VERSION}"/crypt4gh_linux_x86_64.tar.gz | sudo tar -xz -C /usr/bin/ &&
+        sudo chmod +x /usr/bin/crypt4gh
 
-pip3 install crypt4gh
 sudo apt install -y jq s3cmd

sda-download/.github/integration/setup/common/23_create_db_entries.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-
-pip3 install s3cmd
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-
-pip3 install s3cmd
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/tests/common/30_check_db.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+set -e
 
 cd dev_utils || exit 1
 

sda-download/.github/integration/tests/common/50_check_endpoint.sh

@@ -56,11 +56,11 @@ echo "got correct response when POST method used"
 # ------------------
 # Test good token
 
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
 
 ## Test datasets endpoint
 
-check_dataset=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')
+check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]')
 
 if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then
     echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found"
@@ -72,7 +72,7 @@ echo "expected dataset found"
 
 ## Test datasets/files endpoint
 
-check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
+check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId')
 
 if [ "$check_files" != "urn:neic:001-002" ]; then
     echo "file with id urn:neic:001-002 not found"
@@ -88,10 +88,9 @@ echo "expected file found"
 C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
 export C4GH_PASSPHRASE
 
-crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt
-
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt
+crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt
 
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002" --output test-download.txt
 
 cmp --silent old-file.txt test-download.txt
 status=$?
@@ -102,7 +101,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt
 
 dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt
 
@@ -115,7 +114,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt
 
 dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt
 
@@ -128,7 +127,7 @@ else
     exit 1
 fi
 
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt
 
 dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt
 
@@ -141,10 +140,19 @@ else
     exit 1
 fi
 
+# test that downloads of decrypted files from a download instance that
+# serves only encrypted files (here running at port 8443) should fail
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download-fail.txt
+
+if ! grep -q "downloading unencrypted data is not supported" test-download-fail.txt; then
+    echo "got unexpected response when trying to download unencrypted data from encrypted endpoint"
+exit 1
+fi
+
 # ------------------
 # Test get visas failed
 
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[1]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[1]')
 
 ## Test datasets endpoint
 
@@ -162,7 +170,7 @@ echo "got correct response when token has no permissions"
 # Test token with untrusted sources
 # for this test we attach a list of trusted sources
 
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[2]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[2]')
 
 ## Test datasets endpoint
 
@@ -175,3 +183,8 @@ if [ "$check_empty_token" != "200" ]; then
 fi
 
 echo "got correct response when token permissions from untrusted sources"
+
+# cleanup
+rm old-file.txt old-part*.txt test-download*.txt test-part*.txt
+
+echo "OK"

sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-
 TLS="True"
 [[ "$STORAGETYPE" == "s3notls" ]] && TLS="False"
 
@@ -76,9 +75,9 @@ auth_token() {
 #
 # Create s3cmd configs
 #
-
-port="8443"
-[[ "$TLS" == "False" ]] && port="8080"
+# ports 9443/9080 refer to the download instance serving unencrypted files
+port="9443"
+[[ "$TLS" == "False" ]] && port="9080"
 
 cat << EOF >s3cmd.valid
 [default]
@@ -174,6 +173,8 @@ export should_return="70"
 info 'Testing valid file download with an invalid token'
 run_test s3cmd -c s3cmd.invalid get --force "s3://$dataset/dummy_data"
 
+# cleanup
+rm s3cmd.valid s3cmd.invalid
 
 echo
 info " ----- End of S3 Tests ------------------------------------------------- "

sda-download/.github/integration/tests/common/70_check_download.sh

@@ -7,15 +7,15 @@ fi
 cd dev_utils || exit 1
 
 # get a token, set up variables
-token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
+token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r  '.[0]')
 dataset="https://doi.example/ty009.sfrrss/600.45asasga"
 file="dummy_data"
 expected_size=1048605
 C4GH_PASSPHRASE=$(grep -F passphrase config.yaml | sed -e 's/.* //' -e 's/"//g')
 export C4GH_PASSPHRASE
 
 # download decrypted full file,  check file size
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file" --output full1.bam
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/s3/$dataset/$file" --output full1.bam
 file_size=$(stat -c %s full1.bam)  # Get the size of the file
 
 if [ "$file_size" -ne "$expected_size" ]; then
@@ -24,9 +24,14 @@ if [ "$file_size" -ne "$expected_size" ]; then
 fi
 
 # test that start, end=0 returns the whole file
-curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam
+curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/s3/$dataset/$file?startCoordinate=0&endCoordinate=0" --output full2.bam
 
 if ! cmp --silent full1.bam full2.bam; then
     echo "Full decrypted files, with and without coordinates, are different"
     exit 1
 fi
+
+# cleanup
+rm full*.bam
+
+echo "OK"