Merge pull request #28 from FNNDSC/pfcon-v1a

Pfcon version 1.0.0-alpha.1

charts/pfcon/.gitignore

@@ -0,0 +1,2 @@
+/charts/*.tgz
+

charts/pfcon/.helmignore

@@ -22,5 +22,6 @@
 *.tmproj
 .vscode/
 
+# helm-unittest
 /tests
 

charts/pfcon/Chart.lock

@@ -0,0 +1,6 @@
+dependencies:
+- name: util
+  repository: https://fnndsc.github.io/charts
+  version: 0.1.0
+digest: sha256:ceee9595de62e5f7c0cbc7df9f7ac03a33b4dd91964c7321b9e849e0f90453f9
+generated: "2024-11-25T09:45:17.544210765-05:00"

charts/pfcon/Chart.yaml

@@ -7,10 +7,15 @@ sources:
 - https://github.com/FNNDSC/pfcon
 
 type: application
-version: "0.2.3"
+version: "1.0.0-alpha.1"
 appVersion: "5.2.3"
 
 maintainers:
 - name: The FNNDSC Dev Team
   email: [email protected]
   url: https://fnndsc.org
+
+dependencies:
+- name: util
+  version: "~0.1.0"
+  repository: "https://fnndsc.github.io/charts"

charts/pfcon/templates/_helpers.tpl

@@ -61,3 +61,4 @@ Create the name of the service account to use
 {{- define "pfcon.storebase" -}}
 {{ .Values.storage.existingClaim | default (printf "%s-storebase" .Release.Name) }}
 {{- end }}
+

charts/pfcon/templates/deployment.yml

@@ -24,8 +24,11 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       serviceAccountName: {{ include "pfcon.serviceAccountName" . }}
+      {{- $podSecurityContext := merge .Values.podSecurityContext .Values.global.podSecurityContext }}
+      {{- with $podSecurityContext }}
       securityContext:
-        {{- toYaml .Values.global.podSecurityContext | nindent 8 }}
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -34,55 +37,80 @@ spec:
         - name: pfcon
           image: "{{ .Values.pfcon.image.repository }}:{{ .Values.pfcon.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.pfcon.image.pullPolicy }}
-          command: ["gunicorn", "--bind", "0.0.0.0:5005", "--workers", "{{ .Values.pfcon.workers }}", "--timeout", "{{ .Values.pfcon.workerTimeout }}", "pfcon.wsgi:application"]
+          command: ["/entrypoint.sh"]
+          args: ["gunicorn", "-b", "0.0.0.0:{{ .Values.service.port }}", "pfcon.wsgi:application"]
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-pfcon
             - secretRef:
                 name: {{ .Release.Name }}-pfcon
           env:
             - name: COMPUTE_SERVICE_URL
-              # pman and pfcon are in the same pod
-              value: http://localhost:5010/api/v1/
+              value: http://localhost:{{ .Values.pman.containerPort }}/api/v1/
             - name: STOREBASE_MOUNT
               value: /var/local/storeBase
+            {{- range $k, $v := .Values.pfcon.extraEnv }}
+            - name: {{ $k }}
+              value: {{ $v | quote }}
+            {{- end }}
           volumeMounts:
             - mountPath: /var/local/storeBase
               name: storebase
           ports:
             - name: http
-              containerPort: 5005
+              containerPort: {{ .Values.service.port }}
               protocol: TCP
-          livenessProbe:
+              {{- if .Values.pfcon.hostPort }}
+              hostPort: {{ .Values.pfcon.hostPort }}
+              {{- end }}
+          readinessProbe:
             httpGet:
               path: /api/v1/health/
               port: http
-            initialDelaySeconds: 10
             periodSeconds: 3
+          {{- with .Values.pfcon.resources }}
           resources:
-            {{- toYaml .Values.pfcon.resources | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.pfcon.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
         - name: pman
           image: "{{ .Values.pman.image.repository }}:{{ .Values.pman.image.tag | default .Chart.AppVersion }}"
           imagePullPolicy: {{ .Values.pman.image.pullPolicy }}
-          command: ["gunicorn", "--bind", "0.0.0.0:5010", "--workers", "{{ .Values.pman.workers }}", "--timeout", "{{ .Values.pman.workerTimeout }}", "pman.wsgi:application"]
+          command: ["gunicorn", "-b", "0.0.0.0:{{ .Values.pman.containerPort }}", "pman.wsgi:application"]
           envFrom:
             - configMapRef:
                 name: {{ .Release.Name }}-pman
             - secretRef:
                 name: {{ .Release.Name }}-pman
           ports:
             - name: http
-              containerPort: 5010
+              containerPort: {{ .Values.pman.containerPort }}
               protocol: TCP
-          livenessProbe:
+              {{- if .Values.pfcon.hostPort }}
+              hostPort: {{ .Values.pfcon.hostPort }}
+              {{- end }}
+          readinessProbe:
             httpGet:
               path: /api/v1/
               port: http
-            initialDelaySeconds: 10
             periodSeconds: 3
+          {{- with .Values.pman.resources }}
           resources:
-            {{- toYaml .Values.pman.resources | nindent 12 }}
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          {{- with .Values.pman.securityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
       volumes:
         - name: storebase
           persistentVolumeClaim:
             claimName: {{ include "pfcon.storebase" . }}
+        {{- if .Values.pman.podman.enabled }}
+        - name: podman-socket
+          hostPath:
+            path: {{ .Values.pman.podman.socket | required ".Values.pman.podman.socket is required" }}
+        {{- end }}

charts/pfcon/templates/pfcon-config.yml

@@ -6,5 +6,14 @@ metadata:
   labels:
     {{- include "pfcon.labels" . | nindent 4 }}
 data:
+  {{- if (hasKey .Values.pfcon.extraEnv "PFCON_INNETWORK") }}
+  {{- fail "Do not set pfcon.extraEnv.PFCON_INNETWORK. Use pfcon.config.innetwork instead." }}
+  {{- else if (hasKey .Values.pfcon.extraEnv "STORAGE_ENV") }}
+  {{- fail "Do not set pfcon.extraEnv.STORAGE_ENV. Use pfcon.config.storageEnv instead." }}
+  {{- end }}
   PFCON_INNETWORK: {{ .Values.pfcon.config.innetwork | quote }}
   STORAGE_ENV: {{ .Values.pfcon.config.storageEnv }}
+  {{- if (hasKey .Values.pfcon.extraEnv "SECRET_KEY") }}
+  {{- fail "Do not set pfcon.extraEnv.SECRET_KEY. Use pfcon.secretKey instead." }}
+  {{- end }}
+  {{- toYaml .Values.pfcon.extraEnv | nindent 2 }}

charts/pfcon/templates/pfcon-secrets.yml

@@ -1,18 +1,14 @@
+{{- $pfconSecret := (print .Release.Name "-pfcon") -}}
 apiVersion: v1
 kind: Secret
 type: Opaque
 metadata:
-  name: {{ .Release.Name }}-pfcon
+  name: {{ $pfconSecret }}
   namespace: {{ .Release.Namespace }}
   labels:
     {{- include "pfcon.labels" . | nindent 4 }}
+stringData:
+  PFCON_USER: 'pfcon'
 data:
-  {{- with (lookup "v1" "Secret" .Release.Namespace ( print .Release.Name "-pfcon")) }}
-  SECRET_KEY: {{ index .data "SECRET_KEY" }}
-  PFCON_USER: {{ index .data "PFCON_USER" }}
-  PFCON_PASSWORD: {{ index .data "PFCON_PASSWORD" }}
-  {{- else }}
-  SECRET_KEY: {{ randAlphaNum 32 | b64enc | quote }}
-  PFCON_USER: {{ "pfcon" | b64enc | quote }}
-  PFCON_PASSWORD: {{ randAlphaNum 32 | b64enc | quote }}
-  {{- end }}
+  SECRET_KEY: {{ include "util.valueOrLookupOrRandom" (dict "root" . "value" .Values.pfcon.secretKey "secret" $pfconSecret "name" "SECRET_KEY" "length" 32 ) }}
+  PFCON_PASSWORD: {{ include "util.valueOrLookupOrRandom" (dict "root" . "value" .Values.password "secret" $pfconSecret "name" "PFCON_PASSWORD" "length" 32 ) }}

charts/pfcon/templates/pman-config.yml

@@ -10,14 +10,17 @@ data:
   STORAGE_TYPE: kubernetes_pvc
   JOB_NAMESPACE: {{ .Release.Namespace }}
   VOLUME_NAME: {{ include "pfcon.storebase" . }}
-  {{- toYaml .Values.pman.config | nindent 2 }}
+  {{- if (hasKey .Values.pman.extraEnv "SECRET_KEY") }}
+  {{- fail "Do not set pman.extraEnv.SECRET_KEY. Use pman.secretKey instead." }}
+  {{- end }}
+  {{- toYaml .Values.pman.extraEnv | nindent 2 }}
   {{- $setSecurityContextIsSome := kindIs "bool" .Values.pman.setSecurityContext }}
-  {{- if (and (hasKey .Values.pman.config "CONTAINER_USER") $setSecurityContextIsSome (not .Values.pman.setSecurityContext)) }}
-  {{- fail "Cannot set value for .Values.pman.config.CONTAINER_USER because .Values.pman.setSecurityContext=false." }}
+  {{- if (and (hasKey .Values.pman.extraEnv "CONTAINER_USER") $setSecurityContextIsSome (not .Values.pman.setSecurityContext)) }}
+  {{- fail "Cannot set value for .Values.pman.extraEnv.CONTAINER_USER because .Values.pman.setSecurityContext=false." }}
   {{- end }}
   {{- $setSecurityContextIsNone := not $setSecurityContextIsSome }}
   {{- $isOpenShift := .Capabilities.APIVersions.Has "security.openshift.io/v1/SecurityContextConstraints" }}
-  {{- if (and (not (hasKey .Values.pman.config "CONTAINER_USER"))
+  {{- if (and (not (hasKey .Values.pman.extraEnv "CONTAINER_USER"))
          (or (and $setSecurityContextIsSome .Values.pman.setSecurityContext)
              (and $setSecurityContextIsNone (not $isOpenShift)))) }}
   {{- if .Values.global.podSecurityContext }}

charts/pfcon/templates/pman-secrets.yml

@@ -7,8 +7,5 @@ metadata:
   labels:
     {{- include "pfcon.labels" . | nindent 4 }}
 data:
-  {{- with (lookup "v1" "Secret" .Release.Namespace ( print .Release.Name "-pman")) }}
-  SECRET_KEY: {{ index .data "SECRET_KEY" }}
-  {{- else }}
-  SECRET_KEY: {{ randAlphaNum 32 | b64enc | quote }}
-  {{- end }}
+  SECRET_KEY: {{ include "util.valueOrLookupOrRandom" (dict "root" . "value" .Values.pman.secretKey "secret" (print .Release.Name "-pman") "name" "SECRET_KEY" "length" 32 ) }}
+

charts/pfcon/templates/tests/test-connection.yaml

@@ -11,7 +11,7 @@ metadata:
 spec:
   containers:
     - name: test-pfcon
-      image: docker.io/curlimages/curl:8.4.0
+      image: quay.io/curl/curl:8.10.1
       command: ["curl"]
       args:
         - "-sSf"
@@ -23,7 +23,7 @@ spec:
         - "-H"
         - "Content-Type: application/json"
         - "--data"
-        - '{"pfcon_user": "pfcon", "pfcon_password": "$(PFCON_PASSWORD)"}'
+        - '{"pfcon_user": "$(PFCON_USER)", "pfcon_password": "$(PFCON_PASSWORD)"}'
       envFrom:
         - secretRef:
             name: {{ .Release.Name }}-pfcon

charts/pfcon/tests/pfcon_config_validation_test.yaml

@@ -0,0 +1,22 @@
+suite: Validation of pfcon config
+templates:
+  - pfcon-config.yml
+tests:
+  - it: should show an error if pfcon.extraEnv.SECRET_KEY is set
+    set:
+      pfcon.extraEnv.SECRET_KEY: abc123
+    asserts:
+      - failedTemplate:
+          errorMessage: Do not set pfcon.extraEnv.SECRET_KEY. Use pfcon.secretKey instead.
+  - it: should not allow pfcon.extraEnv.PFCON_INNETWORK to be set
+    set:
+      pfcon.extraEnv.PFCON_INNETWORK: true
+    asserts:
+      - failedTemplate:
+          errorMessage: Do not set pfcon.extraEnv.PFCON_INNETWORK. Use pfcon.config.innetwork instead.
+  - it: should not allow pfcon.extraEnv.STORAGE_ENV to be set
+    set:
+      pfcon.extraEnv.STORAGE_ENV: "filesystem"
+    asserts:
+      - failedTemplate:
+          errorMessage: Do not set pfcon.extraEnv.STORAGE_ENV. Use pfcon.config.storageEnv instead.

charts/pfcon/tests/pman_config_disallow_extraEnv_secret_key_test.yaml

@@ -0,0 +1,10 @@
+suite: Disallow insecure setting of SECRET_KEY
+templates:
+  - pman-config.yml
+tests:
+  - it: should show an error if pman.extraEnv.SECRET_KEY is set
+    set:
+      pman.extraEnv.SECRET_KEY: abc123
+    asserts:
+      - failedTemplate:
+          errorMessage: Do not set pman.extraEnv.SECRET_KEY. Use pman.secretKey instead.

charts/pfcon/tests/set_security_context_test.yaml

@@ -25,23 +25,23 @@ tests:
           value: '50505:303'
   - it: should allow CONTAINER_USER to be set when setSecurityContext is unset
     set:
-      pman.config.CONTAINER_USER: '20202:0'
+      pman.extraEnv.CONTAINER_USER: '20202:0'
     asserts:
       - equal:
           path: data.CONTAINER_USER
           value: '20202:0'
   - it: should allow CONTAINER_USER to be set when setSecurityContext=true
     set:
-      pman.config.CONTAINER_USER: '20202:0'
+      pman.extraEnv.CONTAINER_USER: '20202:0'
       pman.setSecurityContext: true
     asserts:
       - equal:
           path: data.CONTAINER_USER
           value: '20202:0'
   - it: should disallow setting of CONTAINER_USER when setSecurityContext=false
     set:
-      pman.config.CONTAINER_USER: '20202:0'
+      pman.extraEnv.CONTAINER_USER: '20202:0'
       pman.setSecurityContext: false
     asserts:
       - failedTemplate:
-          errorMessage: Cannot set value for .Values.pman.config.CONTAINER_USER because .Values.pman.setSecurityContext=false.
+          errorMessage: Cannot set value for .Values.pman.extraEnv.CONTAINER_USER because .Values.pman.setSecurityContext=false.

charts/pfcon/tests/valueOrLookupOrRandom_changed_name_test.yaml

@@ -0,0 +1,29 @@
+suite: util.valueOrLookupOrRandom subsequent run (with changed name)
+templates:
+  - pman-secrets.yml
+release:
+  name: subsequent-run-test
+  namespace: suruntest
+  revision: 2
+  upgrade: true
+kubernetesProvider:
+  scheme:
+    "v1/Secret":
+      gvr:
+        version: v1
+        resource: secrets
+      namespaced: true
+  objects:
+    - kind: Secret
+      apiVersion: v1
+      metadata:
+        name: subsequent-run-test-pman
+        namespace: suruntest
+      data:
+        NOT_SECRET_KEY: 'aWFtYm9sZAo='
+tests:
+  - it: should randomly generate a new secret because old secret does not contain the right key
+    asserts:
+      - matchRegex:
+          path: data.SECRET_KEY
+          pattern: ^[-A-Za-z0-9+/]{40,}={0,3}$

charts/pfcon/tests/valueOrLookupOrRandom_first_run_test.yaml

@@ -0,0 +1,17 @@
+suite: util.valueOrLookupOrRandom first run
+templates:
+  - pman-secrets.yml
+tests:
+  - it: should generate a random secret, base64-encoded
+    asserts:
+      - matchRegex:
+          path: data.SECRET_KEY
+          pattern: ^[-A-Za-z0-9+/]{40,}={0,3}$
+  - it: should use the given secret
+    set:
+      pman.secretKey: namesjamesbond
+    asserts:
+      - equal:
+          path: data.SECRET_KEY
+          value: namesjamesbond
+          decodeBase64: true

charts/pfcon/tests/valueOrLookupOrRandom_subsequent_run_test.yaml

@@ -0,0 +1,39 @@
+suite: util.valueOrLookupOrRandom subsequent run
+templates:
+  - pman-secrets.yml
+release:
+  name: subsequent-run-test
+  namespace: suruntest
+  revision: 2
+  upgrade: true
+kubernetesProvider:
+  scheme:
+    "v1/Secret":
+      gvr:
+        version: v1
+        resource: secrets
+      namespaced: true
+  objects:
+    - kind: Secret
+      apiVersion: v1
+      metadata:
+        name: subsequent-run-test-pman
+        namespace: suruntest
+      data:
+        SECRET_KEY: 'aWFtYm9sZAo='
+tests:
+  - it: should use the previous value
+    asserts:
+      - equal:
+          path: data.SECRET_KEY
+          value: 'aWFtYm9sZAo='
+          # note: decodeBase64 has a bug.
+          # See https://github.com/helm-unittest/helm-unittest/issues/494
+  - it: should use the given secret
+    set:
+      pman.secretKey: namesjamesbond
+    asserts:
+      - equal:
+          path: data.SECRET_KEY
+          value: namesjamesbond
+          decodeBase64: true