PAAS improvements

.github/workflows/test.yaml

@@ -94,7 +94,9 @@ jobs:
       - name: Install Kyverno
         run: |
           helm repo add kyverno https://kyverno.github.io/kyverno/
-          helm install kyverno kyverno/kyverno -n kyverno --create-namespace --version 3.1.4
+          helm install kyverno kyverno/kyverno -n kyverno --create-namespace -f .github/config/kyverno-values.yaml --version 3.1.4
+          timeout 60 /bin/bash -c 'until kubectl get pods -n kyverno -l app.kubernetes.io/component=admission-controller -o jsonpath="{.items[0].status.phase}" | grep Running ; do echo "Waiting for Kyverno" ; sleep 10 ; done'
+          sleep 60
       - name: Install cert-manager
         if: matrix.chart == 'paas'
         run: |
@@ -111,3 +113,4 @@ jobs:
           kubectl describe pod -A
           kubectl describe service -A
           kubectl describe daemonset -A
+          kubectl logs -n kyverno -l app.kubernetes.io/component=admission-controller

Makefile

@@ -37,7 +37,7 @@ kyverno-copy-policies: $(KYVERNO_POLICIES)
 	done
 
 kyverno-test: $(KYVENOR_CLI) kyverno-copy-policies
-	$(KYVENOR_CLI) test $(KYVERNO_POLICY_TESTS_DIR)
+	$(KYVENOR_CLI) test --detailed-results $(KYVERNO_POLICY_TESTS_DIR)
 
 encrypt-private-values: $(PRIVATE_CHARTS)
 	@for d in $(dir $^); do \

charts/kyverno-policies/Chart.yaml

@@ -2,7 +2,7 @@ apiVersion: v2
 name: kyverno-policies
 description: OSC Kyverno policies deployment
 type: application
-version: 0.27.0
+version: 0.28.0
 appVersion: "v1.11.4"
 maintainers:
   - name: treydock

charts/kyverno-policies/templates/add-annotations.yaml

@@ -24,3 +24,20 @@ spec:
         metadata:
           annotations:
             prometheus.io/scrape: 'false'
+  - name: paas-cert-manager
+    match:
+      any:
+        - resources:
+            kinds:
+            - Ingress
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    mutate:
+      patchStrategicMerge:
+        metadata:
+          annotations:
+            cert-manager.io/cluster-issuer: "{{ .Values.paas.certManagerClusterIssuer }}"

charts/kyverno-policies/templates/add-image-pull-secret.yaml

@@ -0,0 +1,34 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-image-pull-secret
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+    - name: paas-osc-registry
+      match:
+        any:
+        - resources:
+            kinds:
+            - Pod
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+      context:
+      - name: secret
+        apiCall:
+          urlPath: "/api/v1/namespaces/{{`{{ request.namespace }}`}}/secrets/osc-registry"
+          jmesPath: "metadata.name || ''"
+      preconditions:
+      - key: "{{`{{ secret || '' }}`}}"
+        operator: NotEquals
+        value: ""
+      mutate:
+        patchStrategicMerge:
+          spec:
+            imagePullSecrets:
+            - name: "{{`{{ secret }}`}}"

charts/kyverno-policies/templates/add-ingress-class-name.yaml

@@ -0,0 +1,24 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: add-ingress-class-name
+spec:
+  validationFailureAction: Enforce
+  background: true
+  rules:
+  - name: add-ingress-class-name
+    match:
+      any:
+        - resources:
+            kinds:
+            - Ingress
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    mutate:
+      patchStrategicMerge:
+        spec:
+          ingressClassName: "{{ .Values.paas.ingressClassName }}"

charts/kyverno-policies/templates/add-service-account.yaml

@@ -35,41 +35,40 @@ spec:
       configMap:
         name: user-gids-map
         namespace: k8-ldap-configmap
-    mutate:
-      patchStrategicMerge:
-        spec:
-          securityContext:
-            runAsUser: "{{`{{ uidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
-            runAsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
-            fsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
-            supplementalGroups: "{{`{{ gidsMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\" | parse_json(@)[*].to_number(@) }}`}}"
-  - name: webservice-service-account-run-as-containers
-    match:
-      any:
-        - resources:
-            kinds:
-            - Pod
-            namespaceSelector:
-              matchExpressions:
-              - key: osc.edu/role
-                operator: In
-                values:
-                - webservice
-    preconditions:
-    - key: "{{`{{ request.object.metadata.labels.\"osc.edu/service-account\" || '' }}`}}"
-      operator: NotEquals
-      value: ""
-    context:
-    - name: uidMap
-      configMap:
-        name: user-uid-map
-        namespace: k8-ldap-configmap
-    - name: gidMap
-      configMap:
-        name: user-gid-map
-        namespace: k8-ldap-configmap
     mutate:
       foreach:
+      - list: "request.object.spec"
+        patchStrategicMerge:
+          spec:
+            securityContext:
+              runAsNonRoot: true
+              runAsUser: "{{`{{ uidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
+              runAsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
+              fsGroup: "{{`{{ gidMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\".to_number(@) }}`}}"
+              supplementalGroups: "{{`{{ gidsMap.data.\"user-{{ request.object.metadata.labels.\"osc.edu/service-account\" }}\" | parse_json(@)[*].to_number(@) }}`}}"
+      - list: "request.object.spec.[containers, initContainers][]"
+        patchStrategicMerge:
+          spec:
+            containers:
+            - (name): "*"
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                  - ALL
+                seccompProfile:
+                  type: RuntimeDefault
+                privileged: false
+            initContainers:
+            - (name): "*"
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                  - ALL
+                seccompProfile:
+                  type: RuntimeDefault
+                privileged: false
       - list: "request.object.spec.[containers, initContainers][]"
         patchStrategicMerge:
           spec:
@@ -135,10 +134,34 @@ spec:
         patchStrategicMerge:
           spec:
             securityContext:
+              runAsNonRoot: true
               runAsUser: "{{`{{ uidMap.data.\"user-{{ serviceAccount }}\".to_number(@) }}`}}"
               runAsGroup: "{{`{{ gidMap.data.\"user-{{ serviceAccount }}\".to_number(@) }}`}}"
               fsGroup: "{{`{{ gidMap.data.\"user-{{ serviceAccount }}\".to_number(@) }}`}}"
               supplementalGroups: "{{`{{ gidsMap.data.\"user-{{ serviceAccount }}\" | parse_json(@)[*].to_number(@) }}`}}"
+      - list: "request.object.spec.[containers, initContainers][]"
+        patchStrategicMerge:
+          spec:
+            containers:
+            - (name): "*"
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                  - ALL
+                seccompProfile:
+                  type: RuntimeDefault
+                privileged: false
+            initContainers:
+            - (name): "*"
+              securityContext:
+                allowPrivilegeEscalation: false
+                capabilities:
+                  drop:
+                  - ALL
+                seccompProfile:
+                  type: RuntimeDefault
+                privileged: false
       - list: "request.object.spec.[containers, initContainers][]"
         patchStrategicMerge:
           spec:

charts/kyverno-policies/templates/ingress-allowed-dns.yaml

@@ -0,0 +1,38 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-allowed-dns
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: allowed-dns
+    match:
+      any:
+        - resources:
+            kinds:
+            - Ingress
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    context:
+    - name: allowed
+      apiCall:
+        urlPath: "/api/v1/namespaces/{{`{{ request.namespace }}`}}"
+        jmesPath: "metadata.annotations.\"osc.edu/allowed-dns\" || ''"
+    validate:
+      message: "DNS host requested is not allowed"
+      foreach:
+      - list: request.object.spec.rules
+        deny:
+          conditions:
+            all:
+            - key: "{{`{{ element.host }}`}}"
+              operator: Equals
+              value: "*.osc.edu"
+            - key: "{{`{{ element.host }}`}}"
+              operator: NotIn
+              value: "{{`{{ allowed | split(@, ',') }}`}}"

charts/kyverno-policies/templates/ingress-annotations.yaml

@@ -0,0 +1,26 @@
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: ingress-annotations
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: deny-external-dns-annotations
+    match:
+      any:
+        - resources:
+            kinds:
+            - Ingress
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    validate:
+      message: "External DNS annotations are now allowed"
+      pattern:
+        metadata:
+          =(annotations):
+            X(external-dns.alpha.kubernetes.io/*): "*?"

charts/kyverno-policies/templates/namespace-account.yaml

@@ -21,3 +21,34 @@ spec:
         metadata:
           labels:
             account: "?*"
+  - name: valid-account
+    match:
+      any:
+      - resources:
+          kinds:
+          - Namespace
+          selector:
+            matchLabels:
+              {{ include "osc.common.roleKey" . }}: paas
+    preconditions:
+    - key: "{{`{{ request.operation }}`}}"
+      operator: In
+      value: ["CREATE","UPDATE"]
+    - key: "{{`{{ request.object.metadata.labels.account || '' }}`}}"
+      operator: NotEquals
+      value: ""
+    - key: "{{`{{`}} request.object.metadata.labels.\"{{ include "osc.common.serviceAccountKey" . }}\" || '' {{`}}`}}"
+      operator: NotEquals
+      value: ""
+    context:
+    - name: userGroupMap
+      configMap:
+        name: user-groups-map
+        namespace: k8-ldap-configmap
+    validate:
+      message: "{{`{{`}} request.object.metadata.labels.\"{{ include "osc.common.serviceAccountKey" . }}\" {{`}}`}} not authorized to charge against account {{`{{ request.object.metadata.labels.account }}`}}"
+      deny:
+        conditions:
+        - key: "{{`{{ request.object.metadata.labels.account }}`}}"
+          operator: NotIn
+          value: "{{`{{`}} userGroupMap.data.\"user-{{`{{`}} request.object.metadata.labels.\"{{ include "osc.common.serviceAccountKey" . }}\" {{`}}`}}\" {{`}}`}}"

charts/kyverno-policies/templates/pod-host-port.yaml

@@ -0,0 +1,39 @@
+# REF: https://kyverno.io/policies/pod-security/baseline/disallow-host-ports/disallow-host-ports/
+apiVersion: kyverno.io/v1
+kind: ClusterPolicy
+metadata:
+  name: pod-host-port
+spec:
+  background: true
+  validationFailureAction: Enforce
+  rules:
+  - name: no-host-port
+    match:
+      any:
+        - resources:
+            kinds:
+            - Pod
+            namespaces:
+            - "user-?*"
+        - resources:
+            kinds:
+            - Pod
+            namespaceSelector:
+              matchExpressions:
+              - key: osc.edu/role
+                operator: In
+                values:
+                - paas
+    validate:
+      message: "Pod hostPort is not allowed"
+      pattern:
+        spec:
+          =(ephemeralContainers):
+            - =(ports):
+                - =(hostPort): 0
+          =(initContainers):
+            - =(ports):
+                - =(hostPort): 0
+          containers:
+            - =(ports):
+                - =(hostPort): 0