bcgov · abimichel · Feb 23, 2024 · Feb 23, 2024 · Feb 23, 2024 · Feb 27, 2024
diff --git a/.github/workflows/docker-publish-opensearch.yml b/.github/workflows/docker-publish-opensearch.yml
@@ -26,7 +26,7 @@ jobs:
         uses: actions/checkout@v3
       # DF-NOTE: delete after merging PR
         with: 
-          ref: security-context-changes
+          ref: a9cb09-dev
 
       - name: Install cosign
         if: github.event_name != 'pull_request'

diff --git a/.gitignore b/.gitignore
@@ -1 +1,3 @@
-charts-1.2.4
+openmetadata-dependencies-1.2.5
+charts-1.2.4
+charts-1.2.5
diff --git a/README.md b/README.md
@@ -72,46 +72,42 @@ To deploy to OpenShift, use OC commands and make sure Helm cli is installed on y
 ```
 brew install helm
 ```
-### First install Dependencies
-Source: https://github.com/open-metadata/openmetadata-helm-charts/tree/main/charts/deps
 
-#### Create secrets
+### Apply all the network policies and pod label policies under 'oc' folder:
+Navigate to the 'oc' folder then:
+```
+oc apply -f .
+```
+
+#### Create default secrets
 ```
-oc create secret generic airflow-mysql-secrets --from-literal=airflow-mysql-password=airflow_pass
+oc create secret generic airflow-db-secrets --from-literal=airflow-db-password=airflow_pass
+oc create secret generic db-secrets --from-literal=openmetadata-db-password=openmetadata_password
+oc create secret generic airflow-secrets --from-literal=openmetadata-airflow-password=admin
 ```
 #### Deploy dependencies to OpenShift
+Source: https://github.com/open-metadata/openmetadata-helm-charts/tree/main/charts/deps
+
 Navigate to the 'deps' chart folder then:
 ```
 helm install openmetadata-dependencies .
 ```
 If you see the below error then get admin access to the dev namespace
 Issues:  User "[email protected]" cannot get resource "roles" in API group "rbac.authorization.k8s.io" in the namespace "a1b9b0-dev"
 
-### Install OpenMetadata
+#### Deploy OpenMetadata to OpenShift
 Source: https://github.com/open-metadata/openmetadata-helm-charts/tree/main/charts/openmetadata
 
-#### Create default Secrets
-```
-oc create secret generic mysql-secrets --from-literal=openmetadata-mysql-password=openmetadata_password
-oc create secret generic airflow-secrets --from-literal=openmetadata-airflow-password=admin
-```
-## Apply the pod label policies under 'oc' folder: 
-```
-oc apply -f [auto-label].yaml 
-```
-## Apply the network policies under 'oc' folder: 
-```
-oc apply -f [net-pol].yaml 
-```
-#### Deploy OpenMetadata to OpenShift:
-Navigate to the 'openmetadata' chart folder then:
+Once all the dependencies are running, navigate to the 'openmetadata' chart folder then:
 ```
 helm install openmetadata .
 ```
-#### Port Forward OpenMetadata to view UI
-```
 
+Note: Always delete old PVC before re-deploying. Old volumes will break new pods.
 
+#### Port forward OpenMetadata to view UI
+```
+oc port-forward service/openmetadata 8585:http
 ```
 
 ##  OpenSearch Dockerfile and Use of GHCR
@@ -123,4 +119,4 @@ docker pull ghcr.io/bcgov/nr-openmetadata-opensearch:main
 ```
 
 ## Helm chart modifications
-To review all Helm chart modifications (i.e. differences between the OpenMetadata default config and this config), search this repo for "DF-NOTE:" annotations. 
+To review all Helm chart modifications (i.e. differences between the OpenMetadata default config and this config), search this repo for "DF-NOTE:" annotations.
diff --git a/charts/deps/Chart.yaml b/charts/deps/Chart.yaml
@@ -16,13 +16,13 @@ type: application
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
-version: 1.2.8
+version: 1.3.1
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.2.5"
+appVersion: "1.3.1"
 
 home: https://open-metadata.org/
 

diff --git a/charts/deps/charts/airflow-8.8.0/charts/postgresql/values.yaml b/charts/deps/charts/airflow-8.8.0/charts/postgresql/values.yaml
@@ -72,7 +72,7 @@ volumePermissions:
   ## pod securityContext.enabled=false and shmVolume.chmod.enabled=false
   ##
   securityContext:
-    runAsUser: 0
+    # runAsUser: 0 # DF-NOTE: disabled to allow for random UID
 
 ## Use an alternate scheduler, e.g. "stork".
 ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
@@ -84,9 +84,9 @@ volumePermissions:
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
 ##
 securityContext:
-  enabled: true
-  fsGroup: 1001
-  runAsUser: 1001
+  enabled: false # DF-NOTE: disabled to allow for random UID
+  # fsGroup: 1001
+  # runAsUser: 1001
 
 ## Pod Service Account
 ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
@@ -307,7 +307,7 @@ persistence:
   # storageClass: "-"
   accessModes:
     - ReadWriteOnce
-  size: 8Gi
+  size: 4Gi # DF-NOTE: previously 8Gi
   annotations: {}
 
 ## updateStrategy for PostgreSQL StatefulSet and its slaves StatefulSets
@@ -506,7 +506,7 @@ metrics:
   ##
   securityContext:
     enabled: false
-    runAsUser: 1001
+    # runAsUser: 1001 # DF-NOTE: disabled to allow for random UID
   ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-probes/#configure-probes)
   ## Configure extra options for liveness and readiness probes
   livenessProbe:

diff --git a/charts/deps/charts/airflow-8.8.0/values.yaml b/charts/deps/charts/airflow-8.8.0/values.yaml
@@ -14,7 +14,7 @@ airflow:
     pullPolicy: IfNotPresent
     pullSecret: ""
 
-    # Abi this causes the error: runAsUser: Invalid value: 50000
+    # DF-NOTE: this causes the error: runAsUser: Invalid value: 50000
     # uid: 50000
 
     gid: 0
@@ -23,7 +23,7 @@ airflow:
   ## - allowed values: "CeleryExecutor", "KubernetesExecutor", "CeleryKubernetesExecutor"
   ## - customize the "KubernetesExecutor" pod-template with `airflow.kubernetesPodTemplate.*`
   ##
-  executor: CeleryExecutor
+  executor: KubernetesExecutor # DF-NOTE: previously CeleryExecutor
 
   ## the fernet encryption key (sets `AIRFLOW__CORE__FERNET_KEY`)
   ## - [WARNING] you must change this value to ensure the security of your airflow
@@ -309,7 +309,14 @@ airflow:
     ## - spec for ResourceRequirements:
     ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
     ##
-    resources: {}
+    resources: # DF-NOTE: previously not enabled
+      requests:
+        cpu: 10m
+        memory: 256Mi
+      limits:
+        cpu: 200m
+        memory: 500Mi
+
 
     ## the nodeSelector configs for the Pod template
     ## - docs for nodeSelector:
@@ -401,7 +408,13 @@ airflow:
     ## - spec for ResourceRequirements:
     ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
     ##
-    resources: {}
+    resources: # DF-NOTE: previously not enabled
+      requests:
+        cpu: 50m
+        memory: 250Mi
+      limits:
+        cpu: 300m
+        memory: 800Mi
 
     ## the nodeSelector configs for the db-migrations Pods
     ## - docs for nodeSelector:
@@ -462,7 +475,13 @@ airflow:
     ## - spec for ResourceRequirements:
     ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
     ##
-    resources: {}
+    resources: # DF-NOTE: previously not enabled
+      requests:
+        cpu: 50m
+        memory: 250Mi
+      limits:
+        cpu: 300m
+        memory: 800Mi
 
     ## the nodeSelector configs for the sync Pods
     ## - docs for nodeSelector:
@@ -521,7 +540,13 @@ scheduler:
   ## - spec of ResourceRequirements:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
   ##
-  resources: {}
+  resources: # DF-NOTE: previously not enabled
+    requests:
+      cpu: 100m
+      memory: 250Mi
+    limits:
+      cpu: 500m
+      memory: 2Gi
 
   ## the nodeSelector configs for the scheduler Pods
   ## - docs for nodeSelector:
@@ -601,7 +626,13 @@ scheduler:
     ## - spec of ResourceRequirements:
     ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
     ##
-    resources: {}
+    resources: # DF-NOTE: previously not enabled
+      requests:
+        cpu: 10m
+        memory: 250Mi
+      limits:
+        cpu: 100m
+        memory: 2560Mi
 
     ## the number of minutes to retain log files (by last-modified time)
     ##
@@ -720,8 +751,13 @@ web:
   ## - spec for ResourceRequirements:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
   ##
-  resources: {}
-
+  resources: # DF-NOTE: previously not enabled
+      requests:
+        cpu: 100m
+        memory: 250Mi
+      limits:
+        cpu: 500m
+        memory: 3Gi
   ## the nodeSelector configs for the web Pods
   ## - docs for nodeSelector:
   ##   https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector
@@ -1047,7 +1083,13 @@ triggerer:
   ## - spec for ResourceRequirements:
   ##   https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.20/#resourcerequirements-v1-core
   ##
-  resources: {}
+  resources: # DF-NOTE: previously not enabled
+    requests:
+      cpu: 50m
+      memory: 250Mi
+    limits:
+      cpu: 500m
+      memory:  1536Mi
 
   ## the nodeSelector configs for the triggerer Pods
   ## - docs for nodeSelector:
@@ -1325,7 +1367,7 @@ logs:
 
     ## the size of PVC to request
     ##
-    size: 1Gi
+    size: 600Mi # DF-NOTE: previously 1Gi
 
 ###################################
 ## CONFIG | Airflow DAGs
@@ -1363,7 +1405,7 @@ dags:
 
     ## the size of PVC to request
     ##
-    size: 1Gi
+    size: 50Mi # DF-NOTE: previously 1Gi
 
   ## configs for the git-sync sidecar (https://github.com/kubernetes/git-sync)
   ##
@@ -1707,12 +1749,12 @@ extraManifests: []
 pgbouncer:
   ## if the pgbouncer Deployment is created
   ##
-  enabled: true
+  enabled: false # DF-NOTE: disabled, as Airflow and OM share a PostgreSQL database
 
   ## configs for the pgbouncer container image
   ##
   image:
-    repository: ghcr.io/airflow-helm/pgbouncer
+    repository: ghcr.io/airflow-helm/pgbouncer # DF-NOTE: image: image-registry.openshift-image-registry.svc:5000/a9cb09-dev/pgbouncer
     tag: 1.18.0-patch.1
     pullPolicy: IfNotPresent
     uid: 1001
@@ -1949,7 +1991,7 @@ postgresql:
 
     ## the size of PVC to request
     ##
-    size: 8Gi
+    size: 500Mi # DF-NOTE: previously 8Gi
 
   ## configs for the postgres StatefulSet
   ##
@@ -2042,7 +2084,7 @@ redis:
   ## - set to `false` if `airflow.executor` is `KubernetesExecutor`
   ## - set to `false` if using `externalRedis.*`
   ##
-  enabled: true
+  enabled: false # DF-NOTE: not needed for KubernetesExecutor
 
   ## configs for the redis container image
   ##
@@ -2125,7 +2167,7 @@ redis:
 
       ## the size of PVC to request
       ##
-      size: 8Gi
+      size: 500Mi # DF-NOTE: previously 8Gi
 
   ## configs for the redis slave StatefulSet
   ## - only used if `redis.cluster.enabled` is `true`
@@ -2178,7 +2220,7 @@ redis:
 
       ## the size of PVC to request
       ##
-      size: 8Gi
+      size: 500Mi # DF-NOTE: previously 8Gi
 
 ###################################
 ## DATABASE | External Redis