Merge remote-tracking branch 'origin/20-vre-catalog-prototype' into 2…

…0-django-api-token # Conflicts: # README.md # vreapis/data_products/fixtures/data_products.yaml # vreapis/workflows/views.py
QCDIS · Sep 6, 2023 · 6fc18ed · 6fc18ed
2 parents 75d8dc3 + 916ef4d
commit 6fc18ed
Show file tree

Hide file tree

Showing 35 changed files with 1,916 additions and 1,369 deletions.
diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml
@@ -34,7 +34,7 @@ jobs:
     - uses: actions/checkout@v3
     - name: Build docker
       run: |
-          cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG
+          cd $DOCKER_FOLDER && docker build . --file Dockerfile -t $TAG --build-arg "NODE_ENV=${{ inputs.environment }}"
 
     - name: Login to github Registry
       if: ${{ inputs.push }}

diff --git a/.github/workflows/make-release.yml b/.github/workflows/make-release.yml
@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-build-container-vreapis:
-    uses: QCDIS/VREPaaS/.github/workflows/build-container.yml@main
+    uses: ./.github/workflows/build-container.yml
     with:
       docker_folder: vreapis
       tag: vreapi
@@ -16,10 +16,16 @@ jobs:
       docker_password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
   call-build-container-vre-panel:
-    uses: QCDIS/VREPaaS/.github/workflows/build-container.yml@main
+    uses: ./.github/workflows/build-container.yml
+    strategy:
+      matrix:
+        environment:
+          - production
+          - test
     with:
       docker_folder: vre-panel
       tag: vreapp
+      environment: ${{ matrix.environment }}
       push: true
     secrets:
       docker_username: ${{ secrets.DOCKERHUB_USERNAME }}

diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml
@@ -21,9 +21,8 @@ jobs:
     strategy:
       matrix:
         environment:
-          - development
-          - staging
           - production
+          - test
     with:
       docker_folder: vre-panel
       tag: vreapp

diff --git a/.gitignore b/.gitignore
@@ -138,6 +138,9 @@ GitHub.sublime-settings
 !.vscode/extensions.json 
 .history
 
+# Pycharm
+.idea/
+
 # Logs
 logs
 *.log
@@ -178,6 +181,4 @@ jspm_packages
 .next
 /package-lock.json
 
-tilt/secrets.yaml
-tilt/vre-panel-secrets.yaml
-tilt/django-secrets.yaml
+tilt/helm-values-secrets.yaml
diff --git a/README.md b/README.md
@@ -14,22 +14,49 @@ Close the terminal and start a new one to activate conda.
 
 Create and activate conda environment:
 ```shell
-conda create -n paas  python=3.9 
-conda activate paas
+conda env update --file environment.yaml
 ```
 
 #### Install tilt
 Install [tilt](https://docs.tilt.dev/install.html) via conda 
 
 ```shell
 conda install -c conda-forge tilt 
+```
+
+#### Install minikube
+
+```shell
 conda install -c conda-forge minikube 
 ```
 
+Follow step 3 of the [minikube ingress-dns setup guide](https://minikube.sigs.k8s.io/docs/handbook/addons/ingress-dns/).
+
+#### Add secrets
+
+Create `tilt/helm-values-secrets.yaml` and fill-in the following:
+
+```yaml
+global:
+  keycloak:
+    url:
+    realm:
+    client_id:
+    client_secret_key:
+
+  argo:
+    namespace:
+    url:
+    token:
+```
+
 #### Start Cluster
 
 ```shell
 minikube start
+minikube addons enable ingress
+minikube addons enable ingress-dns
+minikube dashboard  # optional
 ```
 
 #### Run tilt
@@ -80,13 +107,15 @@ rules:
       - get
       - watch
       - patch
+      - delete
     apiGroups:
       - ''
     resources:
       - pods
   - verbs:
       - get
       - watch
+      - patch
     apiGroups:
       - ''
     resources:
@@ -111,6 +140,7 @@ rules:
       - workflowartifactgctasks
       - workflowtemplates
       - workflows
+      - cronworkflows
   - verbs:
       - patch
     apiGroups:
@@ -181,6 +211,27 @@ echo -n $ARGO_TOKEN | base64 -w 0
 ```
 
 
+# Authorization 
+
+## Token 
+
+1. Create a user in the Django admin panel
+2. Create a token for the user in the Django admin panel
+3. Use the token in the header of the request
+
+```python
+    resp = requests.get(
+        f"{api_endpoint}/api/workflows/",
+        headers={
+            'Authorization': 'Token '+ naavre_api_token
+        }
+    )
+```
+
+# Releases
+If we want to add a new release environment we need to add a new .env.{ENV_NAME} together with a new line in the matrix on the .workflows/make.yaml and .workflows/make-release.yaml  
+
+
 # Install GitGuardian pre-commit hook
 
 ```

diff --git a/Tiltfile b/Tiltfile
@@ -1,7 +1,15 @@
 version_settings(constraint='>=0.22.2')
 secret_settings (disable_scrub=True)
-
-# API
+load('ext://helm_remote', 'helm_remote')
+
+helm_remote(
+    'vrepaas',
+    repo_name='oci://ghcr.io/qcdis/charts',
+    values=[
+        './tilt/helm-values-dev.yaml',
+        './tilt/helm-values-secrets.yaml',
+    ],
+)
 
 docker_build(
     'qcdis/vreapi',
@@ -13,23 +21,9 @@ docker_build(
         run('cd /app && /opt/venv/bin/python manage.py makemigrations'),
         run('cd /app && /opt/venv/bin/python manage.py migrate'),
         run('cd /app && /opt/venv/bin/pip install -r requirements.txt', trigger='./vreapis/requirements.txt'),
-    ]
-)
-
-k8s_yaml(['tilt/vreapis.yaml','tilt/django-secrets.yaml','tilt/vre-api-config.yaml'])
-
-k8s_resource(
-    'vreapi-deployment',
-    port_forwards='8000:8000',
-    labels=['vreapi'],
-    links=[
-        'http://localhost:8000/paas/api/api/',
-        'http://localhost:8000/paas/api/admin/',
-    ]
+    ],
 )
 
-# Panel
-
 docker_build(
     'qcdis/vreapp',
     context='.',
@@ -38,25 +32,20 @@ docker_build(
     live_update=[
         sync('./vre-panel', '/app'),
         run('cd /app && npm install', trigger=['./vre-panel/package.json'])
-    ]
+    ],
 )
 
-k8s_yaml(['tilt/vre-panel.yaml','tilt/vre-panel-secrets.yaml'])
-
 k8s_resource(
-    'vreapp-deployment',
-    port_forwards='3000:3000',
-    labels=['vreapp'],
+    'vrepaas-vreapi',
     links=[
-        'http://localhost:3000/paas/app/',
-    ]
+        'https://paas.minikube.test/vre-api-test/api/',
+        'https://paas.minikube.test/vre-api-test/admin/',
+    ],
 )
 
-# DB
-
-k8s_yaml([ 'tilt/vre-depts-db.yaml'])
-
 k8s_resource(
-    'db-deployment',
-    labels=['db'],
-)
+    'vrepaas-vreapp',
+    links=[
+        'https://paas.minikube.test/vreapp/',
+    ],
+)
diff --git a/environment.yaml b/environment.yaml
@@ -0,0 +1,22 @@
+name: paas
+channels:
+  - conda-forge
+dependencies:
+  - python>=3.9
+  - djangorestframework
+  - tilt
+  - django>=4.0.0,<4.1.0
+  - minikube
+  - requests
+  - whitenoise
+  - django-keycloak-auth
+  - gunicorn
+  - pyyaml
+  - python-dotenv
+  - django-cors-headers
+  - django-extensions
+
+
+
+
+
diff --git a/k8s/test-vreapis/deployment.yaml b/k8s/test-vreapis/deployment.yaml
@@ -23,6 +23,18 @@ spec:
           ports:
           - containerPort: 8000
             protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: "/vre-api-test/api/"
+              port: 8000
+            initialDelaySeconds: 60
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: "/vre-api-test/api/"
+              port: 8000
+            initialDelaySeconds: 60
+            periodSeconds: 10
           env:
             - name: KEYCLOAK_CLIENT_ID
               valueFrom:
@@ -137,3 +149,9 @@ spec:
                 configMapKeyRef:
                   name: vre-api-config
                   key: DB_NAME
+
+            - name: BASE_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: vre-api-config
+                  key: BASE_PATH
diff --git a/k8s/test-vreapis/ingress.yaml b/k8s/test-vreapis/ingress.yaml
@@ -2,18 +2,8 @@ kind: Ingress
 apiVersion: networking.k8s.io/v1
 metadata:
   name: vre-paas
-  annotations:
-    kubernetes.io/ingress.class: nginx
-    kubernetes.io/tls-acme: 'true'
-    nginx.ingress.kubernetes.io/add-base-url: 'true'
-    nginx.ingress.kubernetes.io/backend-protocol: HTTP
-    nginx.ingress.kubernetes.io/force-ssl-redirect: 'false'
-    nginx.ingress.kubernetes.io/proxy-body-size: 600m
-    nginx.ingress.kubernetes.io/rewrite-target: /$2
-    nginx.org/client-max-body-size: 600m
-    nginx.ingress.kubernetes.io/proxy-buffers-number: "8"
-    nginx.ingress.kubernetes.io/proxy-buffer-size: "32k"
 spec:
+  ingressClassName: nginx
   tls:
     - hosts:
         - lifewatch.lab.uvalight.net
@@ -22,17 +12,10 @@ spec:
     - host: lifewatch.lab.uvalight.net
       http:
         paths:
-          - path: /vre-api(/|$)(.*)
+          - path: /vre-api-test
             pathType: Prefix
             backend:
               service:
                 name: vreapi-svc
                 port:
-                  number: 8000
-          - path: /vreapp(/|$)(.*)
-            pathType: Prefix
-            backend:
-              service:
-                name: vreapp-svc
-                port:
-                  number: 3000
+                  number: 8000
diff --git a/k8s/test-vreapis/vre-api-config.yaml b/k8s/test-vreapis/vre-api-config.yaml
@@ -12,7 +12,7 @@ data:
   CALL_BACK_URL: https://lifewatch.lab.uvalight.net:32443/vre-api/
   ARGO_URL: https://lifewatch.lab.uvalight.net:32443/argowf
   ARGO_NAMESPACE: argo
-  BASE_PATH: /vre-api
+  BASE_PATH: /vre-api-test
   DB_USER: vreapi
   DB_HOST: db-service
   DB_PORT: '5432'

diff --git a/k8s/vreapis/deployment.yaml b/k8s/vreapis/deployment.yaml
@@ -100,6 +100,12 @@ spec:
                   name: vre-api-config
                   key: ALLOWED_HOST
 
+            - name: URL_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: vre-api-config
+                  key: URL_PATH
+
             - name: ARGO_NAMESPACE
               valueFrom:
                 configMapKeyRef:
@@ -135,3 +141,9 @@ spec:
                 configMapKeyRef:
                   name: vre-api-config
                   key: DB_NAME
+
+            - name: BASE_PATH
+              valueFrom:
+                configMapKeyRef:
+                  name: vre-api-config
+                  key: BASE_PATH