update kestra: #460

TrivadisPF · Jan 12, 2025 · 632f747 · 632f747
1 parent 5c6d1fe
commit 632f747
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 4 deletions.
diff --git a/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2 b/modern-data-platform-stack/generator-config/templates/docker-compose.yml.j2
@@ -11963,13 +11963,31 @@ services:
       com.platys.description: "Orchestration Engine"
       com.platys.webui.title: "Kestra UI"
       com.platys.webui.url: "http://dataplatform:28245"
+      com.platys.password.envvars: "PLATYS_KESTRA_PASSWORD"
     depends_on:
       postgresql:
         condition: service_started    
     ports:
       - "28245:8080"      
       - "28246:8081"      
     environment:
+      STORAGE_TYPE: {{KESTRA_storage_type}}
+  {%if KESTRA_storage_type == 'local' %}
+      STORAGE_LOCAL_BASEPATH: "/app/storage" 
+  {%elif KESTRA_storage_type == 's3' and external['S3_enable'] | default(false) %}
+      STORAGE_S3_REGION: {{s3DefaultRegion if s3DefaultRegion is defined and s3DefaultRegion else omit}} 
+      STORAGE_S3_ACCESS_KEY: {{KESTRA_bucket_name}}
+      STORAGE_S3_SECRET_KEY: ${PLATYS_AWS_SECRET_ACCESS_KEY:-{{MINIO_secret_key}}}
+      STORAGE_S3_BUCKET: {{KESTRA_bucket_name}}
+  {%elif KESTRA_storage_type == 'minio' and MINIO_enable | default(false) %}
+      STORAGE_MINIO_ENDPOINT: "{{minioEndpoint | default(omit) }}"
+      STORAGE_MINIO_PORT: 9000
+      STORAGE_MINIO_ACCESS_KEY: {{s3AccessKey | mandatory('s3AccessKey must be defined, if using minIO use the MINIO_access_key in config.yml') }}
+      STORAGE_MINIO_SECRET_KEY: ${PLATYS_AWS_SECRET_ACCESS_KEY:-{{MINIO_secret_key}}}
+      STORAGE_MINIO_REGION: {{s3DefaultRegion if s3DefaultRegion is defined and s3DefaultRegion else omit}}
+      STORAGE_MINIO_BUCKET: {{KESTRA_bucket_name}}
+      # STORAGE_MINIO_PARTSIZE: 
+  {% endif -%}   {#  KESTRA_storage_type #}      
       KESTRA_CONFIGURATION: |
         datasources:
           postgres:
@@ -11980,28 +11998,37 @@ services:
         kestra:
           server:
             basicAuth:
-              enabled: false
-              username: "admin@kestra.io" # it must be a valid email address
-              password: kestra
+              enabled: {{KESTRA_auth_enabled | default(false)}}
+              username: {{KESTRA_username if KESTRA_username is defined and KESTRA_username and KESTRA_username | length else 'kestra'}} # it must be a valid email address
+              password: ${PLATYS_KESTRA_PASSWORD:-{{KESTRA_password}}}
           repository:
             type: {{KESTRA_database_type}}
           storage:
-            type: local
+            type: {{KESTRA_storage_type}}            
             local:
               basePath: "/app/storage"
           queue:
             type: {{KESTRA_database_type}}
           tasks:
             tmpDir:
               path: /tmp/kestra-wd/tmp
+          tutorialFlows:
+            enabled: {{KESTRA_provision_tutorial_flows | default(false)}}    
+          variables:
+            envVarsPrefix: KESTRA_    
+              globals:                  
           url: http://${PUBLIC_IP}:28245/
     {%if use_timezone | default(false) %}
       TZ: {{use_timezone}}
     {% endif -%}   {#  use_timezone #}
     volumes:
       - ./data-transfer:/data-transfer
+    {% if KESTRA_volume_map_data | default(false) %}  
       - ./container-volume/kestra/data:/app/storage
+    {% endif %}  
+    {% if KESTRA_volume_map_temp | default(false) %}  
       - ./container-volume/kestra/tmp/kestra-wd:/tmp/kestra-wd
+    {% endif %}  
       - /var/run/docker.sock:/var/run/docker.sock
     {%if use_timezone | default(false) %}
       - "./etc/timezone:/etc/timezone:ro"

diff --git a/modern-data-platform-stack/generator-config/vars/config.yml b/modern-data-platform-stack/generator-config/vars/config.yml
@@ -1169,11 +1169,19 @@
       # ===== Kestra  ========
       #
       KESTRA_enable: false
+      KESTRA_volume_map_data: false
+      KESTRA_volume_map_temp: false
       # one of 'postgres'
       KESTRA_database_type: postgres
       KESTRA_database_user: postgres
       KESTRA_database_password: abc123!
       KESTRA_database_name: postgres
+      # the storage type, one of 'local', 's3', 'minio' 
+      KESTRA_storage_type: local
+      KESTRA_provision_tutorial_flows: false
+      KESTRA_auth_enabled: false
+      KESTRA_username: [email protected]
+      KESTRA_password: abc123!
 
       #
       # ===== Ofelia ========