Merge pull request #93 from eduNEXT/cag/vector

feat: add vector with defaults

charts/harmony-chart/Chart.lock

@@ -32,5 +32,8 @@ dependencies:
 - name: openfaas
   repository: https://openfaas.github.io/faas-netes
   version: 14.2.34
-digest: sha256:42e4f43f0c7fc38842729cf24c72c7e540b5149d49b1bb8899612b53e69dc54f
-generated: "2024-11-28T14:44:09.451667993-05:00"
+- name: vector
+  repository: https://helm.vector.dev
+  version: 0.37.0
+digest: sha256:6620a3f1bac1f49165da56dc845b6435d73fd3d6c83e2ff4876e2d5c7394ae7b
+generated: "2024-12-08T13:24:03.513574144-05:00"

charts/harmony-chart/Chart.yaml

@@ -5,7 +5,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes to the chart and its
 # templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.8.0
+version: 0.9.0
 # This is the version number of the application being deployed. This version number should be incremented each time you
 # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the
 # version the application is using. It is recommended to use it with quotes.
@@ -74,3 +74,8 @@ dependencies:
   version: "14.2.34"
   repository: https://openfaas.github.io/faas-netes
   condition: openfaas.enabled
+
+- name: vector
+  version:  0.37.0
+  repository: https://helm.vector.dev
+  condition: vector.enabled

charts/harmony-chart/values.yaml

@@ -364,3 +364,125 @@ velero:
 
 openfaas:
   enabled: false
+
+vector:
+  enabled: false
+  role: "Agent"
+  podDisruptionBudget: # Optional, but recommended
+    enabled: true
+    minAvailable: 1
+  # Configures a PodMonitor CRD used by the Prometheus operator
+  podMonitor:
+    enabled: false
+  tolerations:
+  - operator: Exists
+  logLevel: "info"
+  customConfig:
+    data_dir: /vector-data-dir
+    api:
+      enabled: false
+      address: 0.0.0.0:8686
+      playground: false
+
+    sources:
+      kubernetes_tutor_logs:
+        type: kubernetes_logs
+        extra_namespace_label_selector: app.kubernetes.io/managed-by=tutor
+      kubernetes_global_logs:
+        type: kubernetes_logs
+        extra_namespace_label_selector: app.kubernetes.io/managed-by!=tutor
+
+    transforms:
+
+      # Filter out application and global logs whose message is empty to prevent Vector process crash when sending logs to Cloudwatch
+      # More details in https://github.com/vectordotdev/vector/issues/15539
+      openedx_logs:
+        type: remap
+        inputs:
+        - kubernetes_tutor_logs
+        source: |-
+          if !includes(["lms", "cms", "cms-worker", "lms-worker", "lms-job", "cms-job"], .kubernetes.pod_labels."app.kubernetes.io/name"){
+            abort
+          }
+          if contains(string!(.message), "[tracking]") {
+            abort
+          }
+          .type = "application"
+        drop_on_abort: true
+        drop_on_error: true
+
+      # Group multiline logs for better observabitlity
+      grouped_openedx_logs:
+        type: reduce
+        merge_strategies:
+          message: concat_newline
+        inputs:
+          - openedx_logs
+        starts_when:
+          type: "vrl"
+          source: |-
+            match(string!(.message), r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*')
+      operation_openedx_logs:
+        type: remap
+        inputs:
+        - kubernetes_tutor_logs
+        source: |-
+          if includes(["lms", "cms", "cms-worker", "lms-worker", "lms-job", "cms-job"], .kubernetes.pod_labels."app.kubernetes.io/name"){
+            abort
+          }
+          .type = "application"
+        drop_on_abort: true
+        drop_on_error: true
+      global_logs:
+        type: filter
+        inputs:
+          - kubernetes_global_logs
+        condition: 'includes(["ingress-nginx"], .kubernetes.pod_labels."app.kubernetes.io/name")'
+      typed_global_logs:
+        type: remap
+        inputs:
+        - global_logs
+        source: |-
+          .type = "global"
+        drop_on_error: true
+        drop_on_abort: true
+      # Appplication logs (OpenedX, ingress-nginx, cert-manager) can be send to cloudwatch
+      # or to s3. It will depend on user needs.
+      application_logs:
+        type: remap
+        inputs:
+        - grouped_openedx_logs
+        - operation_openedx_logs
+        - typed_global_logs
+        source: |-
+          if is_empty(string!(.message)) {
+            log("Events with empty message are discarded", level: "info")
+            abort
+          }
+      # Extract tracking logs from Open edX applications
+      tracking_logs:
+        type: remap
+        inputs:
+        - kubernetes_tutor_logs
+        source: |-
+          parsed, err_regex = parse_regex(.message, r'^.* \[tracking\] [^{}]* (?P<tracking_message>\{.*\})$')
+          if err_regex != null {
+            abort
+          }
+          message = parsed.tracking_message
+          parsed_json, err_json = parse_json(parsed.tracking_message)
+          if err_json != null {
+            log("Unable to parse JSON from tracking log message: " + err_json, level: "info")
+            abort
+          }
+          time, err_timestamp = parse_timestamp(parsed_json.time, "%+")
+          if err_timestamp != null {
+            log("Unable to parse timestamp from tracking log message: " + err_timestamp, level: "info")
+            abort
+          }
+          .time = time
+          .message = message
+          .type = "tracking"
+
+    # Make sure to check out values-example.yml to now how to sink logs to S3, CloudWatch and other services
+    sinks: {}

values-example.yaml

@@ -89,3 +89,103 @@ velero:
 
 openfaas:
   enabled: false
+
+# ClickHouse Vector Sink
+
+vector:
+  enabled: false
+  customConfig:
+    transforms:
+      # Events should be separated per namespace, and a different sink should be
+      # implemented for every namespace with Aspects
+      logs_openedx_demo:
+        type: filter
+        inputs:
+        - kubernetes_tutor_logs
+        condition: '.kubernetes.pod_namespace == "openedx_demo"' # Mkae sure to update the namespace
+
+      xapi_openedx_demo:
+        type: remap
+        inputs:
+        - logs_openedx_demo
+        drop_on_error: true
+        drop_on_abort: true
+        source: |-
+          parsed, err_regex = parse_regex(.message, r'^.* \[xapi_tracking\] [^{}]*
+          (?P<tracking_message>\{.*\})$')
+          if err_regex != null {
+            abort
+          }
+          message, err = strip_whitespace(parsed.tracking_message)
+          parsed_json, err_json = parse_json(parsed.tracking_message)
+          if err_json != null {
+            log("Unable to parse JSON from xapi tracking log message: " + err_json, level: "error")
+            abort
+          }
+          time, err_timestamp = parse_timestamp(parsed_json.timestamp, "%+")
+          if err_timestamp != null {
+            log("Unable to parse timestamp from tracking log 'time' field: " + err_timestamp, level: "warn")
+            time, err_timestamp = parse_timestamp(parsed_json.timestamp, "%+")
+            if err_timestamp != null {
+              log("Unable to parse timestamp from tracking log 'timestamp' field: " + err_timestamp, level: "error")
+              abort
+            }
+          }
+          event_id = parsed_json.id
+          . = {"event_id": event_id, "emission_time": format_timestamp!(time,
+          format: "%+"), "event": encode_json(parsed_json)}
+
+    sinks:
+      # Example ClickHouse Sink
+      clickhouse_openedx_demo:
+        type: clickhouse
+        auth:
+          strategy: basic
+          user: 'ch_vector'
+          password: 'password'
+        encoding:
+          timestamp_format: unix
+        date_time_best_effort: true
+        inputs:
+          - xapi_openedx_demo
+        # http://{{CLICKHOUSE_HOST }}.{{CLICKHOUSE_NAMESPACE}}:{{ CLICKHOUSE_INTERNAL_HTTP_PORT }}
+        endpoint: http://clickhouse-clickhouse.openedx-harmony:8123
+        # ASPECTS_VECTOR_DATABASE
+        database: 'openedx'
+        table: 'xapi_events_all'
+        healthcheck: true
+
+      tracking_logs_to_s3:
+        type: aws_s3
+        inputs:
+        - tracking_logs
+        filename_append_uuid: true
+        filename_time_format: "log-%Y%m%d-%H"
+        # Helm tries to render the .type and .kubernetes variables. We need to escape them to avoid errors
+        # See> https://github.com/helm/helm/issues/2798
+        key_prefix: {{ `{{ .kubernetes.pod_namespace }}/{{ .type }}/{{ .kubernetes.container_name }}/date=%F/` }}
+        compression: gzip
+        encoding:
+          codec: text
+        bucket: "set_me"
+        auth:
+          access_key_id: "set_me"
+          secret_access_key: "set_me"
+        region: "set_me"
+        # When using AWS-compatible services like MinIO, set the endpoint and tweak SSL if necessary
+        # endpoint: "http://minio.{namespace}:9000"
+        # region: none
+        healthcheck:
+          enabled: false
+
+      logs_to_cloudwatch:
+        type: aws_cloudwatch
+        inputs:
+        - application_logs
+        group_name: my-cluster
+        stream_name: {{ `{{ .kubernetes.pod_namespace }}/{{ .kubernetes.container_name }}` }}
+        auth:
+          access_key_id: "set_me"
+          secret_access_key: "set_me"
+        encoding:
+          codec: json