diff --git a/charts/harmony-chart/Chart.lock b/charts/harmony-chart/Chart.lock index 29540cd..9e609b7 100644 --- a/charts/harmony-chart/Chart.lock +++ b/charts/harmony-chart/Chart.lock @@ -32,5 +32,8 @@ dependencies: - name: openfaas repository: https://openfaas.github.io/faas-netes version: 14.2.34 -digest: sha256:42e4f43f0c7fc38842729cf24c72c7e540b5149d49b1bb8899612b53e69dc54f -generated: "2024-11-28T14:44:09.451667993-05:00" +- name: vector + repository: https://helm.vector.dev + version: 0.37.0 +digest: sha256:6620a3f1bac1f49165da56dc845b6435d73fd3d6c83e2ff4876e2d5c7394ae7b +generated: "2024-12-08T13:24:03.513574144-05:00" diff --git a/charts/harmony-chart/Chart.yaml b/charts/harmony-chart/Chart.yaml index 265a392..d5b0c13 100644 --- a/charts/harmony-chart/Chart.yaml +++ b/charts/harmony-chart/Chart.yaml @@ -5,7 +5,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes to the chart and its # templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.8.0 +version: 0.9.0 # This is the version number of the application being deployed. This version number should be incremented each time you # make changes to the application. Versions are not expected to follow Semantic Versioning. They should reflect the # version the application is using. It is recommended to use it with quotes. @@ -74,3 +74,8 @@ dependencies: version: "14.2.34" repository: https://openfaas.github.io/faas-netes condition: openfaas.enabled + +- name: vector + version: 0.37.0 + repository: https://helm.vector.dev + condition: vector.enabled diff --git a/charts/harmony-chart/values.yaml b/charts/harmony-chart/values.yaml index 0734cd2..ab26460 100644 --- a/charts/harmony-chart/values.yaml +++ b/charts/harmony-chart/values.yaml @@ -364,3 +364,125 @@ velero: openfaas: enabled: false + +vector: + enabled: false + role: "Agent" + podDisruptionBudget: # Optional, but recommended + enabled: true + minAvailable: 1 + # Configures a PodMonitor CRD used by the Prometheus operator + podMonitor: + enabled: false + tolerations: + - operator: Exists + logLevel: "info" + customConfig: + data_dir: /vector-data-dir + api: + enabled: false + address: 0.0.0.0:8686 + playground: false + + sources: + kubernetes_tutor_logs: + type: kubernetes_logs + extra_namespace_label_selector: app.kubernetes.io/managed-by=tutor + kubernetes_global_logs: + type: kubernetes_logs + extra_namespace_label_selector: app.kubernetes.io/managed-by!=tutor + + transforms: + + # Filter out application and global logs whose message is empty to prevent Vector process crash when sending logs to Cloudwatch + # More details in https://github.com/vectordotdev/vector/issues/15539 + openedx_logs: + type: remap + inputs: + - kubernetes_tutor_logs + source: |- + if !includes(["lms", "cms", "cms-worker", "lms-worker", "lms-job", "cms-job"], .kubernetes.pod_labels."app.kubernetes.io/name"){ + abort + } + if contains(string!(.message), "[tracking]") { + abort + } + .type = "application" + drop_on_abort: true + drop_on_error: true + + # Group multiline logs for better observabitlity + grouped_openedx_logs: + type: reduce + merge_strategies: + message: concat_newline + inputs: + - openedx_logs + starts_when: + type: "vrl" + source: |- + match(string!(.message), r'^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}.*') + operation_openedx_logs: + type: remap + inputs: + - kubernetes_tutor_logs + source: |- + if includes(["lms", "cms", "cms-worker", "lms-worker", "lms-job", "cms-job"], .kubernetes.pod_labels."app.kubernetes.io/name"){ + abort + } + .type = "application" + drop_on_abort: true + drop_on_error: true + global_logs: + type: filter + inputs: + - kubernetes_global_logs + condition: 'includes(["ingress-nginx"], .kubernetes.pod_labels."app.kubernetes.io/name")' + typed_global_logs: + type: remap + inputs: + - global_logs + source: |- + .type = "global" + drop_on_error: true + drop_on_abort: true + # Appplication logs (OpenedX, ingress-nginx, cert-manager) can be send to cloudwatch + # or to s3. It will depend on user needs. + application_logs: + type: remap + inputs: + - grouped_openedx_logs + - operation_openedx_logs + - typed_global_logs + source: |- + if is_empty(string!(.message)) { + log("Events with empty message are discarded", level: "info") + abort + } + # Extract tracking logs from Open edX applications + tracking_logs: + type: remap + inputs: + - kubernetes_tutor_logs + source: |- + parsed, err_regex = parse_regex(.message, r'^.* \[tracking\] [^{}]* (?P\{.*\})$') + if err_regex != null { + abort + } + message = parsed.tracking_message + parsed_json, err_json = parse_json(parsed.tracking_message) + if err_json != null { + log("Unable to parse JSON from tracking log message: " + err_json, level: "info") + abort + } + time, err_timestamp = parse_timestamp(parsed_json.time, "%+") + if err_timestamp != null { + log("Unable to parse timestamp from tracking log message: " + err_timestamp, level: "info") + abort + } + .time = time + .message = message + .type = "tracking" + + # Make sure to check out values-example.yml to now how to sink logs to S3, CloudWatch and other services + sinks: {} diff --git a/values-example.yaml b/values-example.yaml index 572c71d..b4fe9ba 100644 --- a/values-example.yaml +++ b/values-example.yaml @@ -89,3 +89,103 @@ velero: openfaas: enabled: false + +# ClickHouse Vector Sink + +vector: + enabled: false + customConfig: + transforms: + # Events should be separated per namespace, and a different sink should be + # implemented for every namespace with Aspects + logs_openedx_demo: + type: filter + inputs: + - kubernetes_tutor_logs + condition: '.kubernetes.pod_namespace == "openedx_demo"' # Mkae sure to update the namespace + + xapi_openedx_demo: + type: remap + inputs: + - logs_openedx_demo + drop_on_error: true + drop_on_abort: true + source: |- + parsed, err_regex = parse_regex(.message, r'^.* \[xapi_tracking\] [^{}]* + (?P\{.*\})$') + if err_regex != null { + abort + } + message, err = strip_whitespace(parsed.tracking_message) + parsed_json, err_json = parse_json(parsed.tracking_message) + if err_json != null { + log("Unable to parse JSON from xapi tracking log message: " + err_json, level: "error") + abort + } + time, err_timestamp = parse_timestamp(parsed_json.timestamp, "%+") + if err_timestamp != null { + log("Unable to parse timestamp from tracking log 'time' field: " + err_timestamp, level: "warn") + time, err_timestamp = parse_timestamp(parsed_json.timestamp, "%+") + if err_timestamp != null { + log("Unable to parse timestamp from tracking log 'timestamp' field: " + err_timestamp, level: "error") + abort + } + } + event_id = parsed_json.id + . = {"event_id": event_id, "emission_time": format_timestamp!(time, + format: "%+"), "event": encode_json(parsed_json)} + + sinks: + # Example ClickHouse Sink + clickhouse_openedx_demo: + type: clickhouse + auth: + strategy: basic + user: 'ch_vector' + password: 'password' + encoding: + timestamp_format: unix + date_time_best_effort: true + inputs: + - xapi_openedx_demo + # http://{{CLICKHOUSE_HOST }}.{{CLICKHOUSE_NAMESPACE}}:{{ CLICKHOUSE_INTERNAL_HTTP_PORT }} + endpoint: http://clickhouse-clickhouse.openedx-harmony:8123 + # ASPECTS_VECTOR_DATABASE + database: 'openedx' + table: 'xapi_events_all' + healthcheck: true + + tracking_logs_to_s3: + type: aws_s3 + inputs: + - tracking_logs + filename_append_uuid: true + filename_time_format: "log-%Y%m%d-%H" + # Helm tries to render the .type and .kubernetes variables. We need to escape them to avoid errors + # See> https://github.com/helm/helm/issues/2798 + key_prefix: {{ `{{ .kubernetes.pod_namespace }}/{{ .type }}/{{ .kubernetes.container_name }}/date=%F/` }} + compression: gzip + encoding: + codec: text + bucket: "set_me" + auth: + access_key_id: "set_me" + secret_access_key: "set_me" + region: "set_me" + # When using AWS-compatible services like MinIO, set the endpoint and tweak SSL if necessary + # endpoint: "http://minio.{namespace}:9000" + # region: none + healthcheck: + enabled: false + + logs_to_cloudwatch: + type: aws_cloudwatch + inputs: + - application_logs + group_name: my-cluster + stream_name: {{ `{{ .kubernetes.pod_namespace }}/{{ .kubernetes.container_name }}` }} + auth: + access_key_id: "set_me" + secret_access_key: "set_me" + encoding: + codec: json