Support deploying only LogForwarding

.cruft.json

@@ -1,13 +1,13 @@
 {
   "template": "https://github.com/projectsyn/commodore-component-template.git",
-  "commit": "ea12efff947bce80cf31a3f1ed4412eab40e8b33",
+  "commit": "689cbccb0e39543542eee6d308eb652fd6eb1344",
   "checkout": "main",
   "context": {
     "cookiecutter": {
       "name": "OpenShift4 Logging",
       "slug": "openshift4-logging",
       "parameter_key": "openshift4_logging",
-      "test_cases": "defaults master elasticsearch multilineerr",
+      "test_cases": "defaults master elasticsearch multilineerr forwardingonly",
       "add_lib": "n",
       "add_pp": "n",
       "add_golden": "y",

.github/workflows/test.yaml

@@ -37,6 +37,7 @@ jobs:
           - master
           - elasticsearch
           - multilineerr
+          - forwardingonly
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}
@@ -55,6 +56,7 @@ jobs:
           - master
           - elasticsearch
           - multilineerr
+          - forwardingonly
     defaults:
       run:
         working-directory: ${{ env.COMPONENT_NAME }}

Makefile.vars.mk

@@ -57,4 +57,4 @@ KUBENT_IMAGE    ?= ghcr.io/doitintl/kube-no-trouble:latest
 KUBENT_DOCKER   ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
 
 instance ?= defaults
-test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml
+test_instances = tests/defaults.yml tests/master.yml tests/elasticsearch.yml tests/multilineerr.yml tests/forwardingonly.yml

component/main.jsonnet

@@ -9,6 +9,8 @@ local params = inv.parameters.openshift4_logging;
 local group = 'operators.coreos.com/';
 local clusterLoggingGroupVersion = 'logging.openshift.io/v1';
 
+local forwardingOnly = !params.components.elasticsearch.enabled && !params.components.lokistack.enabled;
+
 local namespace_groups = (
   if std.objectHas(params.clusterLogForwarding, 'namespaces') then
     {
@@ -35,15 +37,26 @@ local legacyCollectionPatch = if std.length(legacyCollectionConfig) > 0 then std
   }
 ) else {};
 
-local clusterLogging = params.clusterLogging {
-  collection: {
-    [it]: params.clusterLogging.collection[it]
-    for it in std.objectFields(params.clusterLogging.collection)
-    if it != 'logs'
-  },
-} + legacyCollectionPatch;
+local clusterLogging = std.mergePatch(
+  params.clusterLogging {
+    collection: {
+      [it]: params.clusterLogging.collection[it]
+      for it in std.objectFields(params.clusterLogging.collection)
+      if it != 'logs'
+    },
+  } + legacyCollectionPatch,
+  {
+    // Patch to remove certain keys, as the ClusterLogging operator would just
+    // deploy elasticsearch or kibana if they are configured
+    [if forwardingOnly then 'logStore']: null,
+  }
+);
 // --- End patch
 
+local pipelineOutputRefs(pipeline) =
+  local default = if forwardingOnly then [] else [ 'default' ];
+  std.get(pipeline, 'forwarders', []) + default;
+
 {
   '00_namespace': kube.Namespace(params.namespace) {
     metadata+: {
@@ -166,41 +179,44 @@ local clusterLogging = params.clusterLogging {
         for group in std.objectFields(namespace_groups)
       ],
     } + com.makeMergeable(
+      local enable_pipeline = std.length(pipelineOutputRefs(params.clusterLogForwarding.application_logs)) > 0;
       local enable_json = com.getValueOrDefault(params.clusterLogForwarding.application_logs, 'json', false);
       local enable_multilineErrors = com.getValueOrDefault(params.clusterLogForwarding.application_logs, 'detectMultilineErrors', false);
       {
-        pipelines: [
+        [if enable_pipeline then 'pipelines']: [
           {
             name: 'application-logs',
             inputRefs: [ 'application' ],
-            outputRefs: com.getValueOrDefault(params.clusterLogForwarding.application_logs, 'forwarders', []) + [ 'default' ],
+            outputRefs: pipelineOutputRefs(params.clusterLogForwarding.application_logs),
             [if enable_json then 'parse']: 'json',
             [if enable_multilineErrors then 'detectMultilineErrors']: true,
           },
         ],
       }
     ) + com.makeMergeable(
+      local enable_pipeline = params.clusterLogForwarding.infrastructure_logs.enabled && std.length(pipelineOutputRefs(params.clusterLogForwarding.infrastructure_logs)) > 0;
       local enable_json = com.getValueOrDefault(params.clusterLogForwarding.infrastructure_logs, 'json', false);
       local enable_multilineErrors = com.getValueOrDefault(params.clusterLogForwarding.infrastructure_logs, 'detectMultilineErrors', false);
       {
-        [if params.clusterLogForwarding.infrastructure_logs.enabled then 'pipelines']: [
+        [if enable_pipeline then 'pipelines']: [
           {
             name: 'infrastructure-logs',
             inputRefs: [ 'infrastructure' ],
-            outputRefs: com.getValueOrDefault(params.clusterLogForwarding.infrastructure_logs, 'forwarders', []) + [ 'default' ],
+            outputRefs: pipelineOutputRefs(params.clusterLogForwarding.infrastructure_logs),
             [if enable_json then 'parse']: 'json',
             [if enable_multilineErrors then 'detectMultilineErrors']: true,
           },
         ],
       }
     ) + com.makeMergeable(
+      local enable_pipeline = params.clusterLogForwarding.audit_logs.enabled && std.length(pipelineOutputRefs(params.clusterLogForwarding.application_logs)) > 0;
       local enable_json = com.getValueOrDefault(params.clusterLogForwarding.audit_logs, 'json', false);
       {
         [if params.clusterLogForwarding.audit_logs.enabled then 'pipelines']: [
           {
             name: 'audit-logs',
             inputRefs: [ 'audit' ],
-            outputRefs: com.getValueOrDefault(params.clusterLogForwarding.audit_logs, 'forwarders', []) + [ 'default' ],
+            outputRefs: pipelineOutputRefs(params.clusterLogForwarding.audit_logs),
           },
         ],
       }

tests/forwardingonly.yml

@@ -0,0 +1,33 @@
+applications:
+  - openshift4-operators as openshift-operators-redhat
+  - openshift4-monitoring
+
+parameters:
+  kapitan:
+    dependencies:
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-operators/v1.0.2/lib/openshift4-operators.libsonnet
+        output_path: vendor/lib/openshift4-operators.libsonnet
+      - type: https
+        source: https://raw.githubusercontent.com/appuio/component-openshift4-monitoring/v2.9.0/lib/openshift4-monitoring-alert-patching.libsonnet
+        output_path: vendor/lib/alert-patching.libsonnet
+    compile:
+      - input_type: jsonnet
+        input_paths:
+          - tests/console-patch.jsonnet
+        output_path: console-patching/
+
+  openshift4_operators:
+    defaultInstallPlanApproval: Automatic
+    defaultSource: openshift-operators-redhat
+    defaultSourceNamespace: openshift-operators-redhat
+
+  openshift4_logging:
+    components:
+      lokistack:
+        enabled: false
+      elasticsearch:
+        enabled: false
+
+    clusterLogForwarding:
+      enabled: true

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/00_namespace.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  annotations:
+    openshift.io/node-selector: ''
+  labels:
+    name: openshift-logging
+    openshift.io/cluster-monitoring: 'true'
+  name: openshift-logging

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/10_operator_group.yaml

@@ -0,0 +1,11 @@
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  annotations: {}
+  labels:
+    name: cluster-logging
+  name: cluster-logging
+  namespace: openshift-logging
+spec:
+  targetNamespaces:
+    - openshift-logging

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/20_subscriptions.yaml

@@ -0,0 +1,21 @@
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  annotations: {}
+  labels:
+    name: cluster-logging
+  name: cluster-logging
+  namespace: openshift-logging
+spec:
+  channel: stable-5.8
+  config:
+    resources:
+      limits:
+        memory: 256Mi
+      requests:
+        cpu: 10m
+        memory: 128Mi
+  installPlanApproval: Automatic
+  name: cluster-logging
+  source: redhat-operators
+  sourceNamespace: openshift-operators-redhat

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/30_cluster_logging.yaml

@@ -0,0 +1,14 @@
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogging
+metadata:
+  annotations:
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+  labels:
+    name: instance
+  name: instance
+  namespace: openshift-logging
+spec:
+  collection:
+    type: vector
+  logStore: {}
+  managementState: Managed

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/31_cluster_logforwarding.yaml

@@ -0,0 +1,9 @@
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogForwarder
+metadata:
+  annotations: {}
+  labels:
+    name: instance
+  name: instance
+  namespace: openshift-logging
+spec: {}

tests/golden/forwardingonly/openshift4-logging/openshift4-logging/60_collector_alerts.yaml

@@ -0,0 +1,61 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  annotations: {}
+  labels:
+    name: syn-collector-rules
+  name: syn-collector-rules
+  namespace: openshift-logging
+spec:
+  groups:
+    - name: logging_collector.alerts
+      rules:
+        - alert: SYN_CollectorNodeDown
+          annotations:
+            message: Prometheus could not scrape {{ $labels.namespace }}/{{ $labels.pod
+              }} collector component for more than 10m.
+            summary: Collector cannot be scraped
+          expr: |
+            up{app_kubernetes_io_component = "collector", app_kubernetes_io_part_of = "cluster-logging"} == 0
+          for: 10m
+          labels:
+            service: collector
+            severity: critical
+            syn: 'true'
+            syn_component: openshift4-logging
+        - alert: SYN_CollectorHighErrorRate
+          annotations:
+            message: '{{ $value }}% of records have resulted in an error by {{ $labels.namespace
+              }}/{{ $labels.pod }} collector component.'
+            summary: '{{ $labels.namespace }}/{{ $labels.pod }} collector component
+              errors are high'
+          expr: |
+            100 * (
+                collector:log_num_errors:sum_rate{app_kubernetes_io_part_of = "cluster-logging"}
+              /
+                collector:received_events:sum_rate{app_kubernetes_io_part_of = "cluster-logging"}
+              ) > 0.001
+          for: 15m
+          labels:
+            service: collector
+            severity: critical
+            syn: 'true'
+            syn_component: openshift4-logging
+        - alert: SYN_CollectorVeryHighErrorRate
+          annotations:
+            message: '{{ $value }}% of records have resulted in an error by {{ $labels.namespace
+              }}/{{ $labels.pod }} collector component.'
+            summary: '{{ $labels.namespace }}/{{ $labels.pod }} collector component
+              errors are very high'
+          expr: |
+            100 * (
+                collector:log_num_errors:sum_rate{app_kubernetes_io_part_of = "cluster-logging"}
+              /
+                collector:received_events:sum_rate{app_kubernetes_io_part_of = "cluster-logging"}
+              ) > 0.05
+          for: 15m
+          labels:
+            service: collector
+            severity: critical
+            syn: 'true'
+            syn_component: openshift4-logging