Support multi-log-forwarder

Supports enabling namespaced ClusterLogForwarder resources.
Enabling the feature requires redeploying the Cluster Logging Operator.

class/defaults.yml

@@ -73,6 +73,9 @@ parameters:
     clusterLogging: {}
     clusterLogForwarder: {}
 
+    namespaceLogForwarder:
+      enabled: false
+
     operatorResources:
       clusterLogging:
         requests:

component/config_forwarding.libsonnet

@@ -1,6 +1,8 @@
 local com = import 'lib/commodore.libjsonnet';
 local kap = import 'lib/kapitan.libjsonnet';
+local kube = import 'lib/kube.libjsonnet';
 local lib = import 'lib/openshift4-logging.libsonnet';
+local utils = import 'utils.libsonnet';
 
 local inv = kap.inventory();
 local params = inv.parameters.openshift4_logging;
@@ -198,37 +200,104 @@ local clusterLogForwarderSpec = std.foldl(
   },
 ) + com.makeMergeable(params.clusterLogForwarder);
 
+// Unfold objects into array for ClusterLogForwarder resource.
+local unfoldSpecs(specs) = {
+  // Unfold objects into array.
+  [if std.length(specs.inputs) > 0 then 'inputs']: [
+    { name: name } + specs.inputs[name]
+    for name in std.objectFields(specs.inputs)
+  ],
+  [if std.length(specs.outputs) > 0 then 'outputs']: [
+    { name: name } + specs.outputs[name]
+    for name in std.objectFields(specs.outputs)
+  ],
+  [if std.length(specs.pipelines) > 0 then 'pipelines']: [
+    { name: name } + specs.pipelines[name]
+    for name in std.objectFields(specs.pipelines)
+  ],
+} + {
+  // Import remaining specs as is.
+  [key]: specs[key]
+  for key in std.objectFields(specs)
+  if !std.member([ 'inputs', 'outputs', 'pipelines' ], key)
+};
+
 // ClusterLogForwarder:
 // Create definitive ClusterLogForwarder resource from specs.
 local clusterLogForwarder = lib.ClusterLogForwarder(params.namespace, 'instance') {
-  spec: {
-    // Unfold objects into array.
-    [if std.length(clusterLogForwarderSpec.inputs) > 0 then 'inputs']: [
-      { name: name } + clusterLogForwarderSpec.inputs[name]
-      for name in std.objectFields(clusterLogForwarderSpec.inputs)
-    ],
-    [if std.length(clusterLogForwarderSpec.outputs) > 0 then 'outputs']: [
-      { name: name } + clusterLogForwarderSpec.outputs[name]
-      for name in std.objectFields(clusterLogForwarderSpec.outputs)
-    ],
-    [if std.length(clusterLogForwarderSpec.pipelines) > 0 then 'pipelines']: [
-      { name: name } + clusterLogForwarderSpec.pipelines[name]
-      for name in std.objectFields(clusterLogForwarderSpec.pipelines)
-    ],
-  } + {
-    // Import remaining specs as is.
-    [key]: clusterLogForwarderSpec[key]
-    for key in std.objectFields(clusterLogForwarderSpec)
-    if !std.member([ 'inputs', 'outputs', 'pipelines' ], key)
-  },
+  spec: unfoldSpecs(clusterLogForwarderSpec),
 };
 
+// namespaceLogForwarder:
+// Create namespaced LogForwarder resource from specs.
+local namespaceLogForwarder = [
+  local specs = { inputs: {}, outputs: {}, pipelines: {} } + com.makeMergeable(params.namespaceLogForwarder[forwarder]);
+  local name = utils.namespacedName(forwarder).name;
+  local namespace = utils.namespacedName(forwarder).namespace;
+  local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name);
+
+  lib.ClusterLogForwarder(namespace, name) {
+    spec: { serviceAccountName: serviceAccount } + com.makeMergeable(unfoldSpecs(specs)),
+  }
+  for forwarder in std.objectFields(params.namespaceLogForwarder)
+  if !std.member([ 'enabled', 'instance', 'openshift-logging/instance' ], forwarder)
+];
+
+// namespaceServiceAccount:
+// Create ServiceAccount for namespaced LogForwarder specs.
+local namespaceServiceAccount = [
+  local specs = params.namespaceLogForwarder[forwarder];
+  local namespace = utils.namespacedName(forwarder).namespace;
+  local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name);
+
+  if serviceAccount == '' then std.trace('Warning! No ServiceAccountName set in namespaceLogForwarder!', {})
+  else kube.ServiceAccount(serviceAccount) {
+    metadata+: {
+      namespace: namespace,
+    },
+    spec: {
+
+    },
+  }
+  for forwarder in std.objectFields(params.namespaceLogForwarder)
+  if !std.member([ 'enabled', 'instance', 'openshift-logging/instance' ], forwarder)
+];
+
+// namespaceRoleBinding:
+// Create RoleBinding for namespaced LogForwarder.
+local namespaceRoleBinding = [
+  local specs = params.namespaceLogForwarder[forwarder];
+  local namespace = utils.namespacedName(forwarder).namespace;
+  local serviceAccount = std.get(specs, 'serviceAccountName', utils.namespacedName(forwarder).name);
+
+  kube.RoleBinding(serviceAccount) {
+    metadata+: {
+      namespace: namespace,
+    },
+    roleRef: {
+      apiGroup: 'rbac.authorization.k8s.io',
+      kind: 'ClusterRole',
+      name: 'collect-application-logs',
+    },
+    subjects: [ {
+      kind: 'ServiceAccount',
+      name: serviceAccount,
+      namespace: namespace,
+    } ],
+  }
+  for forwarder in std.objectFields(params.namespaceLogForwarder)
+  if !std.member([ 'enabled', 'instance', 'openshift-logging/instance' ], forwarder)
+];
+
 local enableLogForwarder = std.length(params.clusterLogForwarder) > 0 || std.get(legacyConfig, 'enabled', false);
 
 // Define outputs below
 if enableLogForwarder then
   {
     '31_cluster_logforwarding': clusterLogForwarder,
+    [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_logforwarding']: namespaceLogForwarder,
+    [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_serviceaccount']: namespaceServiceAccount,
+    [if std.length(params.namespaceLogForwarder) > 1 then '32_namespace_rolebinding']: namespaceRoleBinding,
   }
 else
   std.trace(

component/main.jsonnet

@@ -30,7 +30,7 @@ local operatorGroup = operatorlib.OperatorGroup('cluster-logging') {
     namespace: params.namespace,
   },
   spec: {
-    targetNamespaces: [
+    [if !params.namespaceLogForwarder.enabled then 'targetNamespaces']: [
       params.namespace,
     ],
   },

component/utils.libsonnet

@@ -20,7 +20,14 @@ local isVersion59 =
   else if std.parseInt(major) == 5 && std.parseInt(minor) >= 9 then true
   else false;
 
+local namespacedName(name) = {
+  local namespaced = std.splitLimit(name, '/', 1),
+  namespace: if std.length(namespaced) > 1 then namespaced[0] else params.namespace,
+  name: if std.length(namespaced) > 1 then namespaced[1] else namespaced[0],
+};
+
 {
   isVersion58: isVersion58,
   isVersion59: isVersion59,
+  namespacedName: namespacedName,
 }

docs/modules/ROOT/pages/references/parameters.adoc

@@ -368,6 +368,25 @@ See the https://docs.openshift.com/container-platform/latest/observability/loggi
 IMPORTANT: `clusterLogForwarding` is deprecated, please use `clusterLogForwarder`
 
 
+== `namespaceLogForwarder`
+
+[horizontal]
+type:: dictionary
+default::
++
+[source,yaml]
+----
+namespaceLogForwarder:
+  enabled: false
+----
+
+A dictionary holding the `.spec` for namespaced log forwarding.
+
+See in examples below for configuration.
+
+NOTE: Enabling namespaced log forwarding requires redeploying the logging operator. See https://docs.openshift.com/container-platform/latest/observability/logging/cluster-logging-upgrading.html#logging-operator-upgrading-all-ns_cluster-logging-upgrading[OpenShift docs] for instructions.
+
+
 == Examples
 
 [source,yaml]
@@ -381,6 +400,29 @@ clusterLogging:
       nodeCount: 5
 ----
 
+=== Use namespaced ClusterLogForwarder
+
+Example creates a `ClusterLogForwarder`, `ServiceAccount` and `RoleBinding` in namespace `my-namespace`.
+
+[source,yaml]
+----
+namespaceLogForwarder:
+  enabled: true
+  my-namespace/my-forwarder:
+    outputs:
+      splunk-forwarder:
+        secret:
+          name: splunk-forwarder
+        type: fluentdForward
+        url: tls://splunk-forwarder:24224
+    pipelines:
+      application-logs:
+        inputRefs:
+          - application
+        outputRefs:
+          - splunk-forwarder
+----
+
 === Forward logs for all application logs to third-party
 
 [source,yaml]

tests/golden/master/openshift4-logging/openshift4-logging/10_operator_group.yaml

@@ -6,6 +6,4 @@ metadata:
     name: cluster-logging
   name: cluster-logging
   namespace: openshift-logging
-spec:
-  targetNamespaces:
-    - openshift-logging
+spec: {}

tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_logforwarding.yaml

@@ -0,0 +1,48 @@
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogForwarder
+metadata:
+  annotations: {}
+  labels:
+    name: bar
+  name: bar
+  namespace: foo
+spec:
+  inputs:
+    - application:
+        namespaces:
+          - app-one
+          - app-two
+      name: my-apps
+  outputs:
+    - name: custom-forwarder
+      type: syslog
+  pipelines:
+    - inputRefs:
+        - my-apps
+      name: my-apps
+      outputRefs:
+        - custom-forwarder
+  serviceAccountName: ueli
+---
+apiVersion: logging.openshift.io/v1
+kind: ClusterLogForwarder
+metadata:
+  annotations: {}
+  labels:
+    name: hands
+  name: hands
+  namespace: jazz
+spec:
+  outputs:
+    - name: splunk-forwarder
+      secret:
+        name: splunk-forwarder
+      type: fluentdForward
+      url: tls://splunk-forwarder:24224
+  pipelines:
+    - inputRefs:
+        - application
+      name: application-logs
+      outputRefs:
+        - splunk-forwarder
+  serviceAccountName: hands

tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_rolebinding.yaml

@@ -0,0 +1,33 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations: {}
+  labels:
+    name: ueli
+  name: ueli
+  namespace: foo
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: collect-application-logs
+subjects:
+  - kind: ServiceAccount
+    name: ueli
+    namespace: foo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  annotations: {}
+  labels:
+    name: hands
+  name: hands
+  namespace: jazz
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: collect-application-logs
+subjects:
+  - kind: ServiceAccount
+    name: hands
+    namespace: jazz

tests/golden/master/openshift4-logging/openshift4-logging/32_namespace_serviceaccount.yaml

@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations: {}
+  labels:
+    name: ueli
+  name: ueli
+  namespace: foo
+spec: {}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations: {}
+  labels:
+    name: hands
+  name: hands
+  namespace: jazz
+spec: {}

tests/master.yml

@@ -49,3 +49,36 @@ parameters:
         audit-logs:
           outputRefs:
             - custom-forwarder
+
+    namespaceLogForwarder:
+      enabled: true
+      jazz/hands:
+        outputs:
+          splunk-forwarder:
+            secret:
+              name: splunk-forwarder
+            type: fluentdForward
+            url: tls://splunk-forwarder:24224
+        pipelines:
+          application-logs:
+            inputRefs:
+              - application
+            outputRefs:
+              - splunk-forwarder
+      foo/bar:
+        serviceAccountName: ueli
+        inputs:
+          my-apps:
+            application:
+              namespaces:
+                - app-one
+                - app-two
+        outputs:
+          custom-forwarder:
+            type: syslog
+        pipelines:
+          my-apps:
+            inputRefs:
+              - my-apps
+            outputRefs:
+              - custom-forwarder