allow to specify a list of alertmanager servers + fix attribute name …

…for cloud events Signed-off-by: Thomas Labarussias <[email protected]>
falcosecurity · Nov 21, 2024 · 0f4f794 · 0f4f794
1 parent 5c7e05f
commit 0f4f794
Show file tree

Hide file tree

Showing 9 changed files with 42 additions and 17 deletions.
diff --git a/config.go b/config.go
@@ -94,7 +94,7 @@ var httpOutputDefaults = map[string]map[string]any{
 		"Icon":            "https://raw.githubusercontent.com/falcosecurity/falcosidekick/master/imgs/falcosidekick_color.png",
 	},
 	"Alertmanager": {
-		"HostPort":                 "",
+		"HostPort":                 []string{},
 		"MinimumPriority":          "",
 		"Endpoint":                 "/api/v1/alerts",
 		"ExpiresAfter":             0,
@@ -441,7 +441,7 @@ func getConfig() *types.Configuration {
 		Quickwit:        types.QuickwitOutputConfig{CustomHeaders: make(map[string]string)},
 		OpenObserve:     types.OpenObserveConfig{CustomHeaders: make(map[string]string)},
 		Webhook:         types.WebhookOutputConfig{CustomHeaders: make(map[string]string)},
-		Alertmanager:    types.AlertmanagerOutputConfig{ExtraLabels: make(map[string]string), ExtraAnnotations: make(map[string]string), CustomSeverityMap: make(map[types.PriorityType]string), CustomHeaders: make(map[string]string)},
+		Alertmanager:    types.AlertmanagerOutputConfig{HostPort: make([]string, 0), ExtraLabels: make(map[string]string), ExtraAnnotations: make(map[string]string), CustomSeverityMap: make(map[types.PriorityType]string), CustomHeaders: make(map[string]string)},
 		CloudEvents:     types.CloudEventsOutputConfig{Extensions: make(map[string]string)},
 		GCP:             types.GcpOutputConfig{PubSub: types.GcpPubSub{CustomAttributes: make(map[string]string)}},
 		OTLP: types.OTLPOutputConfig{

diff --git a/config_example.yaml b/config_example.yaml
@@ -84,7 +84,7 @@ datadoglogs:
   # service: "" # The name of the application or service generating the log events.
 
 alertmanager:
-  # hostport: "" # http://{domain or ip}:{port}, if not empty, Alertmanager output is enabled
+  # hostport: "" # Comma separated list of http://{domain or ip}:{port} that will all receive the payload, if not empty, Alertmanager output is enabled
   # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
   # mutualtls: false # if true, checkcert flag will be ignored (server cert will always be checked)
   # checkcert: true # check if ssl certificate of the output is valid (default: true)

diff --git a/docs/outputs/alertmanager.md b/docs/outputs/alertmanager.md
@@ -13,9 +13,9 @@
 
 ## Configuration
 
-| Setting                                 | Env var                                 | Default value                                                        | Description                                                                                                                                                                                                                              |
+|                 Setting                 |                 Env var                 |                            Default value                             |                                                                                                               Description                                                                                                                |
 | --------------------------------------- | --------------------------------------- | -------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `alertmanager.hostport`                 | `ALERTMANAGER_HOSTPORT`                 |                                                                      | http://{domain or ip}:{port}, if not empty, Alertmanager output is **enabled**                                                                                                                                                           |
+| `alertmanager.hostport`                 | `ALERTMANAGER_HOSTPORT`                 |                                                                      | Comma separated list of http://{domain or ip}:{port} that will all receive the payload, if not empty, Alertmanager output is **enabled**                                                                                                 |
 | `alertmanager.mutualtls`                | `ALERTMANAGER_MUTUALTLS`                | `false`                                                              | Authenticate to the output with TLS, if true, checkcert flag will be ignored (server cert will always be checked)                                                                                                                        |
 | `alertmanager.checkcert`                | `ALERTMANAGER_CHECKCERT`                | `true`                                                               | check if ssl certificate of the output is valid                                                                                                                                                                                          |
 | `alertmanager.endpoint`                 | `ALERTMANAGER_ENDPOINT`                 | `/api/v1/alerts`                                                     | Alertmanager endpoint for posting alerts `/api/v1/alerts` or `/api/v2/alerts`                                                                                                                                                            |

diff --git a/handlers.go b/handlers.go
@@ -324,8 +324,10 @@ func forwardEvent(falcopayload types.FalcoPayload) {
 		go discordClient.DiscordPost(falcopayload)
 	}
 
-	if config.Alertmanager.HostPort != "" && (falcopayload.Priority >= types.Priority(config.Alertmanager.MinimumPriority) || falcopayload.Rule == testRule) {
-		go alertmanagerClient.AlertmanagerPost(falcopayload)
+	if len(config.Alertmanager.HostPort) != 0 && (falcopayload.Priority >= types.Priority(config.Alertmanager.MinimumPriority) || falcopayload.Rule == testRule) {
+		for _, i := range alertmanagerClients {
+			go i.AlertmanagerPost(falcopayload)
+		}
 	}
 
 	if config.Elasticsearch.HostPort != "" && (falcopayload.Priority >= types.Priority(config.Elasticsearch.MinimumPriority) || falcopayload.Rule == testRule) {

diff --git a/main.go b/main.go
@@ -37,7 +37,7 @@ var (
 	datadogClient       *outputs.Client
 	datadogLogsClient   *outputs.Client
 	discordClient       *outputs.Client
-	alertmanagerClient  *outputs.Client
+	alertmanagerClients []*outputs.Client
 	elasticsearchClient *outputs.Client
 	quickwitClient      *outputs.Client
 	influxdbClient      *outputs.Client
@@ -247,12 +247,11 @@ func init() {
 		}
 	}
 
-	if config.Alertmanager.HostPort != "" {
+	if len(config.Alertmanager.HostPort) != 0 {
 		var err error
-		endpointUrl := fmt.Sprintf("%s%s", config.Alertmanager.HostPort, config.Alertmanager.Endpoint)
-		alertmanagerClient, err = outputs.NewClient("AlertManager", endpointUrl, config.Alertmanager.CommonConfig, *initClientArgs)
+		alertmanagerClients, err = outputs.NewAlertManagerClient(config.Alertmanager.HostPort, config.Alertmanager.Endpoint, config.Alertmanager.CommonConfig, *initClientArgs)
 		if err != nil {
-			config.Alertmanager.HostPort = ""
+			config.Alertmanager.HostPort = []string{}
 		} else {
 			outputs.EnabledOutputs = append(outputs.EnabledOutputs, "AlertManager")
 		}

diff --git a/outputs/alertmanager.go b/outputs/alertmanager.go
@@ -4,7 +4,7 @@ package outputs
 
 import (
 	"encoding/json"
-	"go.opentelemetry.io/otel/attribute"
+	"fmt"
 	"log"
 	"net/http"
 	"regexp"
@@ -13,6 +13,8 @@ import (
 	"strings"
 	"time"
 
+	"go.opentelemetry.io/otel/attribute"
+
 	"github.com/falcosecurity/falcosidekick/types"
 )
 
@@ -38,6 +40,28 @@ var (
 	reg = regexp.MustCompile("[^a-zA-Z0-9_]")
 )
 
+func NewAlertManagerClient(hostPorts []string, endpoint string, cfg types.CommonConfig, params types.InitClientArgs) ([]*Client, error) {
+	clients := make([]*Client, 0)
+	if len(hostPorts) == 1 {
+		endpointUrl := fmt.Sprintf("%s%s", hostPorts[0], endpoint)
+		c, err := NewClient("AlertManager", endpointUrl, cfg, params)
+		if err != nil {
+			return nil, err
+		}
+		clients = append(clients, c)
+	} else {
+		for i, j := range hostPorts {
+			endpointUrl := fmt.Sprintf("%s%s", j, endpoint)
+			c, err := NewClient(fmt.Sprintf("AlertManager_%v", i), endpointUrl, cfg, params)
+			if err != nil {
+				return nil, err
+			}
+			clients = append(clients, c)
+		}
+	}
+	return clients, nil
+}
+
 func newAlertmanagerPayload(falcopayload types.FalcoPayload, config *types.Configuration) []alertmanagerPayload {
 	var amPayload alertmanagerPayload
 	amPayload.Labels = make(map[string]string)

diff --git a/outputs/cloudevents.go b/outputs/cloudevents.go
@@ -38,7 +38,7 @@ func (c *Client) CloudEventsSend(falcopayload types.FalcoPayload) {
 	event.SetType("falco.rule.output.v1")
 	event.SetExtension("priority", falcopayload.Priority.String())
 	event.SetExtension("rule", falcopayload.Rule)
-	event.SetExtension("event_source", falcopayload.Source)
+	event.SetExtension("eventsource", falcopayload.Source)
 
 	if falcopayload.Hostname != "" {
 		event.SetExtension(Hostname, falcopayload.Hostname)

diff --git a/stats_prometheus.go b/stats_prometheus.go
@@ -54,11 +54,11 @@ func getFalcoNewCounterVec(config *types.Configuration) *prometheus.CounterVec {
 		"k8s_pod_name",
 	}
 	for i := range config.Customfields {
-		if !regPromLabels.MatchString(i) {
+		if !regPromLabels.MatchString(strings.ReplaceAll(i, ".", "_")) {
 			log.Printf("[ERROR] : Custom field '%v' is not a valid prometheus label", i)
 			continue
 		}
-		labelnames = append(labelnames, i)
+		labelnames = append(labelnames, strings.ReplaceAll(i, ".", "_"))
 	}
 	for _, i := range config.Prometheus.ExtraLabelsList {
 		if !regPromLabels.MatchString(strings.ReplaceAll(i, ".", "_")) {

diff --git a/types/types.go b/types/types.go
@@ -267,7 +267,7 @@ type ThresholdConfig struct {
 
 type AlertmanagerOutputConfig struct {
 	CommonConfig             `mapstructure:",squash"`
-	HostPort                 string
+	HostPort                 []string
 	MinimumPriority          string
 	Endpoint                 string
 	ExpiresAfter             int