update(engine): modify append_output format

Signed-off-by: Luca Guerra <[email protected]>

falco.yaml

@@ -595,24 +595,30 @@ outputs_queue:
 # This allows you to add custom fields that can help you filter your Falco events without
 # polluting the message text.
 #
-# Each append_output entry has optional fields (ANDed together) to filter events:
+# Each append_output entry has an optional `match` map which specifies which rules will be
+# affected.
+# `match`:
 #   `rule`: append output only to a specific rule
 #   `source`: append output only to a specific source
-#   `tag`: append output only to a specific tag
-# If none of the above are specified output is appended to all events, if more than one is
-# specified output will be appended to events that match all conditions.
+#   `tags`: append output only to rules that have all of the specified tags
+# If none of the above are specified (or `match` is omitted)
+# output is appended to all events.
+# If more than one match condition is specified output will be appended to events 
+# that match all conditions.
 # And several options to add output:
-#   `format`: add output to the Falco message
-#   `fields`: add new fields to the JSON output and structured output, which will not
+#   `extra_output`: add output to the Falco message
+#   `extra_fields`: add new fields to the JSON output and structured output, which will not
 #             affect the regular Falco message in any way. These can be specified as a
 #             custom name with a custom format or as any supported field
 #             (see: https://falco.org/docs/reference/rules/supported-fields/)
 #
 # Example:
-#
-#   - source: syscall
-#     format: "on CPU %evt.cpu"
-#     fields:
+# 
+# append_output:
+#   - match:
+#       source: syscall
+#     extra_output: "on CPU %evt.cpu"
+#     extra_fields:
 #       - home_directory: "${HOME}"
 #       - evt.hostname
 #

unit_tests/engine/test_extra_output.cpp

@@ -29,7 +29,7 @@ TEST_F(test_falco_engine, extra_format_all)
   priority: INFO
 )END";
 
-	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false);
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", {}, "", false);
 	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
 
 	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type");
@@ -51,7 +51,7 @@ TEST_F(test_falco_engine, extra_format_by_rule)
   priority: INFO
 )END";
 
-	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false);
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", {}, "legit_rule", false);
 	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
 
 	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type");
@@ -74,15 +74,24 @@ TEST_F(test_falco_engine, extra_format_by_tag_rule)
   output: out 2
   priority: INFO
   tags: [tag1]
+
+- rule: a_third_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 3
+  priority: INFO
+  tags: [tag1, tag2]
 )END";
 
-	m_engine->add_extra_output_format("extra 1", "", "tag1", "", false);
-	m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false);
+	m_engine->add_extra_output_format("extra 1", "", {"tag1"}, "", false);
+	m_engine->add_extra_output_format("extra 2", "", {}, "another_rule", false);
+	m_engine->add_extra_output_format("extra 3", "", {"tag1", "tag2"}, "", false);
 
 	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
 
 	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1");
 	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2");
+	EXPECT_EQ(get_compiled_rule_output("a_third_rule"),"out 3 extra 1 extra 3");
 }
 
 TEST_F(test_falco_engine, extra_format_replace_container_info)
@@ -103,7 +112,7 @@ TEST_F(test_falco_engine, extra_format_replace_container_info)
   tags: [tag1]
 )END";
 
-	m_engine->add_extra_output_format("extra 1", "", "", "", true);
+	m_engine->add_extra_output_format("extra 1", "", {}, "", true);
 
 	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
 
@@ -141,7 +150,7 @@ TEST_F(test_falco_engine, extra_fields_all)
 	std::unordered_map<std::string, std::string> extra_formatted_fields = {{"my_field", "hello %evt.num"}};
   for (auto const& f : extra_formatted_fields)
   {
-    m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", "");
+    m_engine->add_extra_output_formatted_field(f.first, f.second, "", {}, "");
   }
 
 	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;

unit_tests/falco/test_configuration_output_options.cpp

@@ -23,20 +23,24 @@ TEST(ConfigurationRuleOutputOptions, parse_yaml)
 	falco_configuration falco_config;
 	ASSERT_NO_THROW(falco_config.init_from_content(R"(
 append_output:
-  - source: syscall
-    tag: persistence
-    rule: some rule name
-    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
-
-  - tag: persistence
-    fields:
+  - match:
+      source: syscall
+      tags: ["persistence"]
+      rule: some rule name
+    
+    extra_output: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - match:
+      tags: ["persistence", "execution"]
+    extra_fields:
       - proc.aname[2]: "%proc.aname[2]"
       - proc.aname[3]: "%proc.aname[3]"
       - proc.aname[4]: "%proc.aname[4]"
-    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+    extra_output: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
 
-  - source: k8s_audit
-    fields:
+  - match:
+      source: k8s_audit
+    extra_fields:
       - ka.verb
       - static_field: "static content"
 
@@ -45,12 +49,15 @@ TEST(ConfigurationRuleOutputOptions, parse_yaml)
 	EXPECT_EQ(falco_config.m_append_output.size(), 3);
 
 	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
-	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tags.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[0].m_tags.count("persistence"), 1);
 	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
 	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
 	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
 
-	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.size(), 2);
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.count("persistence"), 1);
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.count("execution"), 1);
 	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
 
 	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
@@ -73,19 +80,22 @@ TEST(ConfigurationRuleOutputOptions, cli_options)
 
 	ASSERT_NO_THROW(falco_config.init_from_content("", 
 		std::vector<std::string>{
-			R"(append_output[]={"source": "syscall", "tag": "persistence", "rule": "some rule name", "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
-			R"(append_output[]={"tag": "persistence", "fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
-			R"(append_output[]={"source": "k8s_audit", "fields": ["ka.verb", {"static_field": "static content"}]})"}));
+			R"(append_output[]={"match": {"source": "syscall", "tags": ["persistence"], "rule": "some rule name"}, "extra_output": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"match": {"tags": ["persistence", "execution"]}, "extra_fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "extra_output": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"match": {"source": "k8s_audit"}, "extra_fields": ["ka.verb", {"static_field": "static content"}]})"}));
 
 	EXPECT_EQ(falco_config.m_append_output.size(), 3);
 
 	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
-	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tags.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[0].m_tags.count("persistence"), 1);
 	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
 	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
 	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
 
-	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.size(), 2);
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.count("persistence"), 1);
+	EXPECT_EQ(falco_config.m_append_output[1].m_tags.count("execution"), 1);
 	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
 
 	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);

userspace/engine/falco_engine.cpp

@@ -1103,34 +1103,34 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 void falco_engine::add_extra_output_format(
 	const std::string &format,
 	const std::string &source,
-	const std::string &tag,
+	const std::set<std::string> &tags,
 	const std::string &rule,
 	bool replace_container_info
 )
 {
-	m_extra_output_format.push_back({format, source, tag, rule, replace_container_info});
+	m_extra_output_format.push_back({format, source, tags, rule, replace_container_info});
 }
 
 void falco_engine::add_extra_output_formatted_field(
 	const std::string &key,
 	const std::string &format,
 	const std::string &source,
-	const std::string &tag,
+	const std::set<std::string> &tags,
 	const std::string &rule
 )
 {
-	m_extra_output_fields.push_back({key, format, source, tag, rule, false});
+	m_extra_output_fields.push_back({key, format, source, tags, rule, false});
 }
 
 void falco_engine::add_extra_output_raw_field(
 	const std::string &key,
 	const std::string &source,
-	const std::string &tag,
+	const std::set<std::string> &tags,
 	const std::string &rule
 )
 {
 	std::string format = "%" + key;
-	m_extra_output_fields.push_back({key, format, source, tag, rule, true});
+	m_extra_output_fields.push_back({key, format, source, tags, rule, true});
 }
 
 inline bool falco_engine::should_drop_evt() const

userspace/engine/falco_engine.h

@@ -186,7 +186,7 @@ class falco_engine
 	void add_extra_output_format(
 		const std::string &format,
 		const std::string &source,
-		const std::string &tag,
+		const std::set<std::string> &tags,
 		const std::string &rule,
 		bool replace_container_info
 	);
@@ -200,14 +200,14 @@ class falco_engine
 		const std::string &key,
 		const std::string &format,
 		const std::string &source,
-		const std::string &tag,
+		const std::set<std::string> &tags,
 		const std::string &rule
 	);
 
 	void add_extra_output_raw_field(
 		const std::string &key,
 		const std::string &source,
-		const std::string &tag,
+		const std::set<std::string> &tags,
 		const std::string &rule
 	);
 

userspace/engine/rule_loader.h

@@ -270,7 +270,7 @@ namespace rule_loader
 	{
 		std::string m_format;
 		std::string m_source;
-		std::string m_tag;
+		std::set<std::string> m_tags;
 		std::string m_rule;
 		bool m_replace_container_info;
 	};
@@ -280,7 +280,7 @@ namespace rule_loader
 		std::string m_key;
 		std::string m_format;
 		std::string m_source;
-		std::string m_tag;
+		std::set<std::string> m_tags;
 		std::string m_rule;
 		bool m_raw;
 	};

userspace/engine/rule_loader_compiler.cpp

@@ -19,6 +19,7 @@ limitations under the License.
 #include <memory>
 #include <set>
 #include <vector>
+#include <functional>
 
 #include "rule_loader_compiler.h"
 #include "filter_warning_resolver.h"
@@ -501,9 +502,18 @@ void rule_loader::compiler::compile_rule_infos(
 				continue;
 			}
 
-			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			if (extra.m_tags.size() != 0)
 			{
-				continue;
+				std::set<std::string> intersect;
+
+				set_intersection(extra.m_tags.begin(), extra.m_tags.end(),
+					r.tags.begin(), r.tags.end(),
+					inserter(intersect, intersect.begin()));
+
+				if (intersect.size() != extra.m_tags.size())
+				{
+					continue;
+				}
 			}
 
 			if (extra.m_rule != "" && r.name != extra.m_rule)
@@ -541,9 +551,18 @@ void rule_loader::compiler::compile_rule_infos(
 				continue;
 			}
 
-			if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0)
+			if (extra.m_tags.size() != 0)
 			{
-				continue;
+				std::set<std::string> intersect;
+
+				set_intersection(extra.m_tags.begin(), extra.m_tags.end(),
+					r.tags.begin(), r.tags.end(),
+					inserter(intersect, intersect.begin()));
+
+				if (intersect.size() != extra.m_tags.size())
+				{
+					continue;
+				}
 			}
 
 			if (extra.m_rule != "" && r.name != extra.m_rule)

userspace/falco/app/actions/init_falco_engine.cpp

@@ -28,17 +28,17 @@ void configure_output_format(falco::app::state& s)
 	{
 		if (eo.m_format != "")
 		{
-			s.engine->add_extra_output_format(eo.m_format, eo.m_source, eo.m_tag, eo.m_rule, false);
+			s.engine->add_extra_output_format(eo.m_format, eo.m_source, eo.m_tags, eo.m_rule, false);
 		}
 
 		for (auto const& ff : eo.m_formatted_fields)
 		{
-			s.engine->add_extra_output_formatted_field(ff.first, ff.second, eo.m_source, eo.m_tag, eo.m_rule);
+			s.engine->add_extra_output_formatted_field(ff.first, ff.second, eo.m_source, eo.m_tags, eo.m_rule);
 		}
 
 		for (auto const& rf : eo.m_raw_fields)
 		{
-			s.engine->add_extra_output_raw_field(rf, eo.m_source, eo.m_tag, eo.m_rule);
+			s.engine->add_extra_output_raw_field(rf, eo.m_source, eo.m_tags, eo.m_rule);
 		}
 	}
 
@@ -49,23 +49,23 @@ void configure_output_format(falco::app::state& s)
 
 	if(s.options.print_additional == "c" || s.options.print_additional == "container")
 	{
-		s.engine->add_extra_output_format(container_info, falco_common::syscall_source, "", "", true);
+		s.engine->add_extra_output_format(container_info, falco_common::syscall_source, {}, "", true);
 	}
 	else if(s.options.print_additional == "cg" || s.options.print_additional == "container-gvisor")
 	{
-		s.engine->add_extra_output_format(gvisor_info + " " + container_info, falco_common::syscall_source, "", "", true);
+		s.engine->add_extra_output_format(gvisor_info + " " + container_info, falco_common::syscall_source, {}, "", true);
 	}
 	else if(s.options.print_additional == "k" || s.options.print_additional == "kubernetes")
 	{
-		s.engine->add_extra_output_format(container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
+		s.engine->add_extra_output_format(container_info + " " + k8s_info, falco_common::syscall_source, {}, "", true);
 	}
 	else if(s.options.print_additional == "kg" || s.options.print_additional == "kubernetes-gvisor")
 	{
-		s.engine->add_extra_output_format(gvisor_info + " " + container_info + " " + k8s_info, falco_common::syscall_source, "", "", true);
+		s.engine->add_extra_output_format(gvisor_info + " " + container_info + " " + k8s_info, falco_common::syscall_source, {}, "", true);
 	}
 	else if(!s.options.print_additional.empty())
 	{
-		s.engine->add_extra_output_format(s.options.print_additional, "", "", "", false);
+		s.engine->add_extra_output_format(s.options.print_additional, "", {}, "", false);
 	}
 }