new(falco): add append_output configuration option with fields and format

falco.yaml

@@ -576,6 +576,42 @@ rule_matching: first
 outputs_queue:
   capacity: 0
 
+# [Sandbox] `append_output`
+#
+# Add information to the Falco output.
+# With this setting you can add more information to the Falco output message, customizable by
+# rule, tag or source.
+# You can also add additional data that will appear in the output_fields property
+# of JSON formatted messages or gRPC output but will not be part of the regular output message.
+# This allows you to add custom fields that can help you filter your Falco events without
+# polluting the message text.
+#
+# Each append_output entry has optional fields (ANDed together) to filter events:
+#   `rule`: append output only to a specific rule
+#   `source`: append output only to a specific source
+#   `tag`: append output only to a specific tag
+# If none of the above are specified output is appended to all events, if more than one is
+# specified output will be appended to events that match all conditions.
+# And several options to add output:
+#   `format`: add output to the Falco message
+#   `fields`: add new fields to the JSON output and structured output, which will not
+#             affect the regular Falco message in any way. These can be specified as a
+#             custom name with a custom format or as any supported field
+#             (see: https://falco.org/docs/reference/rules/supported-fields/)
+#
+# Example:
+#
+#   - source: syscall
+#     format: "on CPU %evt.cpu"
+#     fields:
+#       - home_directory: "${HOME}"
+#       - evt.hostname
+#
+# In the example above every event coming from the syscall source will get an extra message
+# at the end telling the CPU number. In addition, if `json_output` is true, in the "output_fields"
+# property you will find three new ones: "evt.cpu", "home_directory" which will contain the value of the
+# environment variable $HOME, and "evt.hostname" which will contain the hostname.
+
 
 ##########################
 # Falco outputs channels #

unit_tests/CMakeLists.txt

@@ -36,6 +36,7 @@ add_executable(falco_unit_tests
     engine/test_add_source.cpp
     engine/test_alt_rule_loader.cpp
     engine/test_enable_rule.cpp
+    engine/test_extra_output.cpp
     engine/test_falco_utils.cpp
     engine/test_filter_details_resolver.cpp
     engine/test_filter_macro_resolver.cpp
@@ -47,6 +48,7 @@ add_executable(falco_unit_tests
     falco/test_configuration_rule_selection.cpp
     falco/test_configuration_config_files.cpp
     falco/test_configuration_env_vars.cpp
+    falco/test_configuration_output_options.cpp
     falco/test_configuration_schema.cpp
     falco/app/actions/test_select_event_sources.cpp
     falco/app/actions/test_load_config.cpp

unit_tests/engine/test_extra_output.cpp

@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+
+#include "../test_falco_engine.h"
+
+TEST_F(test_falco_engine, extra_format_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type");
+}
+
+TEST_F(test_falco_engine, extra_format_by_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2");
+}
+
+TEST_F(test_falco_engine, extra_format_by_tag_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "tag1", "", false);
+	m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2");
+}
+
+TEST_F(test_falco_engine, extra_format_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "", "", true);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"), "out 1 (extra 1)");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"), "out 2 extra 1");
+}
+
+TEST_F(test_falco_engine, extra_format_do_not_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto output = get_compiled_rule_output("legit_rule");
+	EXPECT_TRUE(output.find("%container.info") == output.npos);
+}
+
+TEST_F(test_falco_engine, extra_fields_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	std::unordered_map<std::string, std::string> extra_formatted_fields = {{"my_field", "hello %evt.num"}};
+  for (auto const& f : extra_formatted_fields)
+  {
+    m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", "");
+  }
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_formatted_fields("legit_rule"), extra_formatted_fields);
+}

unit_tests/falco/test_configuration_output_options.cpp

@@ -0,0 +1,103 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+
+TEST(ConfigurationRuleOutputOptions, parse_yaml)
+{
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_content(R"(
+append_output:
+  - source: syscall
+    tag: persistence
+    rule: some rule name
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - tag: persistence
+    fields:
+      - proc.aname[2]: "%proc.aname[2]"
+      - proc.aname[3]: "%proc.aname[3]"
+      - proc.aname[4]: "%proc.aname[4]"
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - source: k8s_audit
+    fields:
+      - ka.verb
+      - static_field: "static content"
+
+	)", {}));
+
+	EXPECT_EQ(falco_config.m_append_output.size(), 3);
+
+	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
+	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
+	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
+}
+
+TEST(ConfigurationRuleOutputOptions, cli_options)
+{
+	falco_configuration falco_config;
+
+	ASSERT_NO_THROW(falco_config.init_from_content("", 
+		std::vector<std::string>{
+			R"(append_output[]={"source": "syscall", "tag": "persistence", "rule": "some rule name", "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"tag": "persistence", "fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})",
+			R"(append_output[]={"source": "k8s_audit", "fields": ["ka.verb", {"static_field": "static content"}]})"}));
+
+	EXPECT_EQ(falco_config.m_append_output.size(), 3);
+
+	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
+	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
+	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
+}

unit_tests/falco/test_configuration_schema.cpp

@@ -23,7 +23,7 @@ limitations under the License.
         do { \
 		for(const auto& pair : res) { \
 			auto validation_status = pair.second; \
-			EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)); \
+			EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)) << validation_status; \
 		}                                           \
 	} \
 	while (0)

unit_tests/test_falco_engine.cpp

@@ -85,3 +85,15 @@ std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name
 	auto rule_description = m_engine->describe_rule(&rule_name, {});
 	return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
 }
+
+std::string test_falco_engine::get_compiled_rule_output(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["output_compiled"].template get<std::string>();
+}
+
+std::unordered_map<std::string, std::string> test_falco_engine::get_compiled_rule_formatted_fields(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["extra_output_formatted_fields"].template get<std::unordered_map<std::string, std::string>>();
+}

unit_tests/test_falco_engine.h

@@ -6,6 +6,7 @@
 #include "rule_loading_messages.h"
 
 #include <gtest/gtest.h>
+#include <unordered_map>
 
 class test_falco_engine : public testing::Test
 {
@@ -19,6 +20,8 @@ class test_falco_engine : public testing::Test
 	bool check_warning_message(const std::string& warning_msg) const;
 	bool check_error_message(const std::string& error_msg) const;
 	std::string get_compiled_rule_condition(std::string rule_name = "") const;
+	std::string get_compiled_rule_output(std::string rule_name = "") const;
+	std::unordered_map<std::string, std::string> get_compiled_rule_formatted_fields(std::string rule_name) const;
 
 	std::string m_sample_ruleset = "sample-ruleset";
 	std::string m_sample_source = falco_common::syscall_source;

userspace/engine/falco_common.h

@@ -72,3 +72,5 @@ namespace falco_common
 
 	bool parse_rule_matching(const std::string& v, rule_matching& out);
 };
+
+typedef std::unordered_map<std::string, std::pair<std::string, bool>> extra_output_field_t;