new(app): add append_output configuration option with fields and format

Signed-off-by: Luca Guerra <[email protected]>

unit_tests/CMakeLists.txt

@@ -36,6 +36,7 @@ add_executable(falco_unit_tests
     engine/test_add_source.cpp
     engine/test_alt_rule_loader.cpp
     engine/test_enable_rule.cpp
+    engine/test_extra_output.cpp
     engine/test_falco_utils.cpp
     engine/test_filter_details_resolver.cpp
     engine/test_filter_macro_resolver.cpp
@@ -47,6 +48,7 @@ add_executable(falco_unit_tests
     falco/test_configuration_rule_selection.cpp
     falco/test_configuration_config_files.cpp
     falco/test_configuration_env_vars.cpp
+    falco/test_configuration_output_options.cpp
     falco/test_configuration_schema.cpp
     falco/app/actions/test_select_event_sources.cpp
     falco/app/actions/test_load_config.cpp

unit_tests/engine/test_extra_output.cpp

@@ -0,0 +1,150 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+
+#include "../test_falco_engine.h"
+
+TEST_F(test_falco_engine, extra_format_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type");
+}
+
+TEST_F(test_falco_engine, extra_format_by_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+)END";
+
+	m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false);
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2");
+}
+
+TEST_F(test_falco_engine, extra_format_by_tag_rule)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "tag1", "", false);
+	m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2");
+}
+
+TEST_F(test_falco_engine, extra_format_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+
+- rule: another_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 2
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	m_engine->add_extra_output_format("extra 1", "", "", "", true);
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_output("legit_rule"), "out 1 (extra 1)");
+	EXPECT_EQ(get_compiled_rule_output("another_rule"), "out 2 extra 1");
+}
+
+TEST_F(test_falco_engine, extra_format_do_not_replace_container_info)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: out 1 (%container.info)
+  priority: INFO
+  tags: [tag1]
+)END";
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	auto output = get_compiled_rule_output("legit_rule");
+	EXPECT_TRUE(output.find("%container.info") == output.npos);
+}
+
+TEST_F(test_falco_engine, extra_fields_all)
+{
+	std::string rules_content = R"END(
+- rule: legit_rule
+  desc: legit rule description
+  condition: evt.type=open
+  output: user=%user.name command=%proc.cmdline file=%fd.name
+  priority: INFO
+)END";
+
+	std::unordered_map<std::string, std::string> extra_formatted_fields = {{"my_field", "hello %evt.num"}};
+  for (auto const& f : extra_formatted_fields)
+  {
+    m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", "");
+  }
+
+	ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string;
+
+	EXPECT_EQ(get_compiled_rule_formatted_fields("legit_rule"), extra_formatted_fields);
+}

unit_tests/falco/test_configuration_output_options.cpp

@@ -0,0 +1,68 @@
+// SPDX-License-Identifier: Apache-2.0
+/*
+Copyright (C) 2024 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <gtest/gtest.h>
+#include <falco/configuration.h>
+
+TEST(ConfigurationRuleOutputOptions, parse_yaml)
+{
+	falco_configuration falco_config;
+	ASSERT_NO_THROW(falco_config.init_from_content(R"(
+append_output:
+  - source: syscall
+    tag: persistence
+    rule: some rule name
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - tag: persistence
+    fields:
+      - proc.aname[2]: "%proc.aname[2]"
+      - proc.aname[3]: "%proc.aname[3]"
+      - proc.aname[4]: "%proc.aname[4]"
+    format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"
+
+  - source: k8s_audit
+    fields:
+      - ka.verb
+      - static_field: "static content"
+
+	)", {}));
+
+	EXPECT_EQ(falco_config.m_append_output.size(), 3);
+
+	EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall");
+	EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name");
+	EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0);
+	EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence");
+	EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3);
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]");
+	EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content");
+
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1);
+	EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1);
+}

unit_tests/test_falco_engine.cpp

@@ -85,3 +85,15 @@ std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name
 	auto rule_description = m_engine->describe_rule(&rule_name, {});
 	return rule_description["rules"][0]["details"]["condition_compiled"].template get<std::string>();
 }
+
+std::string test_falco_engine::get_compiled_rule_output(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["output_compiled"].template get<std::string>();
+}
+
+std::unordered_map<std::string, std::string> test_falco_engine::get_compiled_rule_formatted_fields(std::string rule_name) const
+{
+	auto rule_description = m_engine->describe_rule(&rule_name, {});
+	return rule_description["rules"][0]["details"]["extra_output_formatted_fields"].template get<std::unordered_map<std::string, std::string>>();
+}

unit_tests/test_falco_engine.h

@@ -6,6 +6,7 @@
 #include "rule_loading_messages.h"
 
 #include <gtest/gtest.h>
+#include <unordered_map>
 
 class test_falco_engine : public testing::Test
 {
@@ -19,6 +20,8 @@ class test_falco_engine : public testing::Test
 	bool check_warning_message(const std::string& warning_msg) const;
 	bool check_error_message(const std::string& error_msg) const;
 	std::string get_compiled_rule_condition(std::string rule_name = "") const;
+	std::string get_compiled_rule_output(std::string rule_name = "") const;
+	std::unordered_map<std::string, std::string> get_compiled_rule_formatted_fields(std::string rule_name) const;
 
 	std::string m_sample_ruleset = "sample-ruleset";
 	std::string m_sample_source = falco_common::syscall_source;

userspace/engine/falco_engine.cpp

@@ -57,8 +57,7 @@ falco_engine::falco_engine(bool seed_rng)
 	  m_rule_compiler(std::make_shared<rule_loader::compiler>()),
 	  m_next_ruleset_id(0),
 	  m_min_priority(falco_common::PRIORITY_DEBUG),
-	  m_sampling_ratio(1), m_sampling_multiplier(0),
-	  m_replace_container_info(false)
+	  m_sampling_ratio(1), m_sampling_multiplier(0)
 {
 	if(seed_rng)
 	{
@@ -76,6 +75,7 @@ falco_engine::~falco_engine()
 	m_rule_collector->clear();
 	m_rule_stats_manager.clear();
 	m_sources.clear();
+	m_extra_output_format.clear();
 }
 
 sinsp_version falco_engine::engine_version()
@@ -194,8 +194,8 @@ void falco_engine::list_fields(const std::string &source, bool verbose, bool nam
 std::unique_ptr<load_result> falco_engine::load_rules(const std::string &rules_content, const std::string &name)
 {
 	rule_loader::configuration cfg(rules_content, m_sources, name);
-	cfg.output_extra = m_extra;
-	cfg.replace_output_container_info = m_replace_container_info;
+	cfg.extra_output_format = m_extra_output_format;
+	cfg.extra_output_fields = m_extra_output_fields;
 
 	// read rules YAML file and collect its definitions
 	if(m_rule_reader->read(cfg, *m_rule_collector))
@@ -455,6 +455,7 @@ std::unique_ptr<std::vector<falco_engine::rule_result>> falco_engine::process_ev
 		rule_result.priority_num = rule.priority;
 		rule_result.tags = rule.tags;
 		rule_result.exception_fields = rule.exception_fields;
+		rule_result.extra_output_fields = rule.extra_output_fields;
 		m_rule_stats_manager.on_event(rule);
 		res->push_back(rule_result);
 	}
@@ -646,9 +647,22 @@ void falco_engine::get_json_details(
 	out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators);
 	out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields);
 
+	// Get extra requested fields
+	std::vector<std::string> out_fields;
+
+	for(auto const& f : r.extra_output_fields)
+	{
+		// add all the field keys
+		out_fields.emplace_back(f.second.first);
+
+		if (!f.second.second) // formatted field
+		{
+			out["details"]["extra_output_formatted_fields"][f.first] = f.second.first;
+		}
+	}
+
 	// Get fields from output string
 	auto fmt = create_formatter(r.source, r.output);
-	std::vector<std::string> out_fields;
 	fmt->get_field_names(out_fields);
 	out["details"]["output_fields"] = sequence_to_json_array(out_fields);
 
@@ -1082,10 +1096,37 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier)
 	m_sampling_multiplier = sampling_multiplier;
 }
 
-void falco_engine::set_extra(const std::string &extra, bool replace_container_info)
+void falco_engine::add_extra_output_format(
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule,
+	bool replace_container_info
+)
+{
+	m_extra_output_format.push_back({format, source, tag, rule, replace_container_info});
+}
+
+void falco_engine::add_extra_output_formatted_field(
+	const std::string &key,
+	const std::string &format,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
+{
+	m_extra_output_fields.push_back({key, format, source, tag, rule, false});
+}
+
+void falco_engine::add_extra_output_raw_field(
+	const std::string &key,
+	const std::string &source,
+	const std::string &tag,
+	const std::string &rule
+)
 {
-	m_extra = extra;
-	m_replace_container_info = replace_container_info;
+	std::string format = "%" + key;
+	m_extra_output_fields.push_back({key, format, source, tag, rule, true});
 }
 
 inline bool falco_engine::should_drop_evt() const