From d87df4a3e52227c59c20123ac2bf1faf9c41ed11 Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Mon, 26 Aug 2024 15:15:42 +0000 Subject: [PATCH 1/7] new(app): add append_output configuration option with fields and format Signed-off-by: Luca Guerra --- unit_tests/CMakeLists.txt | 2 + unit_tests/engine/test_extra_output.cpp | 150 ++++++++++++++++++ .../test_configuration_output_options.cpp | 68 ++++++++ unit_tests/test_falco_engine.cpp | 12 ++ unit_tests/test_falco_engine.h | 3 + userspace/engine/falco_engine.cpp | 57 ++++++- userspace/engine/falco_engine.h | 36 ++++- userspace/engine/falco_rule.h | 1 + userspace/engine/formats.cpp | 81 +++++----- userspace/engine/formats.h | 4 +- userspace/engine/rule_loader.h | 25 ++- userspace/engine/rule_loader_compiler.cpp | 91 ++++++++--- .../falco/app/actions/init_falco_engine.cpp | 42 ++--- .../falco/app/actions/process_events.cpp | 4 +- userspace/falco/configuration.cpp | 1 + userspace/falco/configuration.h | 119 ++++++++++++++ userspace/falco/falco_outputs.cpp | 28 +++- userspace/falco/falco_outputs.h | 3 +- 18 files changed, 626 insertions(+), 101 deletions(-) create mode 100644 unit_tests/engine/test_extra_output.cpp create mode 100644 unit_tests/falco/test_configuration_output_options.cpp diff --git a/unit_tests/CMakeLists.txt b/unit_tests/CMakeLists.txt index c3d2f5882fb..7c7ce65f5e8 100644 --- a/unit_tests/CMakeLists.txt +++ b/unit_tests/CMakeLists.txt @@ -36,6 +36,7 @@ add_executable(falco_unit_tests engine/test_add_source.cpp engine/test_alt_rule_loader.cpp engine/test_enable_rule.cpp + engine/test_extra_output.cpp engine/test_falco_utils.cpp engine/test_filter_details_resolver.cpp engine/test_filter_macro_resolver.cpp @@ -47,6 +48,7 @@ add_executable(falco_unit_tests falco/test_configuration_rule_selection.cpp falco/test_configuration_config_files.cpp falco/test_configuration_env_vars.cpp + falco/test_configuration_output_options.cpp falco/test_configuration_schema.cpp falco/app/actions/test_select_event_sources.cpp falco/app/actions/test_load_config.cpp diff --git a/unit_tests/engine/test_extra_output.cpp b/unit_tests/engine/test_extra_output.cpp new file mode 100644 index 00000000000..845d22d160c --- /dev/null +++ b/unit_tests/engine/test_extra_output.cpp @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2024 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include + +#include "../test_falco_engine.h" + +TEST_F(test_falco_engine, extra_format_all) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: user=%user.name command=%proc.cmdline file=%fd.name + priority: INFO +)END"; + + m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "", false); + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + EXPECT_EQ(get_compiled_rule_output("legit_rule"),"user=%user.name command=%proc.cmdline file=%fd.name evt.type=%evt.type"); +} + +TEST_F(test_falco_engine, extra_format_by_rule) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: out 1 + priority: INFO + +- rule: another_rule + desc: legit rule description + condition: evt.type=open + output: out 2 + priority: INFO +)END"; + + m_engine->add_extra_output_format("evt.type=%evt.type", "", "", "legit_rule", false); + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 evt.type=%evt.type"); + EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2"); +} + +TEST_F(test_falco_engine, extra_format_by_tag_rule) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: out 1 + priority: INFO + tags: [tag1] + +- rule: another_rule + desc: legit rule description + condition: evt.type=open + output: out 2 + priority: INFO + tags: [tag1] +)END"; + + m_engine->add_extra_output_format("extra 1", "", "tag1", "", false); + m_engine->add_extra_output_format("extra 2", "", "", "another_rule", false); + + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + EXPECT_EQ(get_compiled_rule_output("legit_rule"),"out 1 extra 1"); + EXPECT_EQ(get_compiled_rule_output("another_rule"),"out 2 extra 1 extra 2"); +} + +TEST_F(test_falco_engine, extra_format_replace_container_info) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: out 1 (%container.info) + priority: INFO + tags: [tag1] + +- rule: another_rule + desc: legit rule description + condition: evt.type=open + output: out 2 + priority: INFO + tags: [tag1] +)END"; + + m_engine->add_extra_output_format("extra 1", "", "", "", true); + + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + EXPECT_EQ(get_compiled_rule_output("legit_rule"), "out 1 (extra 1)"); + EXPECT_EQ(get_compiled_rule_output("another_rule"), "out 2 extra 1"); +} + +TEST_F(test_falco_engine, extra_format_do_not_replace_container_info) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: out 1 (%container.info) + priority: INFO + tags: [tag1] +)END"; + + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + auto output = get_compiled_rule_output("legit_rule"); + EXPECT_TRUE(output.find("%container.info") == output.npos); +} + +TEST_F(test_falco_engine, extra_fields_all) +{ + std::string rules_content = R"END( +- rule: legit_rule + desc: legit rule description + condition: evt.type=open + output: user=%user.name command=%proc.cmdline file=%fd.name + priority: INFO +)END"; + + std::unordered_map extra_formatted_fields = {{"my_field", "hello %evt.num"}}; + for (auto const& f : extra_formatted_fields) + { + m_engine->add_extra_output_formatted_field(f.first, f.second, "", "", ""); + } + + ASSERT_TRUE(load_rules(rules_content, "legit_rules.yaml")) << m_load_result_string; + + EXPECT_EQ(get_compiled_rule_formatted_fields("legit_rule"), extra_formatted_fields); +} diff --git a/unit_tests/falco/test_configuration_output_options.cpp b/unit_tests/falco/test_configuration_output_options.cpp new file mode 100644 index 00000000000..6f956f71cbf --- /dev/null +++ b/unit_tests/falco/test_configuration_output_options.cpp @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2024 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include + +TEST(ConfigurationRuleOutputOptions, parse_yaml) +{ + falco_configuration falco_config; + ASSERT_NO_THROW(falco_config.init_from_content(R"( +append_output: + - source: syscall + tag: persistence + rule: some rule name + format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]" + + - tag: persistence + fields: + - proc.aname[2]: "%proc.aname[2]" + - proc.aname[3]: "%proc.aname[3]" + - proc.aname[4]: "%proc.aname[4]" + format: "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]" + + - source: k8s_audit + fields: + - ka.verb + - static_field: "static content" + + )", {})); + + EXPECT_EQ(falco_config.m_append_output.size(), 3); + + EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall"); + EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence"); + EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name"); + EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0); + EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence"); + EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]"); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]"); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit"); + + EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1); + EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content"); + + EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1); + EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1); +} diff --git a/unit_tests/test_falco_engine.cpp b/unit_tests/test_falco_engine.cpp index c6aaa71ee3a..2124085eefc 100644 --- a/unit_tests/test_falco_engine.cpp +++ b/unit_tests/test_falco_engine.cpp @@ -85,3 +85,15 @@ std::string test_falco_engine::get_compiled_rule_condition(std::string rule_name auto rule_description = m_engine->describe_rule(&rule_name, {}); return rule_description["rules"][0]["details"]["condition_compiled"].template get(); } + +std::string test_falco_engine::get_compiled_rule_output(std::string rule_name) const +{ + auto rule_description = m_engine->describe_rule(&rule_name, {}); + return rule_description["rules"][0]["details"]["output_compiled"].template get(); +} + +std::unordered_map test_falco_engine::get_compiled_rule_formatted_fields(std::string rule_name) const +{ + auto rule_description = m_engine->describe_rule(&rule_name, {}); + return rule_description["rules"][0]["details"]["extra_output_formatted_fields"].template get>(); +} diff --git a/unit_tests/test_falco_engine.h b/unit_tests/test_falco_engine.h index 9276efe120d..8fef3214c11 100644 --- a/unit_tests/test_falco_engine.h +++ b/unit_tests/test_falco_engine.h @@ -6,6 +6,7 @@ #include "rule_loading_messages.h" #include +#include class test_falco_engine : public testing::Test { @@ -19,6 +20,8 @@ class test_falco_engine : public testing::Test bool check_warning_message(const std::string& warning_msg) const; bool check_error_message(const std::string& error_msg) const; std::string get_compiled_rule_condition(std::string rule_name = "") const; + std::string get_compiled_rule_output(std::string rule_name = "") const; + std::unordered_map get_compiled_rule_formatted_fields(std::string rule_name) const; std::string m_sample_ruleset = "sample-ruleset"; std::string m_sample_source = falco_common::syscall_source; diff --git a/userspace/engine/falco_engine.cpp b/userspace/engine/falco_engine.cpp index 0d22dffe950..dd2603bf632 100644 --- a/userspace/engine/falco_engine.cpp +++ b/userspace/engine/falco_engine.cpp @@ -57,8 +57,7 @@ falco_engine::falco_engine(bool seed_rng) m_rule_compiler(std::make_shared()), m_next_ruleset_id(0), m_min_priority(falco_common::PRIORITY_DEBUG), - m_sampling_ratio(1), m_sampling_multiplier(0), - m_replace_container_info(false) + m_sampling_ratio(1), m_sampling_multiplier(0) { if(seed_rng) { @@ -76,6 +75,7 @@ falco_engine::~falco_engine() m_rule_collector->clear(); m_rule_stats_manager.clear(); m_sources.clear(); + m_extra_output_format.clear(); } sinsp_version falco_engine::engine_version() @@ -194,8 +194,8 @@ void falco_engine::list_fields(const std::string &source, bool verbose, bool nam std::unique_ptr falco_engine::load_rules(const std::string &rules_content, const std::string &name) { rule_loader::configuration cfg(rules_content, m_sources, name); - cfg.output_extra = m_extra; - cfg.replace_output_container_info = m_replace_container_info; + cfg.extra_output_format = m_extra_output_format; + cfg.extra_output_fields = m_extra_output_fields; // read rules YAML file and collect its definitions if(m_rule_reader->read(cfg, *m_rule_collector)) @@ -455,6 +455,7 @@ std::unique_ptr> falco_engine::process_ev rule_result.priority_num = rule.priority; rule_result.tags = rule.tags; rule_result.exception_fields = rule.exception_fields; + rule_result.extra_output_fields = rule.extra_output_fields; m_rule_stats_manager.on_event(rule); res->push_back(rule_result); } @@ -646,9 +647,22 @@ void falco_engine::get_json_details( out["details"]["condition_operators"] = sequence_to_json_array(compiled_details.operators); out["details"]["condition_fields"] = sequence_to_json_array(compiled_details.fields); + // Get extra requested fields + std::vector out_fields; + + for(auto const& f : r.extra_output_fields) + { + // add all the field keys + out_fields.emplace_back(f.second.first); + + if (!f.second.second) // formatted field + { + out["details"]["extra_output_formatted_fields"][f.first] = f.second.first; + } + } + // Get fields from output string auto fmt = create_formatter(r.source, r.output); - std::vector out_fields; fmt->get_field_names(out_fields); out["details"]["output_fields"] = sequence_to_json_array(out_fields); @@ -1082,10 +1096,37 @@ void falco_engine::set_sampling_multiplier(double sampling_multiplier) m_sampling_multiplier = sampling_multiplier; } -void falco_engine::set_extra(const std::string &extra, bool replace_container_info) +void falco_engine::add_extra_output_format( + const std::string &format, + const std::string &source, + const std::string &tag, + const std::string &rule, + bool replace_container_info +) +{ + m_extra_output_format.push_back({format, source, tag, rule, replace_container_info}); +} + +void falco_engine::add_extra_output_formatted_field( + const std::string &key, + const std::string &format, + const std::string &source, + const std::string &tag, + const std::string &rule +) +{ + m_extra_output_fields.push_back({key, format, source, tag, rule, false}); +} + +void falco_engine::add_extra_output_raw_field( + const std::string &key, + const std::string &source, + const std::string &tag, + const std::string &rule +) { - m_extra = extra; - m_replace_container_info = replace_container_info; + std::string format = "%" + key; + m_extra_output_fields.push_back({key, format, source, tag, rule, true}); } inline bool falco_engine::should_drop_evt() const diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index e7e49293616..ce7c1f003eb 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -176,15 +176,40 @@ class falco_engine // void set_sampling_multiplier(double sampling_multiplier); - // - // You can optionally add "extra" formatting fields to the end + // You can optionally add "extra" output to the end // of all output expressions. You can also choose to replace // %container.info with the extra information or add it to the // end of the expression. This is used in open source falco to // add k8s/container information to outputs when // available. // - void set_extra(const std::string &extra, bool replace_container_info); + void add_extra_output_format( + const std::string &format, + const std::string &source, + const std::string &tag, + const std::string &rule, + bool replace_container_info + ); + + // You can optionally add fields that will only show up in the object + // output (e.g. json, gRPC) alongside other output_fields + // and not in the text message output. + // You can add two types of fields: formatted which will act like + // an additional output format that appears in the output field + void add_extra_output_formatted_field( + const std::string &key, + const std::string &format, + const std::string &source, + const std::string &tag, + const std::string &rule + ); + + void add_extra_output_raw_field( + const std::string &key, + const std::string &source, + const std::string &tag, + const std::string &rule + ); // Represents the result of matching an event against a set of // rules. @@ -196,6 +221,7 @@ class falco_engine std::string format; std::set exception_fields; std::set tags; + std::unordered_map> extra_output_fields; }; // @@ -461,6 +487,6 @@ class falco_engine static const std::string s_default_ruleset; uint32_t m_default_ruleset_id; - std::string m_extra; - bool m_replace_container_info; + std::vector m_extra_output_format; + std::vector m_extra_output_fields; }; diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index 52765ca1cb7..10d1fb4e45c 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -79,6 +79,7 @@ struct falco_rule std::string name; std::string description; std::string output; + std::unordered_map> extra_output_fields; std::set tags; std::set exception_fields; falco_common::priority_type priority; diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index c6826198bf2..d4e7919eb40 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -15,7 +15,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -#include +#include #include "formats.h" #include "falco_engine.h" @@ -35,7 +35,7 @@ falco_formats::~falco_formats() std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source, const std::string &level, const std::string &format, const std::set &tags, - const std::string &hostname) const + const std::string &hostname, const std::unordered_map> &extra_fields) const { std::string line; @@ -48,27 +48,17 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, if(formatter->get_output_format() == sinsp_evt_formatter::OF_JSON) { - std::string json_line; + std::string json_fields; // Format the event into a json object with all fields resolved - formatter->tostring(evt, json_line); - - // The formatted string might have a leading newline. If it does, remove it. - if(json_line[0] == '\n') - { - json_line.erase(0, 1); - } + formatter->tostring(evt, json_fields); // For JSON output, the formatter returned a json-as-text // object containing all the fields in the original format // message as well as the event time in ns. Use this to build // a more detailed object containing the event time, rule, // severity, full output, and fields. - Json::Value event; - Json::Value rule_tags; - Json::FastWriter writer; - std::string full_line; - unsigned int rule_tags_idx = 0; + nlohmann::json event; // Convert the time-as-nanoseconds to a more json-friendly ISO8601. time_t evttime = evt->get_ts() / 1000000000; @@ -94,43 +84,54 @@ std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, if(m_json_include_tags_property) { - if (tags.size() == 0) + event["tags"] = tags; + } + + event["output_fields"] = nlohmann::json::parse(json_fields); + + for (auto const& ef : extra_fields) + { + std::string fformat = ef.second.first; + if (fformat.size() == 0) { - // This sets an empty array - rule_tags = Json::arrayValue; + continue; } - else + + if (!(fformat[0] == '*')) { - for (const auto &tag : tags) - { - rule_tags[rule_tags_idx++] = tag; - } + fformat = "*" + fformat; } - event["tags"] = rule_tags; - } - full_line = writer.write(event); - - // Json::FastWriter may add a trailing newline. If it - // does, remove it. - if(full_line[full_line.length() - 1] == '\n') - { - full_line.resize(full_line.length() - 1); + if(ef.second.second) // raw field + { + std::string json_field_map; + formatter = m_falco_engine->create_formatter(source, fformat); + formatter->tostring_withformat(evt, json_field_map, sinsp_evt_formatter::OF_JSON); + auto json_obj = nlohmann::json::parse(json_field_map); + event["output_fields"][ef.first] = json_obj[ef.first]; + } else + { + event["output_fields"][ef.first] = format_string(evt, fformat, source); + } } - // Cheat-graft the output from the formatter into this - // string. Avoids an unnecessary json parse just to - // merge the formatted fields at the object level. - full_line.pop_back(); - full_line.append(", \"output_fields\": "); - full_line.append(json_line); - full_line.append("}"); - line = full_line; + line = event.dump(); } return line; } +std::string falco_formats::format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const +{ + std::string line; + std::shared_ptr formatter; + + formatter = m_falco_engine->create_formatter(source, format); + formatter->tostring_withformat(evt, line, sinsp_evt_formatter::OF_NORMAL); + + return line; +} + std::map falco_formats::get_field_values(sinsp_evt *evt, const std::string &source, const std::string &format) const { diff --git a/userspace/engine/formats.h b/userspace/engine/formats.h index 3c74609ff5f..97d822612e5 100644 --- a/userspace/engine/formats.h +++ b/userspace/engine/formats.h @@ -31,7 +31,9 @@ class falco_formats std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source, const std::string &level, const std::string &format, const std::set &tags, - const std::string &hostname) const; + const std::string &hostname, const std::unordered_map> &extra_fields) const; + + std::string format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const; std::map get_field_values(sinsp_evt *evt, const std::string &source, const std::string &format) const ; diff --git a/userspace/engine/rule_loader.h b/userspace/engine/rule_loader.h index 828133dd3ff..3ec15f903a9 100644 --- a/userspace/engine/rule_loader.h +++ b/userspace/engine/rule_loader.h @@ -20,6 +20,7 @@ limitations under the License. #include #include #include +#include #include #include #include "falco_source.h" @@ -261,6 +262,25 @@ namespace rule_loader nlohmann::json res_json; }; + struct extra_output_format_conf + { + std::string m_format; + std::string m_source; + std::string m_tag; + std::string m_rule; + bool m_replace_container_info; + }; + + struct extra_output_field_conf + { + std::string m_key; + std::string m_format; + std::string m_source; + std::string m_tag; + std::string m_rule; + bool m_raw; + }; + /*! \brief Contains the info required to load rule definitions */ @@ -278,8 +298,9 @@ namespace rule_loader const std::string& content; const indexed_vector& sources; std::string name; - std::string output_extra; - bool replace_output_container_info = false; + + std::vector extra_output_format; + std::vector extra_output_fields; // outputs std::unique_ptr res; diff --git a/userspace/engine/rule_loader_compiler.cpp b/userspace/engine/rule_loader_compiler.cpp index 2a40bcbfc52..0996045e30a 100644 --- a/userspace/engine/rule_loader_compiler.cpp +++ b/userspace/engine/rule_loader_compiler.cpp @@ -322,22 +322,6 @@ static std::shared_ptr parse_condition( } } -static void apply_output_substitutions( - rule_loader::configuration& cfg, - std::string& out) -{ - if (out.find(s_container_info_fmt) != std::string::npos) - { - if (cfg.replace_output_container_info) - { - out = replace(out, s_container_info_fmt, cfg.output_extra); - return; - } - out = replace(out, s_container_info_fmt, s_default_extra_fmt); - } - out += cfg.output_extra.empty() ? "" : " " + cfg.output_extra; -} - void rule_loader::compiler::compile_list_infos( configuration& cfg, const collector& col, @@ -510,13 +494,64 @@ void rule_loader::compiler::compile_rule_infos( // build rule output message rule.output = r.output; - // plugins sources do not have any container info and so we won't apply -pk, -pc, etc. - // on the other hand, when using plugins you might want to append custom output based on the plugin - // TODO: this is not flexible enough (esp. if you mix plugin with syscalls), - // it would be better to add configuration options to control the output. - if (!cfg.replace_output_container_info || r.source == falco_common::syscall_source) + for (auto& extra : cfg.extra_output_format) + { + if (extra.m_source != "" && r.source != extra.m_source) + { + continue; + } + + if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0) + { + continue; + } + + if (extra.m_rule != "" && r.name != extra.m_rule) + { + continue; + } + + if (extra.m_replace_container_info) + { + if (rule.output.find(s_container_info_fmt) != std::string::npos) + { + rule.output = replace(rule.output, s_container_info_fmt, extra.m_format); + } + else + { + rule.output = rule.output + " " + extra.m_format; + } + } else + { + rule.output = rule.output + " " + extra.m_format; + } + } + + if (rule.output.find(s_container_info_fmt) != std::string::npos) + { + rule.output = replace(rule.output, s_container_info_fmt, s_default_extra_fmt); + } + + // build extra output fields if required + + for (auto const& extra : cfg.extra_output_fields) { - apply_output_substitutions(cfg, rule.output); + if (extra.m_source != "" && r.source != extra.m_source) + { + continue; + } + + if (extra.m_tag != "" && r.tags.count(extra.m_tag) == 0) + { + continue; + } + + if (extra.m_rule != "" && r.name != extra.m_rule) + { + continue; + } + + rule.extra_output_fields[extra.m_key] = {extra.m_format, extra.m_raw}; } // validate the rule's output @@ -538,6 +573,18 @@ void rule_loader::compiler::compile_rule_infos( r.output_ctx); } + // validate the rule's extra fields if any + for (auto const& ef : rule.extra_output_fields) + { + if(!is_format_valid(*cfg.sources.at(r.source), ef.second.first, err)) + { + throw rule_load_exception( + falco::load_result::load_result::LOAD_ERR_COMPILE_OUTPUT, + err, + r.output_ctx); + } + } + if (!compile_condition(cfg, macro_resolver, lists, diff --git a/userspace/falco/app/actions/init_falco_engine.cpp b/userspace/falco/app/actions/init_falco_engine.cpp index 7d2bf930cb6..58286df658a 100644 --- a/userspace/falco/app/actions/init_falco_engine.cpp +++ b/userspace/falco/app/actions/init_falco_engine.cpp @@ -17,49 +17,55 @@ limitations under the License. #include "actions.h" #include +#include using namespace falco::app; using namespace falco::app::actions; void configure_output_format(falco::app::state& s) { + for (auto& eo : s.config->m_append_output) + { + if (eo.m_format != "") + { + s.engine->add_extra_output_format(eo.m_format, eo.m_source, eo.m_tag, eo.m_rule, false); + } + + for (auto const& ff : eo.m_formatted_fields) + { + s.engine->add_extra_output_formatted_field(ff.first, ff.second, eo.m_source, eo.m_tag, eo.m_rule); + } + + for (auto const& rf : eo.m_raw_fields) + { + s.engine->add_extra_output_raw_field(rf, eo.m_source, eo.m_tag, eo.m_rule); + } + } + // See https://falco.org/docs/rules/style-guide/ const std::string container_info = "container_id=%container.id container_image=%container.image.repository container_image_tag=%container.image.tag container_name=%container.name"; const std::string k8s_info = "k8s_ns=%k8s.ns.name k8s_pod_name=%k8s.pod.name"; const std::string gvisor_info = "vpid=%proc.vpid vtid=%thread.vtid"; - std::string output_format; - bool replace_container_info = false; - if(s.options.print_additional == "c" || s.options.print_additional == "container") { - output_format = container_info; - replace_container_info = true; + s.engine->add_extra_output_format(container_info, falco_common::syscall_source, "", "", true); } else if(s.options.print_additional == "cg" || s.options.print_additional == "container-gvisor") { - output_format = gvisor_info + " " + container_info; - replace_container_info = true; + s.engine->add_extra_output_format(gvisor_info + " " + container_info, falco_common::syscall_source, "", "", true); } else if(s.options.print_additional == "k" || s.options.print_additional == "kubernetes") { - output_format = container_info + " " + k8s_info; - replace_container_info = true; + s.engine->add_extra_output_format(container_info + " " + k8s_info, falco_common::syscall_source, "", "", true); } else if(s.options.print_additional == "kg" || s.options.print_additional == "kubernetes-gvisor") { - output_format = gvisor_info + " " + container_info + " " + k8s_info; - replace_container_info = true; + s.engine->add_extra_output_format(gvisor_info + " " + container_info + " " + k8s_info, falco_common::syscall_source, "", "", true); } else if(!s.options.print_additional.empty()) { - output_format = s.options.print_additional; - replace_container_info = false; - } - - if(!output_format.empty()) - { - s.engine->set_extra(output_format, replace_container_info); + s.engine->add_extra_output_format(s.options.print_additional, "", "", "", false); } } diff --git a/userspace/falco/app/actions/process_events.cpp b/userspace/falco/app/actions/process_events.cpp index 2f14c1bf262..33059304326 100644 --- a/userspace/falco/app/actions/process_events.cpp +++ b/userspace/falco/app/actions/process_events.cpp @@ -312,7 +312,9 @@ static falco::app::run_result do_inspect( { for(auto& rule_res : *res) { - s.outputs->handle_event(rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num, rule_res.format, rule_res.tags); + s.outputs->handle_event( + rule_res.evt, rule_res.rule, rule_res.source, rule_res.priority_num, + rule_res.format, rule_res.tags, rule_res.extra_output_fields); } } diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index a179fa2737a..6a3562f500e 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -588,6 +588,7 @@ void falco_configuration::load_yaml(const std::string& config_name) m_metrics_include_empty_values = m_config.get_scalar("metrics.include_empty_values", false); m_config.get_sequence>(m_rules_selection, "rules"); + m_config.get_sequence>(m_append_output, "append_output"); std::vector load_plugins; diff --git a/userspace/falco/configuration.h b/userspace/falco/configuration.h index 424ce66171c..7a3281a6c68 100644 --- a/userspace/falco/configuration.h +++ b/userspace/falco/configuration.h @@ -107,6 +107,15 @@ class falco_configuration std::string m_rule; }; + struct append_output_config { + std::string m_source; + std::string m_tag; + std::string m_rule; + std::string m_format; + std::unordered_map m_formatted_fields; + std::set m_raw_fields; + }; + falco_configuration(); virtual ~falco_configuration() = default; @@ -134,6 +143,8 @@ class falco_configuration std::list m_loaded_rules_folders; // Rule selection options passed by the user std::vector m_rules_selection; + // Append output configuration passed by the user + std::vector m_append_output; bool m_json_output; bool m_json_include_output_property; @@ -219,6 +230,114 @@ class falco_configuration }; namespace YAML { + template<> + struct convert { + static Node encode(const falco_configuration::append_output_config & rhs) { + Node node; + + if(rhs.m_source != "") + { + node["source"] = rhs.m_source; + } + + if(rhs.m_rule != "") + { + node["rule"] = rhs.m_rule; + } + + if(rhs.m_tag != "") + { + node["tag"] = rhs.m_tag; + } + + if(rhs.m_format != "") + { + node["format"] = rhs.m_format; + } + + for(auto const& field : rhs.m_formatted_fields) + { + YAML::Node field_node; + field_node[field.first] = field.second; + node["fields"].push_back(field_node); + } + + for(auto const& field : rhs.m_raw_fields) + { + node["fields"].push_back(field); + } + + return node; + } + + static bool decode(const Node& node, falco_configuration::append_output_config & rhs) { + if(!node.IsMap()) + { + return false; + } + + if(node["source"]) + { + rhs.m_source = node["source"].as(); + } + + if(node["tag"]) + { + rhs.m_tag = node["tag"].as(); + } + + if(node["rule"]) + { + rhs.m_rule = node["rule"].as(); + } + + if(node["format"]) + { + rhs.m_format = node["format"].as(); + } + + if(node["fields"]) + { + if(!node["fields"].IsSequence()) + { + return false; + } + + for(auto& field_definition : node["fields"]) + { + if(field_definition.IsMap() && field_definition.size() == 1) + { + YAML::const_iterator def = field_definition.begin(); + std::string key = def->first.as(); + + // it is an error to redefine an existing key + if (rhs.m_formatted_fields.count(key) != 0 || rhs.m_raw_fields.count(key) != 0) + { + return false; + } + + rhs.m_formatted_fields[key] = def->second.as(); + } else if (field_definition.IsScalar()) + { + std::string key = field_definition.as(); + + // it is an error to redefine an existing key + if (rhs.m_formatted_fields.count(key) != 0) + { + return false; + } + + rhs.m_raw_fields.insert(key); + } else { + return false; + } + } + } + + return true; + } + }; + template<> struct convert { static Node encode(const falco_configuration::rule_selection_config & rhs) { diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index 308092848b6..dcbfb07eb89 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -127,7 +127,8 @@ void falco_outputs::add_output(const falco::outputs::config &oc) } void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source, - falco_common::priority_type priority, const std::string &format, std::set &tags) + falco_common::priority_type priority, const std::string &format, std::set &tags, + std::unordered_map> &extra_fields) { falco_outputs::ctrl_msg cmsg = {}; cmsg.ts = evt->get_ts(); @@ -157,9 +158,30 @@ void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const } cmsg.msg = m_formats->format_event( - evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname + evt, rule, source, falco_common::format_priority(priority), sformat, tags, m_hostname, extra_fields ); - cmsg.fields = m_formats->get_field_values(evt, source, sformat); + + auto fields = m_formats->get_field_values(evt, source, sformat); + for (auto const& ef : extra_fields) + { + // when formatting for the control message we always want strings, + // so we can simply format raw fields as string + std::string fformat = ef.second.first; + if (fformat.size() == 0) + { + continue; + } + + if (!(fformat[0] == '*')) + { + fformat = "*" + fformat; + } + + fields[ef.first] = m_formats->format_string(evt, fformat, source); + } + + cmsg.fields = fields; + cmsg.tags.insert(tags.begin(), tags.end()); cmsg.type = ctrl_msg_type::CTRL_MSG_OUTPUT; diff --git a/userspace/falco/falco_outputs.h b/userspace/falco/falco_outputs.h index 800b4020b95..61de4be9536 100644 --- a/userspace/falco/falco_outputs.h +++ b/userspace/falco/falco_outputs.h @@ -59,7 +59,8 @@ class falco_outputs is an event that has matched some rule). */ void handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source, - falco_common::priority_type priority, const std::string &format, std::set &tags); + falco_common::priority_type priority, const std::string &format, std::set &tags, + std::unordered_map> &extra_fields); /*! \brief Format then send a generic message to all outputs. From ad81dd21c312c440e039800ca2ba3b966ce7ccbb Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Wed, 28 Aug 2024 15:33:08 +0000 Subject: [PATCH 2/7] new(falco): add json schema for append_output Signed-off-by: Luca Guerra --- userspace/falco/configuration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index 6a3562f500e..4e200e06838 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -53,7 +53,7 @@ static re2::RE2 ip_address_re("((^\\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]| // https://learn.microsoft.com/en-us/cpp/cpp/string-and-character-literals-cpp?view=msvc-170#size-of-string-literals // Just use any available online tool, eg: https://jsonformatter.org/json-minify // to format the json, add the new fields, and then minify it again. -static const std::string schema_json_string = R"({"$schema":"http://json-schema.org/draft-06/schema#","$ref":"#/definitions/FalcoConfig","definitions":{"FalcoConfig":{"type":"object","additionalProperties":false,"properties":{"config_files":{"type":"array","items":{"type":"string"}},"watch_config_files":{"type":"boolean"},"rules_files":{"type":"array","items":{"type":"string"}},"rule_files":{"type":"array","items":{"type":"string"}},"rules":{"type":"array","items":{"$ref":"#/definitions/Rule"}},"engine":{"$ref":"#/definitions/Engine"},"load_plugins":{"type":"array","items":{"type":"string"}},"plugins":{"type":"array","items":{"$ref":"#/definitions/Plugin"}},"time_format_iso_8601":{"type":"boolean"},"priority":{"type":"string"},"json_output":{"type":"boolean"},"json_include_output_property":{"type":"boolean"},"json_include_tags_property":{"type":"boolean"},"buffered_outputs":{"type":"boolean"},"rule_matching":{"type":"string"},"outputs_queue":{"$ref":"#/definitions/OutputsQueue"},"stdout_output":{"$ref":"#/definitions/Output"},"syslog_output":{"$ref":"#/definitions/Output"},"file_output":{"$ref":"#/definitions/FileOutput"},"http_output":{"$ref":"#/definitions/HTTPOutput"},"program_output":{"$ref":"#/definitions/ProgramOutput"},"grpc_output":{"$ref":"#/definitions/Output"},"grpc":{"$ref":"#/definitions/Grpc"},"webserver":{"$ref":"#/definitions/Webserver"},"log_stderr":{"type":"boolean"},"log_syslog":{"type":"boolean"},"log_level":{"type":"string"},"libs_logger":{"$ref":"#/definitions/LibsLogger"},"output_timeout":{"type":"integer"},"syscall_event_timeouts":{"$ref":"#/definitions/SyscallEventTimeouts"},"syscall_event_drops":{"$ref":"#/definitions/SyscallEventDrops"},"metrics":{"$ref":"#/definitions/Metrics"},"base_syscalls":{"$ref":"#/definitions/BaseSyscalls"},"falco_libs":{"$ref":"#/definitions/FalcoLibs"},"container_engines":{"type":"object","additionalProperties":false,"properties":{"docker":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"cri":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"sockets":{"type":"array","items":{"type":"string"}},"disable_async":{"type":"boolean"}}},"podman":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"libvirt_lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"bpm":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}}}}},"title":"FalcoConfig"},"BaseSyscalls":{"type":"object","additionalProperties":false,"properties":{"custom_set":{"type":"array","items":{"type":"string"}},"repair":{"type":"boolean"}},"minProperties":1,"title":"BaseSyscalls"},"Engine":{"type":"object","additionalProperties":false,"properties":{"kind":{"type":"string"},"kmod":{"$ref":"#/definitions/Kmod"},"ebpf":{"$ref":"#/definitions/Ebpf"},"modern_ebpf":{"$ref":"#/definitions/ModernEbpf"},"replay":{"$ref":"#/definitions/Replay"},"gvisor":{"$ref":"#/definitions/Gvisor"}},"required":["kind"],"title":"Engine"},"Ebpf":{"type":"object","additionalProperties":false,"properties":{"probe":{"type":"string"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"required":["probe"],"title":"Ebpf"},"Gvisor":{"type":"object","additionalProperties":false,"properties":{"config":{"type":"string"},"root":{"type":"string"}},"required":["config","root"],"title":"Gvisor"},"Kmod":{"type":"object","additionalProperties":false,"properties":{"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"minProperties":1,"title":"Kmod"},"ModernEbpf":{"type":"object","additionalProperties":false,"properties":{"cpus_for_each_buffer":{"type":"integer"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"title":"ModernEbpf"},"Replay":{"type":"object","additionalProperties":false,"properties":{"capture_file":{"type":"string"}},"required":["capture_file"],"title":"Replay"},"FalcoLibs":{"type":"object","additionalProperties":false,"properties":{"thread_table_size":{"type":"integer"}},"minProperties":1,"title":"FalcoLibs"},"FileOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"filename":{"type":"string"}},"minProperties":1,"title":"FileOutput"},"Grpc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"bind_address":{"type":"string"},"threadiness":{"type":"integer"}},"minProperties":1,"title":"Grpc"},"Output":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}},"minProperties":1,"title":"Output"},"HTTPOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"url":{"type":"string","format":"uri","qt-uri-protocols":["http"]},"user_agent":{"type":"string"},"insecure":{"type":"boolean"},"ca_cert":{"type":"string"},"ca_bundle":{"type":"string"},"ca_path":{"type":"string"},"mtls":{"type":"boolean"},"client_cert":{"type":"string"},"client_key":{"type":"string"},"echo":{"type":"boolean"},"compress_uploads":{"type":"boolean"},"keep_alive":{"type":"boolean"}},"minProperties":1,"title":"HTTPOutput"},"LibsLogger":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"severity":{"type":"string"}},"minProperties":1,"title":"LibsLogger"},"Metrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"interval":{"type":"string"},"output_rule":{"type":"boolean"},"output_file":{"type":"string"},"rules_counters_enabled":{"type":"boolean"},"resource_utilization_enabled":{"type":"boolean"},"state_counters_enabled":{"type":"boolean"},"kernel_event_counters_enabled":{"type":"boolean"},"libbpf_stats_enabled":{"type":"boolean"},"plugins_metrics_enabled":{"type":"boolean"},"convert_memory_to_mb":{"type":"boolean"},"include_empty_values":{"type":"boolean"}},"minProperties":1,"title":"Metrics"},"OutputsQueue":{"type":"object","additionalProperties":false,"properties":{"capacity":{"type":"integer"}},"minProperties":1,"title":"OutputsQueue"},"Plugin":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"library_path":{"type":"string"},"init_config":{"type":"string"},"open_params":{"type":"string"}},"required":["library_path","name"],"title":"Plugin"},"ProgramOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"program":{"type":"string"}},"required":["program"],"title":"ProgramOutput"},"Rule":{"type":"object","additionalProperties":false,"properties":{"disable":{"$ref":"#/definitions/Able"},"enable":{"$ref":"#/definitions/Able"}},"minProperties":1,"title":"Rule"},"Able":{"type":"object","additionalProperties":false,"properties":{"rule":{"type":"string"},"tag":{"type":"string"}},"minProperties":1,"title":"Able"},"SyscallEventDrops":{"type":"object","additionalProperties":false,"properties":{"threshold":{"type":"number"},"actions":{"type":"array","items":{"type":"string"}},"rate":{"type":"number"},"max_burst":{"type":"integer"},"simulate_drops":{"type":"boolean"}},"minProperties":1,"title":"SyscallEventDrops"},"SyscallEventTimeouts":{"type":"object","additionalProperties":false,"properties":{"max_consecutives":{"type":"integer"}},"minProperties":1,"title":"SyscallEventTimeouts"},"Webserver":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"threadiness":{"type":"integer"},"listen_port":{"type":"integer"},"listen_address":{"type":"string"},"k8s_healthz_endpoint":{"type":"string"},"prometheus_metrics_enabled":{"type":"boolean"},"ssl_enabled":{"type":"boolean"},"ssl_certificate":{"type":"string"}},"minProperties":1,"title":"Webserver"}}})"; +static const std::string schema_json_string = R"({"$schema":"http://json-schema.org/draft-06/schema#","$ref":"#/definitions/FalcoConfig","definitions":{"FalcoConfig":{"type":"object","additionalProperties":false,"properties":{"append_output":{"$ref":"#/definitions/AppendOutput"},"config_files":{"type":"array","items":{"type":"string"}},"watch_config_files":{"type":"boolean"},"rules_files":{"type":"array","items":{"type":"string"}},"rule_files":{"type":"array","items":{"type":"string"}},"rules":{"type":"array","items":{"$ref":"#/definitions/Rule"}},"engine":{"$ref":"#/definitions/Engine"},"load_plugins":{"type":"array","items":{"type":"string"}},"plugins":{"type":"array","items":{"$ref":"#/definitions/Plugin"}},"time_format_iso_8601":{"type":"boolean"},"priority":{"type":"string"},"json_output":{"type":"boolean"},"json_include_output_property":{"type":"boolean"},"json_include_tags_property":{"type":"boolean"},"buffered_outputs":{"type":"boolean"},"rule_matching":{"type":"string"},"outputs_queue":{"$ref":"#/definitions/OutputsQueue"},"stdout_output":{"$ref":"#/definitions/Output"},"syslog_output":{"$ref":"#/definitions/Output"},"file_output":{"$ref":"#/definitions/FileOutput"},"http_output":{"$ref":"#/definitions/HTTPOutput"},"program_output":{"$ref":"#/definitions/ProgramOutput"},"grpc_output":{"$ref":"#/definitions/Output"},"grpc":{"$ref":"#/definitions/Grpc"},"webserver":{"$ref":"#/definitions/Webserver"},"log_stderr":{"type":"boolean"},"log_syslog":{"type":"boolean"},"log_level":{"type":"string"},"libs_logger":{"$ref":"#/definitions/LibsLogger"},"output_timeout":{"type":"integer"},"syscall_event_timeouts":{"$ref":"#/definitions/SyscallEventTimeouts"},"syscall_event_drops":{"$ref":"#/definitions/SyscallEventDrops"},"metrics":{"$ref":"#/definitions/Metrics"},"base_syscalls":{"$ref":"#/definitions/BaseSyscalls"},"falco_libs":{"$ref":"#/definitions/FalcoLibs"},"container_engines":{"type":"object","additionalProperties":false,"properties":{"docker":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"cri":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"sockets":{"type":"array","items":{"type":"string"}},"disable_async":{"type":"boolean"}}},"podman":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"libvirt_lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"bpm":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}}}}},"title":"FalcoConfig"},"AppendOutput":{"type":"object","additionalProperties":false,"properties":{"source":{"type":"string"},"tag":{"type":"string"},"rule":{"type":"string"},"format":{"type":"string"},"fields":{"type":"array","items":{"anyOf":[{"type":"object","additionalProperties":{"type":"string"}},{"type":"string"}]}}}},"BaseSyscalls":{"type":"object","additionalProperties":false,"properties":{"custom_set":{"type":"array","items":{"type":"string"}},"repair":{"type":"boolean"}},"minProperties":1,"title":"BaseSyscalls"},"Engine":{"type":"object","additionalProperties":false,"properties":{"kind":{"type":"string"},"kmod":{"$ref":"#/definitions/Kmod"},"ebpf":{"$ref":"#/definitions/Ebpf"},"modern_ebpf":{"$ref":"#/definitions/ModernEbpf"},"replay":{"$ref":"#/definitions/Replay"},"gvisor":{"$ref":"#/definitions/Gvisor"}},"required":["kind"],"title":"Engine"},"Ebpf":{"type":"object","additionalProperties":false,"properties":{"probe":{"type":"string"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"required":["probe"],"title":"Ebpf"},"Gvisor":{"type":"object","additionalProperties":false,"properties":{"config":{"type":"string"},"root":{"type":"string"}},"required":["config","root"],"title":"Gvisor"},"Kmod":{"type":"object","additionalProperties":false,"properties":{"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"minProperties":1,"title":"Kmod"},"ModernEbpf":{"type":"object","additionalProperties":false,"properties":{"cpus_for_each_buffer":{"type":"integer"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"title":"ModernEbpf"},"Replay":{"type":"object","additionalProperties":false,"properties":{"capture_file":{"type":"string"}},"required":["capture_file"],"title":"Replay"},"FalcoLibs":{"type":"object","additionalProperties":false,"properties":{"thread_table_size":{"type":"integer"}},"minProperties":1,"title":"FalcoLibs"},"FileOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"filename":{"type":"string"}},"minProperties":1,"title":"FileOutput"},"Grpc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"bind_address":{"type":"string"},"threadiness":{"type":"integer"}},"minProperties":1,"title":"Grpc"},"Output":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}},"minProperties":1,"title":"Output"},"HTTPOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"url":{"type":"string","format":"uri","qt-uri-protocols":["http"]},"user_agent":{"type":"string"},"insecure":{"type":"boolean"},"ca_cert":{"type":"string"},"ca_bundle":{"type":"string"},"ca_path":{"type":"string"},"mtls":{"type":"boolean"},"client_cert":{"type":"string"},"client_key":{"type":"string"},"echo":{"type":"boolean"},"compress_uploads":{"type":"boolean"},"keep_alive":{"type":"boolean"}},"minProperties":1,"title":"HTTPOutput"},"LibsLogger":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"severity":{"type":"string"}},"minProperties":1,"title":"LibsLogger"},"Metrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"interval":{"type":"string"},"output_rule":{"type":"boolean"},"output_file":{"type":"string"},"rules_counters_enabled":{"type":"boolean"},"resource_utilization_enabled":{"type":"boolean"},"state_counters_enabled":{"type":"boolean"},"kernel_event_counters_enabled":{"type":"boolean"},"libbpf_stats_enabled":{"type":"boolean"},"plugins_metrics_enabled":{"type":"boolean"},"convert_memory_to_mb":{"type":"boolean"},"include_empty_values":{"type":"boolean"}},"minProperties":1,"title":"Metrics"},"OutputsQueue":{"type":"object","additionalProperties":false,"properties":{"capacity":{"type":"integer"}},"minProperties":1,"title":"OutputsQueue"},"Plugin":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"library_path":{"type":"string"},"init_config":{"type":"string"},"open_params":{"type":"string"}},"required":["library_path","name"],"title":"Plugin"},"ProgramOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"program":{"type":"string"}},"required":["program"],"title":"ProgramOutput"},"Rule":{"type":"object","additionalProperties":false,"properties":{"disable":{"$ref":"#/definitions/Able"},"enable":{"$ref":"#/definitions/Able"}},"minProperties":1,"title":"Rule"},"Able":{"type":"object","additionalProperties":false,"properties":{"rule":{"type":"string"},"tag":{"type":"string"}},"minProperties":1,"title":"Able"},"SyscallEventDrops":{"type":"object","additionalProperties":false,"properties":{"threshold":{"type":"number"},"actions":{"type":"array","items":{"type":"string"}},"rate":{"type":"number"},"max_burst":{"type":"integer"},"simulate_drops":{"type":"boolean"}},"minProperties":1,"title":"SyscallEventDrops"},"SyscallEventTimeouts":{"type":"object","additionalProperties":false,"properties":{"max_consecutives":{"type":"integer"}},"minProperties":1,"title":"SyscallEventTimeouts"},"Webserver":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"threadiness":{"type":"integer"},"listen_port":{"type":"integer"},"listen_address":{"type":"string"},"k8s_healthz_endpoint":{"type":"string"},"prometheus_metrics_enabled":{"type":"boolean"},"ssl_enabled":{"type":"boolean"},"ssl_certificate":{"type":"string"}},"minProperties":1,"title":"Webserver"}}})"; falco_configuration::falco_configuration(): m_json_output(false), From 606f2d9f774923dafb234a8037ed4a6f323b04db Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Thu, 29 Aug 2024 16:51:56 +0000 Subject: [PATCH 3/7] new(falco): add append_output explanation to falco.yaml Signed-off-by: Luca Guerra --- falco.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/falco.yaml b/falco.yaml index 5d03036ec20..6dd8339d726 100644 --- a/falco.yaml +++ b/falco.yaml @@ -576,6 +576,42 @@ rule_matching: first outputs_queue: capacity: 0 +# [Sandbox] `append_output` +# +# Add information to the Falco output. +# With this setting you can add more information to the Falco output message, customizable by +# rule, tag or source. +# You can also add additional data that will appear in the output_fields property +# of JSON formatted messages or gRPC output but will not be part of the regular output message. +# This allows you to add custom fields that can help you filter your Falco events without +# polluting the message text. +# +# Each append_output entry has optional fields (ANDed together) to filter events: +# `rule`: append output only to a specific rule +# `source`: append output only to a specific source +# `tag`: append output only to a specific tag +# If none of the above are specified output is appended to all events, if more than one is +# specified output will be appended to events that match all conditions. +# And several options to add output: +# `format`: add output to the Falco message +# `fields`: add new fields to the JSON output and structured output, which will not +# affect the regular Falco message in any way. These can be specified as a +# custom name with a custom format or as any supported field +# (see: https://falco.org/docs/reference/rules/supported-fields/) +# +# Example: +# +# - source: syscall +# format: "on CPU %evt.cpu" +# fields: +# - home_directory: "${HOME}" +# - evt.hostname +# +# In the example above every event coming from the syscall source will get an extra message +# at the end telling the CPU number. In addition, if `json_output` is true, in the "output_fields" +# property you will find three new ones: "evt.cpu", "home_directory" which will contain the value of the +# environment variable $HOME, and "evt.hostname" which will contain the hostname. + ########################## # Falco outputs channels # From 69a23be6e7063467ed1ed5d7b9842e9dc427c9a2 Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Thu, 29 Aug 2024 16:03:18 +0000 Subject: [PATCH 4/7] update(tests): add message for failing configuration schema test Signed-off-by: Luca Guerra --- unit_tests/falco/test_configuration_schema.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_tests/falco/test_configuration_schema.cpp b/unit_tests/falco/test_configuration_schema.cpp index b6eef499e9e..b0af76fc4e5 100644 --- a/unit_tests/falco/test_configuration_schema.cpp +++ b/unit_tests/falco/test_configuration_schema.cpp @@ -23,7 +23,7 @@ limitations under the License. do { \ for(const auto& pair : res) { \ auto validation_status = pair.second; \ - EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)); \ + EXPECT_TRUE(sinsp_utils::startswith(validation_status, status)) << validation_status; \ } \ } \ while (0) From 766b459e6f84f9dab8a50ef99bbf6c0f1b8eb8f4 Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Fri, 6 Sep 2024 16:02:50 +0000 Subject: [PATCH 5/7] fix(falco): update json schema Signed-off-by: Luca Guerra --- userspace/falco/configuration.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/userspace/falco/configuration.cpp b/userspace/falco/configuration.cpp index 4e200e06838..b936b81cbc2 100644 --- a/userspace/falco/configuration.cpp +++ b/userspace/falco/configuration.cpp @@ -53,7 +53,7 @@ static re2::RE2 ip_address_re("((^\\s*((([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]| // https://learn.microsoft.com/en-us/cpp/cpp/string-and-character-literals-cpp?view=msvc-170#size-of-string-literals // Just use any available online tool, eg: https://jsonformatter.org/json-minify // to format the json, add the new fields, and then minify it again. -static const std::string schema_json_string = R"({"$schema":"http://json-schema.org/draft-06/schema#","$ref":"#/definitions/FalcoConfig","definitions":{"FalcoConfig":{"type":"object","additionalProperties":false,"properties":{"append_output":{"$ref":"#/definitions/AppendOutput"},"config_files":{"type":"array","items":{"type":"string"}},"watch_config_files":{"type":"boolean"},"rules_files":{"type":"array","items":{"type":"string"}},"rule_files":{"type":"array","items":{"type":"string"}},"rules":{"type":"array","items":{"$ref":"#/definitions/Rule"}},"engine":{"$ref":"#/definitions/Engine"},"load_plugins":{"type":"array","items":{"type":"string"}},"plugins":{"type":"array","items":{"$ref":"#/definitions/Plugin"}},"time_format_iso_8601":{"type":"boolean"},"priority":{"type":"string"},"json_output":{"type":"boolean"},"json_include_output_property":{"type":"boolean"},"json_include_tags_property":{"type":"boolean"},"buffered_outputs":{"type":"boolean"},"rule_matching":{"type":"string"},"outputs_queue":{"$ref":"#/definitions/OutputsQueue"},"stdout_output":{"$ref":"#/definitions/Output"},"syslog_output":{"$ref":"#/definitions/Output"},"file_output":{"$ref":"#/definitions/FileOutput"},"http_output":{"$ref":"#/definitions/HTTPOutput"},"program_output":{"$ref":"#/definitions/ProgramOutput"},"grpc_output":{"$ref":"#/definitions/Output"},"grpc":{"$ref":"#/definitions/Grpc"},"webserver":{"$ref":"#/definitions/Webserver"},"log_stderr":{"type":"boolean"},"log_syslog":{"type":"boolean"},"log_level":{"type":"string"},"libs_logger":{"$ref":"#/definitions/LibsLogger"},"output_timeout":{"type":"integer"},"syscall_event_timeouts":{"$ref":"#/definitions/SyscallEventTimeouts"},"syscall_event_drops":{"$ref":"#/definitions/SyscallEventDrops"},"metrics":{"$ref":"#/definitions/Metrics"},"base_syscalls":{"$ref":"#/definitions/BaseSyscalls"},"falco_libs":{"$ref":"#/definitions/FalcoLibs"},"container_engines":{"type":"object","additionalProperties":false,"properties":{"docker":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"cri":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"sockets":{"type":"array","items":{"type":"string"}},"disable_async":{"type":"boolean"}}},"podman":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"libvirt_lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"bpm":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}}}}},"title":"FalcoConfig"},"AppendOutput":{"type":"object","additionalProperties":false,"properties":{"source":{"type":"string"},"tag":{"type":"string"},"rule":{"type":"string"},"format":{"type":"string"},"fields":{"type":"array","items":{"anyOf":[{"type":"object","additionalProperties":{"type":"string"}},{"type":"string"}]}}}},"BaseSyscalls":{"type":"object","additionalProperties":false,"properties":{"custom_set":{"type":"array","items":{"type":"string"}},"repair":{"type":"boolean"}},"minProperties":1,"title":"BaseSyscalls"},"Engine":{"type":"object","additionalProperties":false,"properties":{"kind":{"type":"string"},"kmod":{"$ref":"#/definitions/Kmod"},"ebpf":{"$ref":"#/definitions/Ebpf"},"modern_ebpf":{"$ref":"#/definitions/ModernEbpf"},"replay":{"$ref":"#/definitions/Replay"},"gvisor":{"$ref":"#/definitions/Gvisor"}},"required":["kind"],"title":"Engine"},"Ebpf":{"type":"object","additionalProperties":false,"properties":{"probe":{"type":"string"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"required":["probe"],"title":"Ebpf"},"Gvisor":{"type":"object","additionalProperties":false,"properties":{"config":{"type":"string"},"root":{"type":"string"}},"required":["config","root"],"title":"Gvisor"},"Kmod":{"type":"object","additionalProperties":false,"properties":{"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"minProperties":1,"title":"Kmod"},"ModernEbpf":{"type":"object","additionalProperties":false,"properties":{"cpus_for_each_buffer":{"type":"integer"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"title":"ModernEbpf"},"Replay":{"type":"object","additionalProperties":false,"properties":{"capture_file":{"type":"string"}},"required":["capture_file"],"title":"Replay"},"FalcoLibs":{"type":"object","additionalProperties":false,"properties":{"thread_table_size":{"type":"integer"}},"minProperties":1,"title":"FalcoLibs"},"FileOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"filename":{"type":"string"}},"minProperties":1,"title":"FileOutput"},"Grpc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"bind_address":{"type":"string"},"threadiness":{"type":"integer"}},"minProperties":1,"title":"Grpc"},"Output":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}},"minProperties":1,"title":"Output"},"HTTPOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"url":{"type":"string","format":"uri","qt-uri-protocols":["http"]},"user_agent":{"type":"string"},"insecure":{"type":"boolean"},"ca_cert":{"type":"string"},"ca_bundle":{"type":"string"},"ca_path":{"type":"string"},"mtls":{"type":"boolean"},"client_cert":{"type":"string"},"client_key":{"type":"string"},"echo":{"type":"boolean"},"compress_uploads":{"type":"boolean"},"keep_alive":{"type":"boolean"}},"minProperties":1,"title":"HTTPOutput"},"LibsLogger":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"severity":{"type":"string"}},"minProperties":1,"title":"LibsLogger"},"Metrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"interval":{"type":"string"},"output_rule":{"type":"boolean"},"output_file":{"type":"string"},"rules_counters_enabled":{"type":"boolean"},"resource_utilization_enabled":{"type":"boolean"},"state_counters_enabled":{"type":"boolean"},"kernel_event_counters_enabled":{"type":"boolean"},"libbpf_stats_enabled":{"type":"boolean"},"plugins_metrics_enabled":{"type":"boolean"},"convert_memory_to_mb":{"type":"boolean"},"include_empty_values":{"type":"boolean"}},"minProperties":1,"title":"Metrics"},"OutputsQueue":{"type":"object","additionalProperties":false,"properties":{"capacity":{"type":"integer"}},"minProperties":1,"title":"OutputsQueue"},"Plugin":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"library_path":{"type":"string"},"init_config":{"type":"string"},"open_params":{"type":"string"}},"required":["library_path","name"],"title":"Plugin"},"ProgramOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"program":{"type":"string"}},"required":["program"],"title":"ProgramOutput"},"Rule":{"type":"object","additionalProperties":false,"properties":{"disable":{"$ref":"#/definitions/Able"},"enable":{"$ref":"#/definitions/Able"}},"minProperties":1,"title":"Rule"},"Able":{"type":"object","additionalProperties":false,"properties":{"rule":{"type":"string"},"tag":{"type":"string"}},"minProperties":1,"title":"Able"},"SyscallEventDrops":{"type":"object","additionalProperties":false,"properties":{"threshold":{"type":"number"},"actions":{"type":"array","items":{"type":"string"}},"rate":{"type":"number"},"max_burst":{"type":"integer"},"simulate_drops":{"type":"boolean"}},"minProperties":1,"title":"SyscallEventDrops"},"SyscallEventTimeouts":{"type":"object","additionalProperties":false,"properties":{"max_consecutives":{"type":"integer"}},"minProperties":1,"title":"SyscallEventTimeouts"},"Webserver":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"threadiness":{"type":"integer"},"listen_port":{"type":"integer"},"listen_address":{"type":"string"},"k8s_healthz_endpoint":{"type":"string"},"prometheus_metrics_enabled":{"type":"boolean"},"ssl_enabled":{"type":"boolean"},"ssl_certificate":{"type":"string"}},"minProperties":1,"title":"Webserver"}}})"; +static const std::string schema_json_string = R"({"$schema":"http://json-schema.org/draft-06/schema#","$ref":"#/definitions/FalcoConfig","definitions":{"FalcoConfig":{"type":"object","additionalProperties":false,"properties":{"append_output":{"type":"array","items":{"$ref":"#/definitions/AppendOutput"}},"config_files":{"type":"array","items":{"type":"string"}},"watch_config_files":{"type":"boolean"},"rules_files":{"type":"array","items":{"type":"string"}},"rule_files":{"type":"array","items":{"type":"string"}},"rules":{"type":"array","items":{"$ref":"#/definitions/Rule"}},"engine":{"$ref":"#/definitions/Engine"},"load_plugins":{"type":"array","items":{"type":"string"}},"plugins":{"type":"array","items":{"$ref":"#/definitions/Plugin"}},"time_format_iso_8601":{"type":"boolean"},"priority":{"type":"string"},"json_output":{"type":"boolean"},"json_include_output_property":{"type":"boolean"},"json_include_tags_property":{"type":"boolean"},"buffered_outputs":{"type":"boolean"},"rule_matching":{"type":"string"},"outputs_queue":{"$ref":"#/definitions/OutputsQueue"},"stdout_output":{"$ref":"#/definitions/Output"},"syslog_output":{"$ref":"#/definitions/Output"},"file_output":{"$ref":"#/definitions/FileOutput"},"http_output":{"$ref":"#/definitions/HTTPOutput"},"program_output":{"$ref":"#/definitions/ProgramOutput"},"grpc_output":{"$ref":"#/definitions/Output"},"grpc":{"$ref":"#/definitions/Grpc"},"webserver":{"$ref":"#/definitions/Webserver"},"log_stderr":{"type":"boolean"},"log_syslog":{"type":"boolean"},"log_level":{"type":"string"},"libs_logger":{"$ref":"#/definitions/LibsLogger"},"output_timeout":{"type":"integer"},"syscall_event_timeouts":{"$ref":"#/definitions/SyscallEventTimeouts"},"syscall_event_drops":{"$ref":"#/definitions/SyscallEventDrops"},"metrics":{"$ref":"#/definitions/Metrics"},"base_syscalls":{"$ref":"#/definitions/BaseSyscalls"},"falco_libs":{"$ref":"#/definitions/FalcoLibs"},"container_engines":{"type":"object","additionalProperties":false,"properties":{"docker":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"cri":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"sockets":{"type":"array","items":{"type":"string"}},"disable_async":{"type":"boolean"}}},"podman":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"libvirt_lxc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}},"bpm":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}}}}}},"title":"FalcoConfig"},"AppendOutput":{"type":"object","additionalProperties":false,"properties":{"source":{"type":"string"},"tag":{"type":"string"},"rule":{"type":"string"},"format":{"type":"string"},"fields":{"type":"array","items":{"anyOf":[{"type":"object","additionalProperties":{"type":"string"}},{"type":"string"}]}}}},"BaseSyscalls":{"type":"object","additionalProperties":false,"properties":{"custom_set":{"type":"array","items":{"type":"string"}},"repair":{"type":"boolean"}},"minProperties":1,"title":"BaseSyscalls"},"Engine":{"type":"object","additionalProperties":false,"properties":{"kind":{"type":"string"},"kmod":{"$ref":"#/definitions/Kmod"},"ebpf":{"$ref":"#/definitions/Ebpf"},"modern_ebpf":{"$ref":"#/definitions/ModernEbpf"},"replay":{"$ref":"#/definitions/Replay"},"gvisor":{"$ref":"#/definitions/Gvisor"}},"required":["kind"],"title":"Engine"},"Ebpf":{"type":"object","additionalProperties":false,"properties":{"probe":{"type":"string"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"required":["probe"],"title":"Ebpf"},"Gvisor":{"type":"object","additionalProperties":false,"properties":{"config":{"type":"string"},"root":{"type":"string"}},"required":["config","root"],"title":"Gvisor"},"Kmod":{"type":"object","additionalProperties":false,"properties":{"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"minProperties":1,"title":"Kmod"},"ModernEbpf":{"type":"object","additionalProperties":false,"properties":{"cpus_for_each_buffer":{"type":"integer"},"buf_size_preset":{"type":"integer"},"drop_failed_exit":{"type":"boolean"}},"title":"ModernEbpf"},"Replay":{"type":"object","additionalProperties":false,"properties":{"capture_file":{"type":"string"}},"required":["capture_file"],"title":"Replay"},"FalcoLibs":{"type":"object","additionalProperties":false,"properties":{"thread_table_size":{"type":"integer"}},"minProperties":1,"title":"FalcoLibs"},"FileOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"filename":{"type":"string"}},"minProperties":1,"title":"FileOutput"},"Grpc":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"bind_address":{"type":"string"},"threadiness":{"type":"integer"}},"minProperties":1,"title":"Grpc"},"Output":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"}},"minProperties":1,"title":"Output"},"HTTPOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"url":{"type":"string","format":"uri","qt-uri-protocols":["http"]},"user_agent":{"type":"string"},"insecure":{"type":"boolean"},"ca_cert":{"type":"string"},"ca_bundle":{"type":"string"},"ca_path":{"type":"string"},"mtls":{"type":"boolean"},"client_cert":{"type":"string"},"client_key":{"type":"string"},"echo":{"type":"boolean"},"compress_uploads":{"type":"boolean"},"keep_alive":{"type":"boolean"}},"minProperties":1,"title":"HTTPOutput"},"LibsLogger":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"severity":{"type":"string"}},"minProperties":1,"title":"LibsLogger"},"Metrics":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"interval":{"type":"string"},"output_rule":{"type":"boolean"},"output_file":{"type":"string"},"rules_counters_enabled":{"type":"boolean"},"resource_utilization_enabled":{"type":"boolean"},"state_counters_enabled":{"type":"boolean"},"kernel_event_counters_enabled":{"type":"boolean"},"libbpf_stats_enabled":{"type":"boolean"},"plugins_metrics_enabled":{"type":"boolean"},"convert_memory_to_mb":{"type":"boolean"},"include_empty_values":{"type":"boolean"}},"minProperties":1,"title":"Metrics"},"OutputsQueue":{"type":"object","additionalProperties":false,"properties":{"capacity":{"type":"integer"}},"minProperties":1,"title":"OutputsQueue"},"Plugin":{"type":"object","additionalProperties":false,"properties":{"name":{"type":"string"},"library_path":{"type":"string"},"init_config":{"type":"string"},"open_params":{"type":"string"}},"required":["library_path","name"],"title":"Plugin"},"ProgramOutput":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"keep_alive":{"type":"boolean"},"program":{"type":"string"}},"required":["program"],"title":"ProgramOutput"},"Rule":{"type":"object","additionalProperties":false,"properties":{"disable":{"$ref":"#/definitions/Able"},"enable":{"$ref":"#/definitions/Able"}},"minProperties":1,"title":"Rule"},"Able":{"type":"object","additionalProperties":false,"properties":{"rule":{"type":"string"},"tag":{"type":"string"}},"minProperties":1,"title":"Able"},"SyscallEventDrops":{"type":"object","additionalProperties":false,"properties":{"threshold":{"type":"number"},"actions":{"type":"array","items":{"type":"string"}},"rate":{"type":"number"},"max_burst":{"type":"integer"},"simulate_drops":{"type":"boolean"}},"minProperties":1,"title":"SyscallEventDrops"},"SyscallEventTimeouts":{"type":"object","additionalProperties":false,"properties":{"max_consecutives":{"type":"integer"}},"minProperties":1,"title":"SyscallEventTimeouts"},"Webserver":{"type":"object","additionalProperties":false,"properties":{"enabled":{"type":"boolean"},"threadiness":{"type":"integer"},"listen_port":{"type":"integer"},"listen_address":{"type":"string"},"k8s_healthz_endpoint":{"type":"string"},"prometheus_metrics_enabled":{"type":"boolean"},"ssl_enabled":{"type":"boolean"},"ssl_certificate":{"type":"string"}},"minProperties":1,"title":"Webserver"}}})"; falco_configuration::falco_configuration(): m_json_output(false), From f6057b3020e2902c1a833b20ef23192063b11c4e Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Fri, 6 Sep 2024 16:15:43 +0000 Subject: [PATCH 6/7] cleanup(falco): apply review suggestion about extra_output_field_t Signed-off-by: Luca Guerra Co-authored-by: Federico Di Pierro --- userspace/engine/falco_common.h | 2 ++ userspace/engine/falco_engine.h | 2 +- userspace/engine/falco_rule.h | 2 +- userspace/engine/formats.cpp | 2 +- userspace/engine/formats.h | 2 +- userspace/falco/falco_outputs.cpp | 2 +- userspace/falco/falco_outputs.h | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) diff --git a/userspace/engine/falco_common.h b/userspace/engine/falco_common.h index 704ea044a29..95b2ef62efe 100644 --- a/userspace/engine/falco_common.h +++ b/userspace/engine/falco_common.h @@ -72,3 +72,5 @@ namespace falco_common bool parse_rule_matching(const std::string& v, rule_matching& out); }; + +typedef std::unordered_map> extra_output_field_t; diff --git a/userspace/engine/falco_engine.h b/userspace/engine/falco_engine.h index ce7c1f003eb..5af6d1c96b8 100644 --- a/userspace/engine/falco_engine.h +++ b/userspace/engine/falco_engine.h @@ -221,7 +221,7 @@ class falco_engine std::string format; std::set exception_fields; std::set tags; - std::unordered_map> extra_output_fields; + extra_output_field_t extra_output_fields; }; // diff --git a/userspace/engine/falco_rule.h b/userspace/engine/falco_rule.h index 10d1fb4e45c..f002646d26b 100644 --- a/userspace/engine/falco_rule.h +++ b/userspace/engine/falco_rule.h @@ -79,7 +79,7 @@ struct falco_rule std::string name; std::string description; std::string output; - std::unordered_map> extra_output_fields; + extra_output_field_t extra_output_fields; std::set tags; std::set exception_fields; falco_common::priority_type priority; diff --git a/userspace/engine/formats.cpp b/userspace/engine/formats.cpp index d4e7919eb40..4f4ff41b1ba 100644 --- a/userspace/engine/formats.cpp +++ b/userspace/engine/formats.cpp @@ -35,7 +35,7 @@ falco_formats::~falco_formats() std::string falco_formats::format_event(sinsp_evt *evt, const std::string &rule, const std::string &source, const std::string &level, const std::string &format, const std::set &tags, - const std::string &hostname, const std::unordered_map> &extra_fields) const + const std::string &hostname, const extra_output_field_t &extra_fields) const { std::string line; diff --git a/userspace/engine/formats.h b/userspace/engine/formats.h index 97d822612e5..e2d732580d3 100644 --- a/userspace/engine/formats.h +++ b/userspace/engine/formats.h @@ -31,7 +31,7 @@ class falco_formats std::string format_event(sinsp_evt *evt, const std::string &rule, const std::string &source, const std::string &level, const std::string &format, const std::set &tags, - const std::string &hostname, const std::unordered_map> &extra_fields) const; + const std::string &hostname, const extra_output_field_t &extra_fields) const; std::string format_string(sinsp_evt *evt, const std::string &format, const std::string &source) const; diff --git a/userspace/falco/falco_outputs.cpp b/userspace/falco/falco_outputs.cpp index dcbfb07eb89..8585d462cfa 100644 --- a/userspace/falco/falco_outputs.cpp +++ b/userspace/falco/falco_outputs.cpp @@ -128,7 +128,7 @@ void falco_outputs::add_output(const falco::outputs::config &oc) void falco_outputs::handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source, falco_common::priority_type priority, const std::string &format, std::set &tags, - std::unordered_map> &extra_fields) + extra_output_field_t &extra_fields) { falco_outputs::ctrl_msg cmsg = {}; cmsg.ts = evt->get_ts(); diff --git a/userspace/falco/falco_outputs.h b/userspace/falco/falco_outputs.h index 61de4be9536..c3c2cdbed9e 100644 --- a/userspace/falco/falco_outputs.h +++ b/userspace/falco/falco_outputs.h @@ -60,7 +60,7 @@ class falco_outputs */ void handle_event(sinsp_evt *evt, const std::string &rule, const std::string &source, falco_common::priority_type priority, const std::string &format, std::set &tags, - std::unordered_map> &extra_fields); + extra_output_field_t &extra_fields); /*! \brief Format then send a generic message to all outputs. From c481491548e945e9371a50b4a53716743b5b8d74 Mon Sep 17 00:00:00 2001 From: Luca Guerra Date: Fri, 6 Sep 2024 16:41:58 +0000 Subject: [PATCH 7/7] update(libsinsp/tests): add CLI options test for append output Signed-off-by: Luca Guerra --- .../test_configuration_output_options.cpp | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/unit_tests/falco/test_configuration_output_options.cpp b/unit_tests/falco/test_configuration_output_options.cpp index 6f956f71cbf..1d10decc173 100644 --- a/unit_tests/falco/test_configuration_output_options.cpp +++ b/unit_tests/falco/test_configuration_output_options.cpp @@ -66,3 +66,38 @@ TEST(ConfigurationRuleOutputOptions, parse_yaml) EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1); EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1); } + +TEST(ConfigurationRuleOutputOptions, cli_options) +{ + falco_configuration falco_config; + + ASSERT_NO_THROW(falco_config.init_from_content("", + std::vector{ + R"(append_output[]={"source": "syscall", "tag": "persistence", "rule": "some rule name", "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})", + R"(append_output[]={"tag": "persistence", "fields": [{"proc.aname[2]": "%proc.aname[2]"}, {"proc.aname[3]": "%proc.aname[3]"}, {"proc.aname[4]": "%proc.aname[4]"}], "format": "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"})", + R"(append_output[]={"source": "k8s_audit", "fields": ["ka.verb", {"static_field": "static content"}]})"})); + + EXPECT_EQ(falco_config.m_append_output.size(), 3); + + EXPECT_EQ(falco_config.m_append_output[0].m_source, "syscall"); + EXPECT_EQ(falco_config.m_append_output[0].m_tag, "persistence"); + EXPECT_EQ(falco_config.m_append_output[0].m_rule, "some rule name"); + EXPECT_EQ(falco_config.m_append_output[0].m_formatted_fields.size(), 0); + EXPECT_EQ(falco_config.m_append_output[0].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[1].m_tag, "persistence"); + EXPECT_EQ(falco_config.m_append_output[1].m_format, "gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields.size(), 3); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[2]"], "%proc.aname[2]"); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[3]"], "%proc.aname[3]"); + EXPECT_EQ(falco_config.m_append_output[1].m_formatted_fields["proc.aname[4]"], "%proc.aname[4]"); + + EXPECT_EQ(falco_config.m_append_output[2].m_source, "k8s_audit"); + + EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields.size(), 1); + EXPECT_EQ(falco_config.m_append_output[2].m_formatted_fields["static_field"], "static content"); + + EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.size(), 1); + EXPECT_EQ(falco_config.m_append_output[2].m_raw_fields.count("ka.verb"), 1); +}