From 11b4f739dc2866d3362661b3130b12e605347f5e Mon Sep 17 00:00:00 2001 From: Andrea Terzolo Date: Thu, 14 Sep 2023 17:46:31 +0200 Subject: [PATCH 1/3] new(libsinsp): extract `pod_uid` from cgroups Signed-off-by: Andrea Terzolo Co-authored-by: Aldo Lacuku --- userspace/libsinsp/filterchecks.cpp | 7 ++ userspace/libsinsp/parsers.cpp | 12 +++ .../test/classes/sinsp_threadinfo.cpp | 78 +++++++++++++++++++ userspace/libsinsp/test/filterchecks/k8s.cpp | 46 +++++++++++ userspace/libsinsp/test/state.ut.cpp | 2 +- userspace/libsinsp/threadinfo.cpp | 37 +++++++++ userspace/libsinsp/threadinfo.h | 9 +++ 7 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 userspace/libsinsp/test/filterchecks/k8s.cpp diff --git a/userspace/libsinsp/filterchecks.cpp b/userspace/libsinsp/filterchecks.cpp index 1d82d16e9d..e18cd91750 100644 --- a/userspace/libsinsp/filterchecks.cpp +++ b/userspace/libsinsp/filterchecks.cpp @@ -9052,6 +9052,13 @@ uint8_t* sinsp_filter_check_k8s::extract(sinsp_evt *evt, OUT uint32_t* len, bool return NULL; } m_tstr.clear(); + + // We can extract the pod_id directly from cgroups + if((m_field_id == TYPE_K8S_POD_ID) && !tinfo->m_pod_uid.empty()) + { + RETURN_EXTRACT_STRING(tinfo->m_pod_uid); + } + // there is metadata we can pull from the container directly instead of the k8s apiserver const sinsp_container_info::ptr_t container_info = m_inspector->m_container_manager.get_container(tinfo->m_container_id); diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index c48f3c0aae..a2a8c684d8 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -2154,6 +2154,12 @@ void sinsp_parser::parse_clone_exit_child(sinsp_evt *evt) child_tinfo->set_group(child_tinfo->m_group.gid); } + /* Set the pod uid if available + * OPTIMIZATION: we could use the same pod_uid of the parent instead + * of computing it again since the pod_uid should never change from parent to child. We need to check that we are not excluding some corner cases. + */ + child_tinfo->set_pod_uid(); + // // If there's a listener, invoke it // @@ -2742,6 +2748,12 @@ void sinsp_parser::parse_execve_exit(sinsp_evt *evt) evt->m_tinfo->set_group(evt->m_tinfo->m_group.gid); } + /* Set the pod uid if available + * We need to reinforce the pod_uid since it could change between the clone and + * the corresponding execve. + */ + evt->m_tinfo->set_pod_uid(); + // // If there's a listener, invoke it // diff --git a/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp b/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp index c023a1c928..685f4198a7 100644 --- a/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp +++ b/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp @@ -125,3 +125,81 @@ TEST_F(sinsp_with_test_input, THRD_INFO_assign_children_to_a_nullptr) ASSERT_THREAD_CHILDREN(p2_t1_tid, 0, 0); ASSERT_THREAD_INFO_PIDS(p3_t1_tid, p3_t1_pid, 0); } + +// This test asserts that our regex is solid against all possible cgroup layouts +TEST(sinsp_threadinfo, check_pod_uid_regex) +{ + // RGX_POD is defined in `threadinfo.h` + re2::RE2 pattern(RGX_POD, re2::RE2::POSIX); + + // CgroupV1, driver cgroup + std::string expected_pod_uid = "pod05869489-8c7f-45dc-9abd-1b1620787bb1"; + std::string actual_pod_uid = ""; + ASSERT_TRUE(re2::RE2::PartialMatch("/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", pattern, &actual_pod_uid)); + ASSERT_EQ(expected_pod_uid, actual_pod_uid); + + // CgroupV1, driver systemd + expected_pod_uid = "pod0f90f31c_ebeb_4192_a2b0_92e076c43817"; + ASSERT_TRUE(re2::RE2::PartialMatch("/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod0f90f31c_ebeb_4192_a2b0_92e076c43817.slice/4c97d83b89df14eea65dbbab1f506b405758341616ab75437d66fd8bab0e2beb", pattern, &actual_pod_uid)); + ASSERT_EQ(expected_pod_uid, actual_pod_uid); + + // CgroupV2, driver cgroup + expected_pod_uid = "podaf4fa4cf-129e-4699-a2af-65548fb8977d"; + ASSERT_TRUE(re2::RE2::PartialMatch("/kubepods/besteffort/podaf4fa4cf-129e-4699-a2af-65548fb8977d/fc16540dcd776bb475437b722c47de798fa1b07687db1ba7d4609c23d5d1a088", pattern, &actual_pod_uid)); + ASSERT_EQ(expected_pod_uid, actual_pod_uid); + + // CgroupV2, driver systemd + expected_pod_uid = "pod43f23404_e33c_48c7_8114_28ee4b7043ec"; + ASSERT_TRUE(re2::RE2::PartialMatch("/kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-pod43f23404_e33c_48c7_8114_28ee4b7043ec.slice/cri-containerd-b59ce319955234d0b051a93dac5efa8fc07df08d8b0188195b434174efc44e73.scope", pattern, &actual_pod_uid)); + ASSERT_EQ(expected_pod_uid, actual_pod_uid); + + // Not match, wrong pod_uid format + ASSERT_FALSE(re2::RE2::PartialMatch("cpuset=/kubepods/besteffort/pod05869489W-8c7fWW-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", pattern)); +} + +// This test asserts that our parsers (clone/execve) can extract the pod_uid from cgroups. +TEST_F(sinsp_with_test_input, THRD_INFO_extract_pod_uid) +{ + add_default_init_thread(); + open_inspector(); + + int64_t p1_tid = 2; + int64_t p1_pid = 2; + int64_t p1_ptid = INIT_TID; + int64_t p1_vtid = 1; + int64_t p1_vpid = 1; + + uint64_t not_relevant_64 = 0; + uint32_t not_relevant_32 = 0; + + // cgroupfs driver format + std::vector cgroups1 = { + "cpuset=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "cpu=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "cpuacct=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "io=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "memory=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "devices=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + "freezer=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + }; + std::string cgroupsv = test_utils::to_null_delimited(cgroups1); + scap_const_sized_buffer empty_bytebuf = {/*.buf =*/ nullptr, /*.size =*/ 0}; + auto evt = add_event_advance_ts(increasing_ts(), p1_tid, PPME_SYSCALL_CLONE_20_X, 21, (int64_t)0, "init", empty_bytebuf, p1_tid, p1_pid, p1_ptid, "", not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_32, not_relevant_32, not_relevant_32, "init", scap_const_sized_buffer{cgroupsv.data(), cgroupsv.size()}, (int32_t)0, not_relevant_32, not_relevant_32, p1_vtid, p1_vpid); + + ASSERT_TRUE(evt->get_thread_info()); + ASSERT_EQ(evt->get_thread_info()->m_pod_uid, "05869489-8c7f-45dc-9abd-1b1620787bb1"); + + // Now we simulate a change of cgroups in the execve event. + + // systemd driver format + // Only one cgroup subsystem is enough + std::vector cgroups2 = { + "cpuset=/kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-besteffort.slice/kubelet-kubepods-besteffort-pod47bf324a_ed3e_4a7c_be4a_7d119755bfcb.slice", + }; + + cgroupsv = test_utils::to_null_delimited(cgroups2); + evt = add_event_advance_ts(increasing_ts(), p1_tid, PPME_SYSCALL_EXECVE_19_X, 28, (int64_t)0, "/bin/new-prog", empty_bytebuf, p1_tid, p1_pid, p1_ptid, "", not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_32, not_relevant_32, not_relevant_32, "new-prog", scap_const_sized_buffer{cgroupsv.data(), cgroupsv.size()}, empty_bytebuf, not_relevant_32, not_relevant_32, not_relevant_32, (int32_t) PPM_EXE_WRITABLE, not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_64, "/bin/new-prog"); + + ASSERT_TRUE(evt->get_thread_info()); + ASSERT_EQ(evt->get_thread_info()->m_pod_uid, "47bf324a-ed3e-4a7c-be4a-7d119755bfcb"); +} diff --git a/userspace/libsinsp/test/filterchecks/k8s.cpp b/userspace/libsinsp/test/filterchecks/k8s.cpp new file mode 100644 index 0000000000..a73d436109 --- /dev/null +++ b/userspace/libsinsp/test/filterchecks/k8s.cpp @@ -0,0 +1,46 @@ +/* +Copyright (C) 2023 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +/* K8s filterchecks are not defined in the minimal build so we need this ifdef */ +#if !defined(CYGWING_AGENT) && !defined(MINIMAL_BUILD) && !defined(__EMSCRIPTEN__) +#include +TEST_F(sinsp_with_test_input, K8S_FILTER_pod_id) +{ + add_default_init_thread(); + open_inspector(); + + int64_t p1_tid = 2; + int64_t p1_pid = 2; + int64_t p1_ptid = INIT_TID; + int64_t p1_vtid = 1; + int64_t p1_vpid = 1; + + uint64_t not_relevant_64 = 0; + uint32_t not_relevant_32 = 0; + + // cgroupfs driver format + std::vector cgroups1 = { + "cpuset=/kubepods/besteffort/pod05869489-8c7f-45dc-9abd-1b1620787bb1/691e0ffb65010b2b611f3a15b7f76c48466192e673e156f38bd2f8e25acd6bbc", + }; + std::string cgroupsv = test_utils::to_null_delimited(cgroups1); + scap_const_sized_buffer empty_bytebuf = {/*.buf =*/ nullptr, /*.size =*/ 0}; + auto evt = add_event_advance_ts(increasing_ts(), p1_tid, PPME_SYSCALL_CLONE_20_X, 21, 0, "init", empty_bytebuf, p1_tid, p1_pid, p1_ptid, "", not_relevant_64, not_relevant_64, not_relevant_64, not_relevant_32, not_relevant_32, not_relevant_32, "init", scap_const_sized_buffer{cgroupsv.data(), cgroupsv.size()}, 0, not_relevant_32, not_relevant_32, p1_vtid, p1_vpid); + + ASSERT_EQ(get_field_as_string(evt, "k8s.pod.id"), "05869489-8c7f-45dc-9abd-1b1620787bb1"); +} + +#endif diff --git a/userspace/libsinsp/test/state.ut.cpp b/userspace/libsinsp/test/state.ut.cpp index 4a594a070a..7fee33a616 100644 --- a/userspace/libsinsp/test/state.ut.cpp +++ b/userspace/libsinsp/test/state.ut.cpp @@ -301,7 +301,7 @@ TEST(thread_manager, table_access) { // note: used for regression checks, keep this updated as we make // new fields available - static const int s_threadinfo_static_fields_count = 20; + static const int s_threadinfo_static_fields_count = 21; sinsp inspector; auto table = static_cast*>(inspector.m_thread_manager); diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index 9ea3120aeb..3d4e5fa9a4 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -74,6 +74,7 @@ sinsp_threadinfo::sinsp_threadinfo(sinsp* inspector, std::shared_ptrget_thread_ref(m_ptid, false).get(); } +static re2::RE2 pattern(RGX_POD, re2::RE2::POSIX); + +void sinsp_threadinfo::set_pod_uid() +{ + m_pod_uid = ""; + + // Set pod_uid to `""` if: + // 1. We don't have cgroups for this thread. + // 2. We are not in a container. + if(this->cgroups().empty() || !this->is_in_pid_namespace()) + { + return; + } + + // If `this->cgroups()` is not empty we should have at least the first element + // An example of `this->cgroups()[0].second` layout is: + // /kubelet.slice/kubelet-kubepods.slice/kubelet-kubepods-pod1b011081_6e8e_4839_b225_4685eae5fd59.slice/cri-containerd-2f92446a3fbfd0b7a73457b45e96c75a25c5e44e7b1bcec165712b906551c261.scope + if(!re2::RE2::PartialMatch(this->cgroups()[0].second, pattern, &m_pod_uid)) + { + return; + } + + // Here `m_pod_uid` could have 2 possible layouts: + // - (driver cgroup) pod05869489-8c7f-45dc-9abd-1b1620787bb1 + // - (driver systemd) pod05869489_8c7f_45dc_9abd_1b1620787bb1 + + // remove the "pod" prefix from `m_pod_uid` + m_pod_uid.erase(0, 3); + + // convert `_` into `-` if we are in `systemd` notation + std::replace(m_pod_uid.begin(), m_pod_uid.end(), '_', '-'); + + // The final `pod_uid` layout is: + // 05869489-8c7f-45dc-9abd-1b1620787bb1 +} + sinsp_fdinfo_t* sinsp_threadinfo::add_fd(int64_t fd, sinsp_fdinfo_t *fdinfo) { sinsp_fdtable* fd_table_ptr = get_fd_table(); diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 2f992cb883..eb39eeaecd 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -39,6 +39,9 @@ struct iovec { #include "internal_metrics.h" #include "state/table.h" #include "thread_group_info.h" +#include + +#define RGX_POD "(pod[a-z0-9]{8}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{12})" class sinsp_delays_info; class sinsp_tracerparser; @@ -238,6 +241,11 @@ class SINSP_PUBLIC sinsp_threadinfo: public libsinsp::state::table_entry return possible_main; } + /*! + \brief Extract the pod uid from the cgroups if they are available. + */ + void set_pod_uid(); + /*! \brief Get the process that launched this thread's process. */ @@ -417,6 +425,7 @@ class SINSP_PUBLIC sinsp_threadinfo: public libsinsp::state::table_entry std::vector m_env; ///< Environment variables std::unique_ptr m_cgroups; ///< subsystem-cgroup pairs std::string m_container_id; ///< heuristic-based container id + std::string m_pod_uid; ///< pod universally unique identifier extracted from cgroups. uint32_t m_flags; ///< The thread flags. See the PPM_CL_* declarations in ppm_events_public.h. int64_t m_fdlimit; ///< The maximum number of FDs this thread can open scap_userinfo m_user; ///< user infos From 25ae33524d1ef2b0db22e5ef7a3bb30da9d58a03 Mon Sep 17 00:00:00 2001 From: Andrea Terzolo Date: Tue, 3 Oct 2023 15:42:15 +0200 Subject: [PATCH 2/3] fix: fix build in shared library build Signed-off-by: Andrea Terzolo --- userspace/libsinsp/CMakeLists.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt index d92fc4fbeb..997a049541 100644 --- a/userspace/libsinsp/CMakeLists.txt +++ b/userspace/libsinsp/CMakeLists.txt @@ -200,11 +200,10 @@ endif() set_sinsp_target_properties(sinsp) target_link_libraries(sinsp - PUBLIC scap + PUBLIC scap "${RE2_LIB}" PRIVATE "${CURL_LIBRARIES}" "${JSONCPP_LIB}" - "${RE2_LIB}" ) set(SINSP_PKGCONFIG_LIBRARIES From b0fdb31c26256aa110a5ac6c395f845770952faa Mon Sep 17 00:00:00 2001 From: Andrea Terzolo Date: Tue, 3 Oct 2023 17:39:22 +0200 Subject: [PATCH 3/3] fix: fix build on windows Signed-off-by: Andrea Terzolo --- userspace/libsinsp/pod_regex.h | 20 +++++++++++++++++++ .../test/classes/sinsp_threadinfo.cpp | 3 ++- userspace/libsinsp/threadinfo.cpp | 3 +++ userspace/libsinsp/threadinfo.h | 3 --- 4 files changed, 25 insertions(+), 4 deletions(-) create mode 100644 userspace/libsinsp/pod_regex.h diff --git a/userspace/libsinsp/pod_regex.h b/userspace/libsinsp/pod_regex.h new file mode 100644 index 0000000000..80bdf03fd6 --- /dev/null +++ b/userspace/libsinsp/pod_regex.h @@ -0,0 +1,20 @@ +/* +Copyright (C) 2023 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +#pragma once + +#define RGX_POD "(pod[a-z0-9]{8}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{12})" diff --git a/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp b/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp index 685f4198a7..bebaab2cea 100644 --- a/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp +++ b/userspace/libsinsp/test/classes/sinsp_threadinfo.cpp @@ -16,6 +16,8 @@ limitations under the License. */ #include +#include +#include "pod_regex.h" TEST(sinsp_threadinfo, get_main_thread) { @@ -129,7 +131,6 @@ TEST_F(sinsp_with_test_input, THRD_INFO_assign_children_to_a_nullptr) // This test asserts that our regex is solid against all possible cgroup layouts TEST(sinsp_threadinfo, check_pod_uid_regex) { - // RGX_POD is defined in `threadinfo.h` re2::RE2 pattern(RGX_POD, re2::RE2::POSIX); // CgroupV1, driver cgroup diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index 3d4e5fa9a4..0fafa60d50 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -31,6 +31,9 @@ limitations under the License. #include "tracer_emitter.h" #endif +#include +#include "pod_regex.h" + constexpr static const char* s_thread_table_name = "threads"; extern sinsp_evttables g_infotables; diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index eb39eeaecd..62e1fa9881 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -39,9 +39,6 @@ struct iovec { #include "internal_metrics.h" #include "state/table.h" #include "thread_group_info.h" -#include - -#define RGX_POD "(pod[a-z0-9]{8}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{4}[-_][a-z0-9]{12})" class sinsp_delays_info; class sinsp_tracerparser;