From 513cfc715753bbfdb6682c69cfe31c19c713a1fc Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Thu, 19 Oct 2023 05:57:23 +0000 Subject: [PATCH 01/10] feat(libsinsp): move resource utilization to sinsp stats - Prepare m_sinsp_stats_v2 buffer to hold more stats and metrics than resource utilization metrics - Expand resource utilization metrics (e.g. include overall host CPU and memory usages) Signed-off-by: Melissa Kilby --- userspace/libsinsp/CMakeLists.txt | 3 +- userspace/libsinsp/sinsp.h | 4 +- .../libsinsp/sinsp_resource_utilization.cpp | 208 ----------- .../libsinsp/sinsp_resource_utilization.h | 47 --- userspace/libsinsp/stats.cpp | 333 ++++++++++++++++++ userspace/libsinsp/stats.h | 34 +- 6 files changed, 369 insertions(+), 260 deletions(-) delete mode 100644 userspace/libsinsp/sinsp_resource_utilization.cpp delete mode 100644 userspace/libsinsp/sinsp_resource_utilization.h diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt index 8529ce13a1..24ecedaa68 100644 --- a/userspace/libsinsp/CMakeLists.txt +++ b/userspace/libsinsp/CMakeLists.txt @@ -124,7 +124,6 @@ set(SINSP_SOURCES threadinfo.cpp tuples.cpp sinsp.cpp - stats.cpp token_bucket.cpp stopwatch.cpp utils.cpp @@ -170,7 +169,7 @@ if(NOT MINIMAL_BUILD AND NOT EMSCRIPTEN) list(APPEND SINSP_SOURCES cgroup_limits.cpp container_engine/cri.cpp - sinsp_resource_utilization.cpp + stats.cpp grpc_channel_registry.cpp ) endif() diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index 087a0b6324..c3cb58b013 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -73,7 +73,7 @@ limitations under the License. #include "container.h" #include "user.h" #include "utils.h" -#include "sinsp_resource_utilization.h" +#include "stats.h" #ifndef VISIBILITY_PRIVATE // Some code defines VISIBILITY_PRIVATE to nothing to get private access to sinsp @@ -1102,7 +1102,7 @@ VISIBILITY_PRIVATE bool m_is_dumping; const scap_machine_info* m_machine_info; const scap_agent_info* m_agent_info; - scap_stats_v2 m_sinsp_stats_v2[SINSP_MAX_RESOURCE_UTILIZATION]; + scap_stats_v2 m_sinsp_stats_v2[SINSP_MAX_STATS_V2]; uint32_t m_num_cpus; bool m_is_tracers_capture_enabled; bool m_flush_memory_dump; diff --git a/userspace/libsinsp/sinsp_resource_utilization.cpp b/userspace/libsinsp/sinsp_resource_utilization.cpp deleted file mode 100644 index cf1a350d89..0000000000 --- a/userspace/libsinsp/sinsp_resource_utilization.cpp +++ /dev/null @@ -1,208 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -/* -Copyright (C) 2023 The Falco Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ - -#include -#include -#include -#include -#include -#include "sinsp_resource_utilization.h" -#include "utils.h" -#include -#include "sinsp_public.h" -#include "strl.h" - -void get_rss_vsz_pss_memory(uint32_t &rss, uint32_t &vsz, uint32_t &pss) -{ - char line[512]; - - FILE* f = fopen("/proc/self/status", "r"); - if(!f) - { - ASSERT(false); - return; - } - - while(fgets(line, sizeof(line), f) != NULL) - { - if(strncmp(line, "VmSize:", 7) == 0) - { - sscanf(line, "VmSize: %u", &vsz); /* memory size returned in kb */ - } - else if(strncmp(line, "VmRSS:", 6) == 0) - { - sscanf(line, "VmRSS: %u", &rss); /* memory size returned in kb */ - } - } - fclose(f); - - f = fopen("/proc/self/smaps_rollup", "r"); - if(!f) - { - ASSERT(false); - return; - } - - while(fgets(line, sizeof(line), f) != NULL) - { - if(strncmp(line, "Pss:", 4) == 0) - { - sscanf(line, "Pss: %u", &pss); /* memory size returned in kb */ - break; - } - } - fclose(f); -} - -double get_cpu_usage(double start_time) -{ - double cpu_usage_perc = 0.0; - - struct tms time; - if (times (&time) == (clock_t) -1) - { - return 0; - } - - /* Number of clock ticks per second, often referred to as USER_HZ / jiffies. */ - long hz = 100; -#ifdef _SC_CLK_TCK - if ((hz = sysconf(_SC_CLK_TCK)) < 0) - { - ASSERT(false); - hz = 100; - } -#endif - - /* Current utime is amount of processor time in user mode of calling process. Convert to seconds. */ - double user_sec = (double)time.tms_utime / hz; - - /* Current stime is amount of time the calling process has been scheduled in kernel mode. Convert to seconds. */ - double system_sec = (double)time.tms_stime / hz; - - /* Current uptime of the host machine in seconds. - * /proc/uptime offers higher precision w/ 2 decimals. - */ - double machine_uptime_sec = 0; - char filepath[512]; - snprintf(filepath, sizeof(filepath), "%s/proc/uptime", scap_get_host_root()); - FILE* f = fopen(filepath, "r"); - if(!f) - { - ASSERT(false); - return 0; - } - - int matched = fscanf(f, "%lf", &machine_uptime_sec); - fclose(f); - - if (matched != 1) { - return 0; - } - - /* CPU usage as percentage is computed by dividing the time the process uses the CPU by the - * currently elapsed time of the calling process. Compare to `ps` linux util. */ - double elapsed_sec = machine_uptime_sec - start_time; - if (elapsed_sec > 0) - { - cpu_usage_perc = (double)100.0 * (user_sec + system_sec) / elapsed_sec; - cpu_usage_perc = std::round(cpu_usage_perc * 10.0) / 10.0; // round to 1 decimal - } - - return cpu_usage_perc; -} - -uint64_t get_container_memory_usage() -{ - /* In Kubernetes `container_memory_working_set_bytes` is the memory measure the OOM killer uses - * and values from `/sys/fs/cgroup/memory/memory.usage_in_bytes` are close enough. - * -> contrasted numbers from multiple sources in a real-life Kubernetes cluster. - * - * Please note that `kubectl top pod` numbers would reflect the sum of containers in a pod and - * typically libs clients (e.g. Falco) pods contain sidekick containers that use memory as well. - * This metric accounts only for the container with the security monitoring agent running. - */ - - const char* filepath = getenv(SINSP_AGENT_CGROUP_MEM_PATH_ENV_VAR); - if (filepath == nullptr) - { - filepath = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; - } - - FILE* f = fopen(filepath, "r"); - if(!f) - { - ASSERT(false); - return 0; - } - unsigned long long memory_used = 0; - - /* memory size returned in bytes */ - int fscanf_matched = fscanf(f, "%llu", &memory_used); - fclose(f); - - if (fscanf_matched != 1) { - return 0; - } - - return memory_used; -} - -const scap_stats_v2* libsinsp::resource_utilization::get_resource_utilization(const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc) -{ - if (!stats) - { - *nstats = 0; - *rc = SCAP_FAILURE; - return NULL; - } - - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d = get_cpu_usage(agent_info->start_time); - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, "cpu_usage_perc", STATS_NAME_MAX); - - uint32_t rss = 0; - uint32_t vsz = 0; - uint32_t pss = 0; - get_rss_vsz_pss_memory(rss, vsz, pss); - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32 = rss; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].name, "memory_rss", STATS_NAME_MAX); - - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32 = vsz; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].name, "memory_vsz", STATS_NAME_MAX); - - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].value.u32 = pss; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].name, "memory_pss", STATS_NAME_MAX); - - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].value.u64 = get_container_memory_usage(); - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].name, "container_memory_used", STATS_NAME_MAX); - - *nstats = SINSP_MAX_RESOURCE_UTILIZATION; - *rc = SCAP_SUCCESS; - - return stats; -} diff --git a/userspace/libsinsp/sinsp_resource_utilization.h b/userspace/libsinsp/sinsp_resource_utilization.h deleted file mode 100644 index 1101956253..0000000000 --- a/userspace/libsinsp/sinsp_resource_utilization.h +++ /dev/null @@ -1,47 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -/* -Copyright (C) 2023 The Falco Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ -#pragma once - -#include "utils.h" - -typedef enum sinsp_resource_utilization_stats { - SINSP_RESOURCE_UTILIZATION_CPU_PERC = 0, ///< Current CPU usage, `ps` like, unit: percentage of one CPU. - SINSP_RESOURCE_UTILIZATION_MEMORY_RSS, ///< Current RSS (Resident Set Size), unit: kb. - SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ, ///< Current VSZ (Virtual Memory Size), unit: kb. - SINSP_RESOURCE_UTILIZATION_MEMORY_PSS, ///< Current PSS (Proportional Set Size), unit: kb. - SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY, ///< Cgroup current memory used, default Kubernetes /sys/fs/cgroup/memory/memory.usage_in_bytes, unit: bytes. - SINSP_MAX_RESOURCE_UTILIZATION -}sinsp_resource_utilization_stats; - -namespace libsinsp { -namespace resource_utilization { - - /*! - \brief Retrieve current resource utilization metrics. - \param agent_info Pointer to a \ref scap_agent_info containing relevant constants from the agent start up moment. - \param stats Pointer to a \ref scap_stats_v2 pre-allocated sinsp stats v2 buffer w/ scap_stats_v2 schema. - \param nstats Pointer reflecting number of statistics in returned buffer - \param rc Pointer to return code - \note Intended to be called once every x hours. - - \return Pointer to a \ref scap_stats_v2 buffer filled with the current resource utilization metrics - */ - const scap_stats_v2* get_resource_utilization(const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc); - -} -} diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index fdd8ab7294..e8c6c9efe5 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -21,6 +21,339 @@ limitations under the License. //////////////////////////////////////////////////////////////////////////// #include "sinsp.h" #include "sinsp_int.h" +#include "sinsp_public.h" +#include +#include +#include +#include +#include +#include "stats.h" +#include "strl.h" + +void get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &memory_used_host, uint64_t &open_fds_host) +{ + FILE* f; + char filepath[512]; + char line[512]; + + /* + * Get memory usage of the agent itself (referred to as calling process meaning /proc/self/) + */ + + f = fopen("/proc/self/status", "r"); + if(!f) + { + ASSERT(false); + return; + } + + while(fgets(line, sizeof(line), f) != NULL) + { + if(strncmp(line, "VmSize:", 7) == 0) + { + sscanf(line, "VmSize: %u", &vsz); /* memory size returned in kb */ + } + else if(strncmp(line, "VmRSS:", 6) == 0) + { + sscanf(line, "VmRSS: %u", &rss); /* memory size returned in kb */ + } + } + fclose(f); + + f = fopen("/proc/self/smaps_rollup", "r"); + if(!f) + { + ASSERT(false); + return; + } + + while(fgets(line, sizeof(line), f) != NULL) + { + if(strncmp(line, "Pss:", 4) == 0) + { + sscanf(line, "Pss: %u", &pss); /* memory size returned in kb */ + break; + } + } + fclose(f); + + /* + * Get total host memory usage + */ + + snprintf(filepath, sizeof(filepath), "%s/proc/meminfo", scap_get_host_root()); + f = fopen(filepath, "r"); + if(!f) + { + ASSERT(false); + return; + } + + unsigned long long mem_total, mem_free, mem_buff, mem_cache = 0; + + while(fgets(line, sizeof(line), f) != NULL) + { + if(strncmp(line, "MemTotal:", 9) == 0) + { + sscanf(line, "MemTotal: %llu", &mem_total); /* memory size returned in kb */ + } + else if(strncmp(line, "MemFree:", 8) == 0) + { + sscanf(line, "MemFree: %llu", &mem_free); /* memory size returned in kb */ + } + else if(strncmp(line, "Buffers:", 8) == 0) + { + sscanf(line, "Buffers: %llu", &mem_buff); /* memory size returned in kb */ + } + else if(strncmp(line, "Cached:", 7) == 0) + { + sscanf(line, "Cached: %llu", &mem_cache); /* memory size returned in kb */ + } + } + fclose(f); + memory_used_host = mem_total - mem_free - mem_buff - mem_cache; + + /* + * Get total number of allocated file descriptors (not all open files!) + * File descriptor is a data structure used by a program to get a handle on a file + */ + + snprintf(filepath, sizeof(filepath), "%s/proc/sys/fs/file-nr", scap_get_host_root()); + f = fopen(filepath, "r"); + if(!f) + { + ASSERT(false); + return; + } + int matched_fds = fscanf(f, "%llu", &open_fds_host); + fclose(f); + + if (matched_fds != 1) { + ASSERT(false); + return; + } + +} + +void get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &cpu_usage_perc_total_host, uint32_t &procs_running_host) +{ + FILE* f; + char filepath[512]; + char line[512]; + + struct tms time; + if (times (&time) == (clock_t) -1) + { + return; + } + + /* Number of clock ticks per second, often referred to as USER_HZ / jiffies. */ + long hz = 100; +#ifdef _SC_CLK_TCK + if ((hz = sysconf(_SC_CLK_TCK)) < 0) + { + ASSERT(false); + hz = 100; + } +#endif + /* Current uptime of the host machine in seconds. + * /proc/uptime offers higher precision w/ 2 decimals. + */ + + snprintf(filepath, sizeof(filepath), "%s/proc/uptime", scap_get_host_root()); + f = fopen(filepath, "r"); + if(!f) + { + ASSERT(false); + return; + } + + double machine_uptime_sec = 0; + int matched_uptime = fscanf(f, "%lf", &machine_uptime_sec); + fclose(f); + + if (matched_uptime != 1) { + ASSERT(false); + return; + } + + /* + * Get CPU usage of the agent itself (referred to as calling process meaning /proc/self/) + */ + + /* Current utime is amount of processor time in user mode of calling process. Convert to seconds. */ + double user_sec = (double)time.tms_utime / hz; + + /* Current stime is amount of time the calling process has been scheduled in kernel mode. Convert to seconds. */ + double system_sec = (double)time.tms_stime / hz; + + + /* CPU usage as percentage is computed by dividing the time the process uses the CPU by the + * currently elapsed time of the calling process. Compare to `ps` linux util. */ + double elapsed_sec = machine_uptime_sec - start_time; + if (elapsed_sec > 0) + { + cpu_usage_perc = (double)100.0 * (user_sec + system_sec) / elapsed_sec; + cpu_usage_perc = std::round(cpu_usage_perc * 10.0) / 10.0; // round to 1 decimal + } + + /* + * Get total host CPU usage (all CPUs) as percentage and retrieve number of procs currently running. + */ + + snprintf(filepath, sizeof(filepath), "%s/proc/stat", scap_get_host_root()); + f = fopen(filepath, "r"); + if(!f) + { + ASSERT(false); + return; + } + + /* Need only first 7 columns of /proc/stat cpu line */ + unsigned long long user, nice, system, idle, iowait, irq, softirq = 0; + while(fgets(line, sizeof(line), f) != NULL) + { + if(strncmp(line, "cpu ", 4) == 0) + { + /* Always first line in /proc/stat file, unit: jiffies */ + sscanf(line, "cpu %llu %llu %llu %llu %llu %llu %llu", &user, &nice, &system, &idle, &iowait, &irq, &softirq); + } + else if(strncmp(line, "procs_running ", 14) == 0) + { + sscanf(line, "procs_running %u", &procs_running_host); + break; + } + } + fclose(f); + auto sum = user + nice + system + idle + iowait + irq + softirq; + if (sum > 0) + { + cpu_usage_perc_total_host = 100.0 - ((idle * 100.0) / sum); + cpu_usage_perc_total_host = std::round(cpu_usage_perc_total_host * 10.0) / 10.0; // round to 1 decimal + } + +} + +uint64_t get_container_memory_usage() +{ + /* In Kubernetes `container_memory_working_set_bytes` is the memory measure the OOM killer uses + * and values from `/sys/fs/cgroup/memory/memory.usage_in_bytes` are close enough. + * -> contrasted numbers from multiple sources in a real-life Kubernetes cluster. + * + * Please note that `kubectl top pod` numbers would reflect the sum of containers in a pod and + * typically libs clients (e.g. Falco) pods contain sidekick containers that use memory as well. + * This metric accounts only for the container with the security monitoring agent running. + */ + + const char* filepath = getenv(SINSP_AGENT_CGROUP_MEM_PATH_ENV_VAR); + if (filepath == nullptr) + { + filepath = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; + } + + FILE* f = fopen(filepath, "r"); + if(!f) + { + ASSERT(false); + return 0; + } + unsigned long long memory_used = 0; + + /* memory size returned in bytes */ + int fscanf_matched = fscanf(f, "%llu", &memory_used); + fclose(f); + + if (fscanf_matched != 1) { + return 0; + } + + return memory_used; +} + +const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc) +{ + if (!stats) + { + *nstats = 0; + *rc = SCAP_FAILURE; + return NULL; + } + + if((flags & PPM_SCAP_STATS_RESOURCE_UTILIZATION)) + { + uint32_t rss = 0; + uint32_t vsz = 0; + uint32_t pss = 0; + uint64_t memory_used_host = 0; + uint64_t open_fds_host = 0; + double cpu_usage_perc = 0.0; + double cpu_usage_perc_total_host = 0.0; + uint32_t procs_running_host = 0; + + if(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name != nullptr && strncmp(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, "cpu_usage_perc", 15) != 0) + { + // Init + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, "cpu_usage_perc", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].type = STATS_VALUE_TYPE_U32; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].name, "memory_rss", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].type = STATS_VALUE_TYPE_U32; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].name, "memory_vsz", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].type = STATS_VALUE_TYPE_U32; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].name, "memory_pss", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].type = STATS_VALUE_TYPE_U64; + stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].name, "container_memory_used", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].type = STATS_VALUE_TYPE_D; + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].name, "cpu_usage_perc_total_host", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].type = STATS_VALUE_TYPE_U32; + stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].name, "procs_running_host", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].name, "memory_used_host", STATS_NAME_MAX); + + stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; + stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].name, "open_fds_host", STATS_NAME_MAX); + } + + // Get stats / metrics snapshot + + get_cpu_usage_and_total_procs(agent_info->start_time, cpu_usage_perc, cpu_usage_perc_total_host, procs_running_host); + get_rss_vsz_pss_total_memory_and_open_fds(rss, vsz, pss, memory_used_host, open_fds_host); + + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d = cpu_usage_perc; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32 = rss; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32 = vsz; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].value.u32 = pss; + stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].value.u64 = get_container_memory_usage(); + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].value.d = cpu_usage_perc_total_host; + stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].value.u32 = procs_running_host; + stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].value.u64 = memory_used_host; + stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].value.u64 = open_fds_host; + + *nstats = SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1; + + } + + *rc = SCAP_SUCCESS; + + return stats; +} + #ifdef GATHER_INTERNAL_STATS diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index a38845e241..4da78e2c18 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -17,9 +17,41 @@ limitations under the License. */ #pragma once - +#include +#include #include "internal_metrics.h" +typedef enum sinsp_stats_v2 { + SINSP_RESOURCE_UTILIZATION_CPU_PERC = 0, ///< Current CPU usage, `ps` like, unit: percentage of one CPU. + SINSP_RESOURCE_UTILIZATION_MEMORY_RSS, ///< Current RSS (Resident Set Size), unit: kb. + SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ, ///< Current VSZ (Virtual Memory Size), unit: kb. + SINSP_RESOURCE_UTILIZATION_MEMORY_PSS, ///< Current PSS (Proportional Set Size), unit: kb. + SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY, ///< Cgroup current memory used, default Kubernetes /sys/fs/cgroup/memory/memory.usage_in_bytes, unit: bytes. + SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST, ///< Current total host CPU usage (all CPUs), unit: percentage. + SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST, ///< Current total memory used out of available host memory, unit: kb. + SINSP_RESOURCE_UTILIZATION_PROCS_HOST, ///< Number of processes currently running on CPUs on the host, unit: count. + SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST, ///< Number of allocated fds on the host, unit: count. + SINSP_MAX_STATS_V2 +}sinsp_stats_v2; + +namespace libsinsp { +namespace stats { + + /*! + \brief Retrieve current sinsp stats v2 including resource utilization metrics. + \param agent_info Pointer to a \ref scap_agent_info containing relevant constants from the agent start up moment. + \param stats Pointer to a \ref scap_stats_v2 pre-allocated sinsp stats v2 buffer w/ scap_stats_v2 schema. + \param nstats Pointer reflecting number of statistics in returned buffer + \param rc Pointer to return code + \note Intended to be called once every x hours. + + \return Pointer to a \ref scap_stats_v2 buffer filled with the current sinsp stats v2 including resource utilization metrics. + */ + const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc); + +} +} + #ifdef GATHER_INTERNAL_STATS // From 837df5505c0ee3350a90f7acb41162cfed4a4e69 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Thu, 19 Oct 2023 05:59:28 +0000 Subject: [PATCH 02/10] new(test): add sinsp_stats unit test Signed-off-by: Melissa Kilby --- userspace/libsinsp/test/CMakeLists.txt | 1 + userspace/libsinsp/test/sinsp_stats.ut.cpp | 82 ++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 userspace/libsinsp/test/sinsp_stats.ut.cpp diff --git a/userspace/libsinsp/test/CMakeLists.txt b/userspace/libsinsp/test/CMakeLists.txt index 9dacf395f8..d170c4464a 100644 --- a/userspace/libsinsp/test/CMakeLists.txt +++ b/userspace/libsinsp/test/CMakeLists.txt @@ -113,6 +113,7 @@ set(LIBSINSP_UNIT_TESTS_SOURCES state.ut.cpp eventformatter.ut.cpp savefile.ut.cpp + sinsp_stats.ut.cpp thread_table.ut.cpp public_sinsp_API/event_related.cpp public_sinsp_API/sinsp_logger.cpp diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp new file mode 100644 index 0000000000..fb62b0f15e --- /dev/null +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: Apache-2.0 +/* +Copyright (C) 2023 The Falco Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +*/ + +#include +#include "utils.h" +#include "sinsp_with_test_input.h" +#include "test_utils.h" +#include "scap_stats_v2.h" + + +TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) +{ +#ifdef __linux__ + + open_inspector(SCAP_MODE_LIVE); + const scap_agent_info* agent_info = m_inspector.get_agent_info(); + + uint32_t nstats = 0; + int32_t rc; + const scap_stats_v2* sinsp_stats_v2_snapshot; + auto buffer = m_inspector.get_sinsp_stats_v2_buffer(); + uint32_t flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); + /* Extra call */ + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); + + ASSERT_EQ(nstats, SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1); + ASSERT_EQ(rc, SCAP_SUCCESS); + + /* These names should always be available */ + std::unordered_set minimal_stats_names = {"cpu_usage_perc", "memory_rss", "memory_vsz", "memory_pss", "container_memory_used", "cpu_usage_perc_total_host", "procs_running_host", "memory_used_host", "open_fds_host"}; + + uint32_t i = 0; + for(const auto& stat_name : minimal_stats_names) + { + for(i = 0; i < nstats; i++) + { + if(stat_name.compare(sinsp_stats_v2_snapshot[i].name) == 0) + { + break; + } + } + + if(i == nstats) + { + FAIL() << "unable to find stat '" << stat_name << "' in the sinsp_stats_v2 buffer"; + } + } + + /* Assert values are greater than 0 */ + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].value.d, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].value.u64, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].value.u32, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].value.u64, 0); + + /* Empty call */ + nstats = 0; + flags = 0; + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); + ASSERT_EQ(nstats, 0); + ASSERT_EQ(rc, SCAP_SUCCESS); + +#endif +} From 71b395f8929402a46504b5ae97af45fe66ecf206 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Fri, 20 Oct 2023 21:34:28 +0000 Subject: [PATCH 03/10] cleanup(libsinsp): refactor legacy libsinsp stats - Finalize exposing valuable legacy libsinsp stats / metrics / counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities) to the consolidated libsinsp stats - Add counter of currently cached containers as well - Includes additional cleanups Signed-off-by: Melissa Kilby --- userspace/libscap/scap_stats_v2.h | 1 + userspace/libsinsp/fdinfo.cpp | 66 ++++++++++ userspace/libsinsp/fdinfo.h | 39 +----- userspace/libsinsp/parsers.cpp | 36 ++++++ userspace/libsinsp/sinsp.cpp | 21 +++ userspace/libsinsp/sinsp.h | 10 +- userspace/libsinsp/stats.cpp | 122 ++++++++++++++---- userspace/libsinsp/stats.h | 49 ++++++- userspace/libsinsp/test/filterchecks/proc.cpp | 2 +- userspace/libsinsp/test/sinsp_stats.ut.cpp | 32 +++-- userspace/libsinsp/threadinfo.cpp | 24 ++++ userspace/libsinsp/threadinfo.h | 1 - 12 files changed, 321 insertions(+), 82 deletions(-) diff --git a/userspace/libscap/scap_stats_v2.h b/userspace/libscap/scap_stats_v2.h index bc0105c35b..e587c8def0 100644 --- a/userspace/libscap/scap_stats_v2.h +++ b/userspace/libscap/scap_stats_v2.h @@ -37,6 +37,7 @@ extern "C" { #define PPM_SCAP_STATS_KERNEL_COUNTERS (1 << 0) #define PPM_SCAP_STATS_LIBBPF_STATS (1 << 1) #define PPM_SCAP_STATS_RESOURCE_UTILIZATION (1 << 2) +#define PPM_SCAP_STATS_STATE_COUNTERS (1 << 3) typedef union scap_stats_v2_value { uint32_t u32; diff --git a/userspace/libsinsp/fdinfo.cpp b/userspace/libsinsp/fdinfo.cpp index 571e856050..c693bf33cc 100644 --- a/userspace/libsinsp/fdinfo.cpp +++ b/userspace/libsinsp/fdinfo.cpp @@ -345,6 +345,58 @@ sinsp_fdtable::sinsp_fdtable(sinsp* inspector) reset_cache(); } +sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) +{ + std::unordered_map::iterator fdit; + + // + // Try looking up in our simple cache + // + if(m_last_accessed_fd != -1 && fd == m_last_accessed_fd) + { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_cached_fd_lookups++; + } + #ifdef GATHER_INTERNAL_STATS + m_inspector->m_stats->m_n_cached_fd_lookups++; + #endif + return m_last_accessed_fdinfo; + } + + // + // Caching failed, do a real lookup + // + fdit = m_table.find(fd); + + if(fdit == m_table.end()) + { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; + } + #ifdef GATHER_INTERNAL_STATS + m_inspector->m_stats->m_n_failed_fd_lookups++; + #endif + return NULL; + } + else + { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; + } + + #ifdef GATHER_INTERNAL_STATS + m_inspector->m_stats->m_n_noncached_fd_lookups++; + #endif + m_last_accessed_fd = fd; + m_last_accessed_fdinfo = &(fdit->second); + lookup_device(&(fdit->second), fd); + return &(fdit->second); + } +} + sinsp_fdinfo_t* sinsp_fdtable::add(int64_t fd, sinsp_fdinfo_t* fdinfo) { // @@ -365,6 +417,11 @@ sinsp_fdinfo_t* sinsp_fdtable::add(int64_t fd, sinsp_fdinfo_t* fdinfo) // No entry in the table, this is the normal case // m_last_accessed_fd = -1; + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_added_fds++; + } + #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_added_fds++; #endif @@ -435,6 +492,10 @@ void sinsp_fdtable::erase(int64_t fd) // keep going. // ASSERT(false); + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_failed_fd_lookups++; #endif @@ -442,6 +503,11 @@ void sinsp_fdtable::erase(int64_t fd) else { m_table.erase(fdit); + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; + m_inspector->m_sinsp_stats_v2.m_n_removed_fds++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_noncached_fd_lookups++; m_inspector->m_stats->m_n_removed_fds++; diff --git a/userspace/libsinsp/fdinfo.h b/userspace/libsinsp/fdinfo.h index 2b341429d6..77b230246b 100644 --- a/userspace/libsinsp/fdinfo.h +++ b/userspace/libsinsp/fdinfo.h @@ -532,44 +532,7 @@ class sinsp_fdtable public: sinsp_fdtable(sinsp* inspector); - inline sinsp_fdinfo_t* find(int64_t fd) - { - std::unordered_map::iterator fdit; - - // - // Try looking up in our simple cache - // - if(m_last_accessed_fd != -1 && fd == m_last_accessed_fd) - { - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_cached_fd_lookups++; - #endif - return m_last_accessed_fdinfo; - } - - // - // Caching failed, do a real lookup - // - fdit = m_table.find(fd); - - if(fdit == m_table.end()) - { - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_failed_fd_lookups++; - #endif - return NULL; - } - else - { - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_noncached_fd_lookups++; - #endif - m_last_accessed_fd = fd; - m_last_accessed_fdinfo = &(fdit->second); - lookup_device(&(fdit->second), fd); - return &(fdit->second); - } - } + sinsp_fdinfo_t* find(int64_t fd); // If the key is already present, overwrite the existing value and return false. sinsp_fdinfo_t* add(int64_t fd, sinsp_fdinfo_t* fdinfo); diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index 36aaf746dc..4183983934 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -734,6 +734,10 @@ bool sinsp_parser::reset(sinsp_evt *evt) etype == PPME_SYSCALL_VFORK_20_X || etype == PPME_SYSCALL_CLONE3_X) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups--; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_thread_manager->m_failed_lookups->decrement(); #endif @@ -882,6 +886,10 @@ void sinsp_parser::store_event(sinsp_evt *evt) // we won't be able to parse the corresponding exit event and we'll have // to drop the information it carries. // + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_store_evts_drops++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_store_drops++; #endif @@ -917,6 +925,10 @@ void sinsp_parser::store_event(sinsp_evt *evt) memcpy(tinfo->m_lastevent_data, evt->m_pevt, elen); tinfo->m_lastevent_cpuid = evt->get_cpuid(); + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_stored_evts++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_stored_evts++; #endif @@ -941,6 +953,10 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev // This happen especially at the beginning of trace files, where events // can be truncated // + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_retrieve_drops++; #endif @@ -963,6 +979,10 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev && enter_evt->get_type() == PPME_SYSCALL_EXECVEAT_E) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_retrieved_evts++; #endif @@ -977,11 +997,19 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev { //ASSERT(false); exit_evt->m_tinfo->set_lastevent_data_validity(false); + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_retrieve_drops++; #endif return false; } + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_retrieved_evts++; @@ -3778,11 +3806,19 @@ void sinsp_parser::parse_close_exit(sinsp_evt *evt) // It is normal when a close fails that the fd lookup failed, so we revert the // increment of m_n_failed_fd_lookups (for the enter event too if there's one). // + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_failed_fd_lookups--; #endif if(evt->m_tinfo && evt->m_tinfo->is_lastevent_data_valid()) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; + } #ifdef GATHER_INTERNAL_STATS m_inspector->m_stats->m_n_failed_fd_lookups--; #endif diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index 3950f832e2..d663fc686c 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -43,6 +43,7 @@ limitations under the License. #include "plugin_filtercheck.h" #include "strl.h" #include "scap-int.h" +#include "stats.h" #if defined(HAS_CAPTURE) && !defined(CYGWING_AGENT) && !defined(MINIMAL_BUILD) && !defined(__EMSCRIPTEN__) #include @@ -353,6 +354,21 @@ void sinsp::init() m_stats->clear(); #endif + m_sinsp_stats_v2.m_n_added_fds = 0; + m_sinsp_stats_v2.m_n_cached_fd_lookups = 0; + m_sinsp_stats_v2.m_n_failed_fd_lookups = 0; + m_sinsp_stats_v2.m_n_added_fds = 0; + m_sinsp_stats_v2.m_n_removed_fds = 0; + m_sinsp_stats_v2.m_n_stored_evts = 0; + m_sinsp_stats_v2.m_n_store_evts_drops = 0; + m_sinsp_stats_v2.m_n_retrieved_evts = 0; + m_sinsp_stats_v2.m_n_retrieve_evts_drops = 0; + m_sinsp_stats_v2.m_n_noncached_thread_lookups = 0; + m_sinsp_stats_v2.m_n_cached_thread_lookups = 0; + m_sinsp_stats_v2.m_n_failed_thread_lookups = 0; + m_sinsp_stats_v2.m_n_added_threads = 0; + m_sinsp_stats_v2.m_n_removed_threads = 0; + m_nevts = 0; m_tid_to_remove = -1; m_lastevent_ts = 0; @@ -2039,6 +2055,11 @@ const scap_agent_info* sinsp::get_agent_info() } scap_stats_v2* sinsp::get_sinsp_stats_v2_buffer() +{ + return m_sinsp_stats_v2_buffer; +} + +sinsp_stats_v2 sinsp::get_sinsp_stats_v2() { return m_sinsp_stats_v2; } diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index c3cb58b013..1a7e739e6f 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -503,6 +503,13 @@ class SINSP_PUBLIC sinsp : public capture_stats_source */ scap_stats_v2* get_sinsp_stats_v2_buffer(); + /*! + \brief Return sinsp stats v2 containing continually updated counters around thread and fd state tables. + + \note sinsp stats may be refactored near-term. + */ + sinsp_stats_v2 get_sinsp_stats_v2(); + /*! \brief Look up a thread given its tid and return its information, and optionally go dig into proc if the thread is not in the thread table. @@ -1102,7 +1109,8 @@ VISIBILITY_PRIVATE bool m_is_dumping; const scap_machine_info* m_machine_info; const scap_agent_info* m_agent_info; - scap_stats_v2 m_sinsp_stats_v2[SINSP_MAX_STATS_V2]; + sinsp_stats_v2 m_sinsp_stats_v2; + scap_stats_v2 m_sinsp_stats_v2_buffer[SINSP_MAX_STATS_V2]; uint32_t m_num_cpus; bool m_is_tracers_capture_enabled; bool m_flush_memory_dump; diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index e8c6c9efe5..4b24c35087 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -30,6 +30,35 @@ limitations under the License. #include "stats.h" #include "strl.h" +static const char *const sinsp_stats_v2_resource_utilization_names[] = { + [SINSP_RESOURCE_UTILIZATION_CPU_PERC] = "cpu_usage_perc", + [SINSP_RESOURCE_UTILIZATION_MEMORY_RSS] = "memory_rss", + [SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ] = "memory_vsz", + [SINSP_RESOURCE_UTILIZATION_MEMORY_PSS] = "memory_pss", + [SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY] = "container_memory_used", + [SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST] = "cpu_usage_perc_total_host", + [SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST] = "memory_used_host", + [SINSP_RESOURCE_UTILIZATION_PROCS_HOST] = "procs_running_host", + [SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST] = "open_fds_host", + [SINSP_STATS_V2_N_THREADS] = "n_threads", + [SINSP_STATS_V2_N_FDS] = "n_fds", + [SINSP_STATS_V2_NONCACHED_FD_LOOKUPS] = "n_noncached_fd_lookups", + [SINSP_STATS_V2_CACHED_FD_LOOKUPS] = "n_cached_fd_lookups", + [SINSP_STATS_V2_FAILED_FD_LOOKUPS] = "n_failed_fd_lookups", + [SINSP_STATS_V2_ADDED_FDS] = "n_added_fds", + [SINSP_STATS_V2_REMOVED_FDS] = "n_removed_fds", + [SINSP_STATS_V2_STORED_EVTS] = "n_stored_evts", + [SINSP_STATS_V2_STORE_EVTS_DROPS] = "n_store_evts_drops", + [SINSP_STATS_V2_RETRIEVED_EVTS] = "n_retrieved_evts", + [SINSP_STATS_V2_RETRIEVE_EVTS_DROPS] = "n_retrieve_evts_drops", + [SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS] = "n_noncached_thread_lookups", + [SINSP_STATS_V2_CACHED_THREAD_LOOKUPS] = "n_cached_thread_lookups", + [SINSP_STATS_V2_FAILED_THREAD_LOOKUPS] = "n_failed_thread_lookups", + [SINSP_STATS_V2_ADDED_THREADS] = "n_added_threads", + [SINSP_STATS_V2_REMOVED_THREADS] = "n_removed_threads", + [SINSP_STATS_V2_N_CONTAINERS] = "n_containers", +}; + void get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uint32_t &pss, uint64_t &memory_used_host, uint64_t &open_fds_host) { FILE* f; @@ -270,7 +299,7 @@ uint64_t get_container_memory_usage() return memory_used; } -const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc) +const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 sinsp_stats_v2_counters, scap_stats_v2* stats, uint32_t n_containers, uint32_t* nstats, int32_t* rc) { if (!stats) { @@ -290,44 +319,24 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s double cpu_usage_perc_total_host = 0.0; uint32_t procs_running_host = 0; - if(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name != nullptr && strncmp(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, "cpu_usage_perc", 15) != 0) + if(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name != nullptr && strncmp(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CPU_PERC], 15) != 0) { // Init - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, "cpu_usage_perc", STATS_NAME_MAX); + for(uint32_t i = SINSP_RESOURCE_UTILIZATION_CPU_PERC; i < SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1; i++) + { + stats[i].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(stats[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); + } + stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].name, "memory_rss", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].name, "memory_vsz", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].name, "memory_pss", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].name, "container_memory_used", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].type = STATS_VALUE_TYPE_D; - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].name, "cpu_usage_perc_total_host", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].name, "procs_running_host", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].name, "memory_used_host", STATS_NAME_MAX); - stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].name, "open_fds_host", STATS_NAME_MAX); } // Get stats / metrics snapshot @@ -348,6 +357,63 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s *nstats = SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1; } + + if((flags & PPM_SCAP_STATS_STATE_COUNTERS)) + { + if(stats[SINSP_STATS_V2_N_THREADS].name != nullptr && strncmp(stats[SINSP_STATS_V2_N_THREADS].name, sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], 10) != 0) + { + // Init + for(uint32_t i = SINSP_STATS_V2_N_THREADS; i < SINSP_MAX_STATS_V2; i++) + { + stats[i].flags = PPM_SCAP_STATS_STATE_COUNTERS; + strlcpy(stats[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); + } + + stats[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_CACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_FAILED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_ADDED_FDS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_REMOVED_FDS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_STORED_EVTS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_STORE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_RETRIEVED_EVTS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_ADDED_THREADS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_REMOVED_THREADS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_N_CONTAINERS].type = STATS_VALUE_TYPE_U32; + + } + + // Get stats / metrics snapshot + stats[SINSP_STATS_V2_N_THREADS].value.u64 = thread_manager->get_thread_count(); + stats[SINSP_STATS_V2_N_FDS].value.u64 = 0; + threadinfo_map_t* threadtable = thread_manager->get_threads(); + threadtable->loop([&] (sinsp_threadinfo& tinfo) { + sinsp_fdtable* fdtable = tinfo.get_fd_table(); + stats[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); + return true; + }); + stats[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_noncached_fd_lookups; + stats[SINSP_STATS_V2_CACHED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_cached_fd_lookups; + stats[SINSP_STATS_V2_FAILED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_failed_fd_lookups; + stats[SINSP_STATS_V2_ADDED_FDS].value.u64 = sinsp_stats_v2_counters.m_n_added_fds; + stats[SINSP_STATS_V2_REMOVED_FDS].value.u64 = sinsp_stats_v2_counters.m_n_removed_fds; + stats[SINSP_STATS_V2_STORED_EVTS].value.u64 = sinsp_stats_v2_counters.m_n_stored_evts; + stats[SINSP_STATS_V2_STORE_EVTS_DROPS].value.u64 = sinsp_stats_v2_counters.m_n_store_evts_drops; + stats[SINSP_STATS_V2_RETRIEVED_EVTS].value.u64 = sinsp_stats_v2_counters.m_n_retrieved_evts; + stats[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].value.u64 = sinsp_stats_v2_counters.m_n_retrieve_evts_drops; + stats[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_noncached_thread_lookups; + stats[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_cached_thread_lookups; + stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_failed_thread_lookups; + stats[SINSP_STATS_V2_ADDED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_added_threads; + stats[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_removed_threads; + stats[SINSP_STATS_V2_N_CONTAINERS].value.u32 = n_containers; + + *nstats = SINSP_MAX_STATS_V2; + } *rc = SCAP_SUCCESS; diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index 4da78e2c18..903a48e398 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -20,8 +20,27 @@ limitations under the License. #include #include #include "internal_metrics.h" +#include "threadinfo.h" -typedef enum sinsp_stats_v2 { +typedef struct sinsp_stats_v2 +{ + uint64_t m_n_noncached_fd_lookups; + uint64_t m_n_cached_fd_lookups; + uint64_t m_n_failed_fd_lookups; + uint64_t m_n_added_fds; + uint64_t m_n_removed_fds; + uint64_t m_n_stored_evts; + uint64_t m_n_store_evts_drops; + uint64_t m_n_retrieved_evts; + uint64_t m_n_retrieve_evts_drops; + uint64_t m_n_noncached_thread_lookups; + uint64_t m_n_cached_thread_lookups; + uint64_t m_n_failed_thread_lookups; + uint64_t m_n_added_threads; + uint64_t m_n_removed_threads; +}sinsp_stats_v2; + +typedef enum sinsp_stats_v2_resource_utilization { SINSP_RESOURCE_UTILIZATION_CPU_PERC = 0, ///< Current CPU usage, `ps` like, unit: percentage of one CPU. SINSP_RESOURCE_UTILIZATION_MEMORY_RSS, ///< Current RSS (Resident Set Size), unit: kb. SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ, ///< Current VSZ (Virtual Memory Size), unit: kb. @@ -30,9 +49,26 @@ typedef enum sinsp_stats_v2 { SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST, ///< Current total host CPU usage (all CPUs), unit: percentage. SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST, ///< Current total memory used out of available host memory, unit: kb. SINSP_RESOURCE_UTILIZATION_PROCS_HOST, ///< Number of processes currently running on CPUs on the host, unit: count. - SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST, ///< Number of allocated fds on the host, unit: count. + SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST, ///< Number of allocated fds on the host, unit: count. + SINSP_STATS_V2_N_THREADS, ///< Total number of threads currently stored in the sinsp state thread table, unit: count. + SINSP_STATS_V2_N_FDS, ///< Total number of fds currently stored across all threadtables associated with each active thread in the sinsp state thread table, unit: count. + SINSP_STATS_V2_NONCACHED_FD_LOOKUPS, ///< fdtable state related counters, unit: count. + SINSP_STATS_V2_CACHED_FD_LOOKUPS, ///< fdtable state related counters, unit: count. + SINSP_STATS_V2_FAILED_FD_LOOKUPS, ///< fdtable state related counters, unit: count. + SINSP_STATS_V2_ADDED_FDS, ///< fdtable state related counters, unit: count. + SINSP_STATS_V2_REMOVED_FDS, ///< fdtable state related counters, unit: count. + SINSP_STATS_V2_STORED_EVTS, ///< evt parsing related counters, unit: count. + SINSP_STATS_V2_STORE_EVTS_DROPS, ///< evt parsing related counters, unit: count. + SINSP_STATS_V2_RETRIEVED_EVTS, ///< evt parsing related counters, unit: count. + SINSP_STATS_V2_RETRIEVE_EVTS_DROPS, ///< evt parsing related counters, unit: count. + SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_CACHED_THREAD_LOOKUPS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_FAILED_THREAD_LOOKUPS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_ADDED_THREADS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_REMOVED_THREADS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_N_CONTAINERS, ///< Number of containers currently cached by sinsp_container_manager, unit: count. SINSP_MAX_STATS_V2 -}sinsp_stats_v2; +}sinsp_stats_v2_resource_utilization; namespace libsinsp { namespace stats { @@ -40,14 +76,17 @@ namespace stats { /*! \brief Retrieve current sinsp stats v2 including resource utilization metrics. \param agent_info Pointer to a \ref scap_agent_info containing relevant constants from the agent start up moment. - \param stats Pointer to a \ref scap_stats_v2 pre-allocated sinsp stats v2 buffer w/ scap_stats_v2 schema. + \param thread_manager Pointer to a \ref thread_manager to access threadtable properties. + \param sinsp_stats_v2_counters sinsp_stats_v2 struct containing counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities). + \param stats Pointer to a \ref scap_stats_v2 pre-allocated sinsp_stats_v2_buffer (aka scap_stats_v2 schema). + \param n_containers Number of containers currently cached by sinsp_container_manager. \param nstats Pointer reflecting number of statistics in returned buffer \param rc Pointer to return code \note Intended to be called once every x hours. \return Pointer to a \ref scap_stats_v2 buffer filled with the current sinsp stats v2 including resource utilization metrics. */ - const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, scap_stats_v2* stats, uint32_t* nstats, int32_t* rc); + const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 sinsp_stats_v2_counters, scap_stats_v2* stats, uint32_t n_containers, uint32_t* nstats, int32_t* rc); } } diff --git a/userspace/libsinsp/test/filterchecks/proc.cpp b/userspace/libsinsp/test/filterchecks/proc.cpp index d9439518ea..92560351c2 100644 --- a/userspace/libsinsp/test/filterchecks/proc.cpp +++ b/userspace/libsinsp/test/filterchecks/proc.cpp @@ -18,7 +18,7 @@ limitations under the License. #include -TEST_F(sinsp_with_test_input, PROC_FILTER_nthreds) +TEST_F(sinsp_with_test_input, PROC_FILTER_nthreads) { DEFAULT_TREE diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp index fb62b0f15e..635d3536d5 100644 --- a/userspace/libsinsp/test/sinsp_stats.ut.cpp +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -17,33 +17,46 @@ limitations under the License. */ #include +#include "sinsp.h" #include "utils.h" #include "sinsp_with_test_input.h" #include "test_utils.h" #include "scap_stats_v2.h" - +#include TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) { #ifdef __linux__ open_inspector(SCAP_MODE_LIVE); + // Adopted from test: TEST_F(sinsp_with_test_input, PROC_FILTER_nthreads) + DEFAULT_TREE + /* we call a random event to obtain an event associated with this thread info */ + auto evt = generate_random_event(p2_t1_tid); + ASSERT_EQ(get_field_as_string(evt, "proc.nthreads"), "3"); + const scap_agent_info* agent_info = m_inspector.get_agent_info(); - uint32_t nstats = 0; int32_t rc; const scap_stats_v2* sinsp_stats_v2_snapshot; auto buffer = m_inspector.get_sinsp_stats_v2_buffer(); + sinsp_stats_v2 sinsp_stats_v2_counters = m_inspector.get_sinsp_stats_v2(); + sinsp_thread_manager* thread_manager = m_inspector.m_thread_manager; + uint32_t n_containers = m_inspector.m_container_manager.get_containers()->size(); uint32_t flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); /* Extra call */ - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); - + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); ASSERT_EQ(nstats, SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1); ASSERT_EQ(rc, SCAP_SUCCESS); + flags = (PPM_SCAP_STATS_RESOURCE_UTILIZATION | PPM_SCAP_STATS_STATE_COUNTERS); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); + ASSERT_EQ(nstats, SINSP_MAX_STATS_V2); + ASSERT_EQ(rc, SCAP_SUCCESS); + /* These names should always be available */ - std::unordered_set minimal_stats_names = {"cpu_usage_perc", "memory_rss", "memory_vsz", "memory_pss", "container_memory_used", "cpu_usage_perc_total_host", "procs_running_host", "memory_used_host", "open_fds_host"}; + std::unordered_set minimal_stats_names = {"cpu_usage_perc", "memory_rss", "open_fds_host", "n_threads", "n_fds", "n_added_fds", "n_added_threads", "n_removed_threads", "n_containers"}; uint32_t i = 0; for(const auto& stat_name : minimal_stats_names) @@ -63,7 +76,6 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) } /* Assert values are greater than 0 */ - ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d, 0); ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32, 0); ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32, 0); ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].value.d, 0); @@ -71,10 +83,14 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].value.u32, 0); ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].value.u64, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_STATS_V2_N_THREADS].value.u64, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_STATS_V2_N_FDS].value.u64, 0); + ASSERT_GT(sinsp_stats_v2_snapshot[SINSP_STATS_V2_ADDED_THREADS].value.u64, 0); + /* Empty call */ nstats = 0; flags = 0; - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, buffer, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); ASSERT_EQ(nstats, 0); ASSERT_EQ(rc, SCAP_SUCCESS); diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index 74965ba6e6..33f2162d54 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -1517,6 +1517,10 @@ std::unique_ptr sinsp_thread_manager::new_threadinfo() const */ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_scap_proctable) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_added_threads++; + } #ifdef GATHER_INTERNAL_STATS m_added_threads->increment(); #endif @@ -1701,6 +1705,10 @@ void sinsp_thread_manager::remove_thread(int64_t tid) /* This should never happen but just to be sure. */ if(thread_to_remove == nullptr) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; + } #ifdef GATHER_INTERNAL_STATS m_failed_lookups->increment(); #endif @@ -1822,6 +1830,10 @@ void sinsp_thread_manager::remove_thread(int64_t tid) * the cache just to be sure. */ m_last_tid = -1; + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_removed_threads++; + } #ifdef GATHER_INTERNAL_STATS m_removed_threads->increment(); #endif @@ -2179,6 +2191,10 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look thr = m_last_tinfo.lock(); if (thr) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_cached_thread_lookups++; + } #ifdef GATHER_INTERNAL_STATS m_cached_lookups->increment(); #endif @@ -2196,6 +2212,10 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look if(thr) { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_noncached_thread_lookups++; + } #ifdef GATHER_INTERNAL_STATS m_non_cached_lookups->increment(); #endif @@ -2210,6 +2230,10 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look } else { + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; + } #ifdef GATHER_INTERNAL_STATS m_failed_lookups->increment(); #endif diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 3f1152f18e..7fa20479d0 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -531,7 +531,6 @@ class SINSP_PUBLIC sinsp_threadinfo: public libsinsp::state::table_entry } }; -protected: /* Note that `fd_table` should be shared with the main thread only if `PPM_CL_CLONE_FILES` * is specified. Today we always specify `PPM_CL_CLONE_FILES` for all threads. */ From 0d6d5cf08389eed11aa37fa8f1b5fdcc0447d25c Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Fri, 20 Oct 2023 21:58:50 +0000 Subject: [PATCH 04/10] feat(libsinsp): remove legacy libsinsp stats GATHER_INTERNAL_STATS Signed-off-by: Melissa Kilby --- userspace/libsinsp/CMakeLists.txt | 1 - userspace/libsinsp/fdinfo.cpp | 19 --- userspace/libsinsp/internal_metrics.cpp | 45 ------- userspace/libsinsp/internal_metrics.h | 151 ------------------------ userspace/libsinsp/parsers.cpp | 27 ----- userspace/libsinsp/settings.h | 5 - userspace/libsinsp/sinsp.cpp | 54 --------- userspace/libsinsp/sinsp.h | 11 -- userspace/libsinsp/sinsp_int.h | 1 - userspace/libsinsp/stats.cpp | 57 --------- userspace/libsinsp/stats.h | 41 ------- userspace/libsinsp/threadinfo.cpp | 45 ------- userspace/libsinsp/threadinfo.h | 9 -- 13 files changed, 466 deletions(-) delete mode 100644 userspace/libsinsp/internal_metrics.cpp delete mode 100644 userspace/libsinsp/internal_metrics.h diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt index 24ecedaa68..91faf419d6 100644 --- a/userspace/libsinsp/CMakeLists.txt +++ b/userspace/libsinsp/CMakeLists.txt @@ -112,7 +112,6 @@ set(SINSP_SOURCES ifinfo.cpp memmem.cpp tracers.cpp - internal_metrics.cpp logger.cpp parsers.cpp ../plugin/plugin_loader.c diff --git a/userspace/libsinsp/fdinfo.cpp b/userspace/libsinsp/fdinfo.cpp index c693bf33cc..911f039dea 100644 --- a/userspace/libsinsp/fdinfo.cpp +++ b/userspace/libsinsp/fdinfo.cpp @@ -358,9 +358,6 @@ sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) { m_inspector->m_sinsp_stats_v2.m_n_cached_fd_lookups++; } - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_cached_fd_lookups++; - #endif return m_last_accessed_fdinfo; } @@ -375,9 +372,6 @@ sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) { m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; } - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_failed_fd_lookups++; - #endif return NULL; } else @@ -387,9 +381,6 @@ sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; } - #ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_noncached_fd_lookups++; - #endif m_last_accessed_fd = fd; m_last_accessed_fdinfo = &(fdit->second); lookup_device(&(fdit->second), fd); @@ -422,9 +413,6 @@ sinsp_fdinfo_t* sinsp_fdtable::add(int64_t fd, sinsp_fdinfo_t* fdinfo) m_inspector->m_sinsp_stats_v2.m_n_added_fds++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_added_fds++; -#endif std::pair::iterator, bool> insert_res = m_table.emplace(fd, *fdinfo); return &(insert_res.first->second); } @@ -496,9 +484,6 @@ void sinsp_fdtable::erase(int64_t fd) { m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_failed_fd_lookups++; -#endif } else { @@ -508,10 +493,6 @@ void sinsp_fdtable::erase(int64_t fd) m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; m_inspector->m_sinsp_stats_v2.m_n_removed_fds++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_noncached_fd_lookups++; - m_inspector->m_stats->m_n_removed_fds++; -#endif } } diff --git a/userspace/libsinsp/internal_metrics.cpp b/userspace/libsinsp/internal_metrics.cpp deleted file mode 100644 index 7ddea2282d..0000000000 --- a/userspace/libsinsp/internal_metrics.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -/* -Copyright (C) 2023 The Falco Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ - -#include "sinsp.h" -#include "sinsp_int.h" - -#ifdef GATHER_INTERNAL_STATS -namespace internal_metrics -{ - -counter::~counter() -{ -} - -counter::counter() -{ - m_value = 0; -} - -void registry::clear_all_metrics() -{ - for(metric_map_iterator_t it = get_metrics().begin(); it != get_metrics().end(); it++) - { - it->second->clear(); - } - -} - -} -#endif diff --git a/userspace/libsinsp/internal_metrics.h b/userspace/libsinsp/internal_metrics.h deleted file mode 100644 index bac9580449..0000000000 --- a/userspace/libsinsp/internal_metrics.h +++ /dev/null @@ -1,151 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 -/* -Copyright (C) 2023 The Falco Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -*/ - -#pragma once -#ifdef GATHER_INTERNAL_STATS -#include -#include -#include - -#define INTERNAL_COUNTER(X) internal_metrics::counter *X - -namespace internal_metrics { - -class metric; -class registry; -class counter; - -class SINSP_PUBLIC metric_name -{ -public: - metric_name(std::string name, std::string description) - { - m_name = name; - m_description = description; - } - - bool operator<(const metric_name& other) const - { - return m_name> metric_map_t; - typedef metric_map_t::iterator metric_map_iterator_t; - - counter& register_counter(const metric_name& name) - { - std::shared_ptr p; - p = std::make_shared(); - m_metrics[name] = p; - return *p.get(); - } - - metric_map_t& get_metrics() - { - return m_metrics; - } - - void clear_all_metrics(); - -private: - //template T& create_metric(const metric_name& name, Args... args) - //{ - // if (m_metrics.find(name) == std::end(m_metrics)) - // { - // m_metrics[name] = std::make_shared(args...); - // } - // return dynamic_cast(*m_metrics[name]); - //} - - metric_map_t m_metrics; - -}; - - -class SINSP_PUBLIC counter : public metric -{ -public: - ~counter(); - counter(); - - void increment() - { - m_value++; - } - - void decrement() - { - m_value--; - } - - void clear() - { - m_value = 0; - } - - const uint64_t get_value() - { - return m_value; - } - - void process(processor& metric_processor) - { - metric_processor.process(*this); - } - -private: - - uint64_t m_value; -}; - -} -#else -#define INTERNAL_COUNTER(X) -#endif // GATHER_INTERNAL_STATS diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index 4183983934..f28914c990 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -738,9 +738,6 @@ bool sinsp_parser::reset(sinsp_evt *evt) { m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups--; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_thread_manager->m_failed_lookups->decrement(); -#endif } return false; } @@ -890,9 +887,6 @@ void sinsp_parser::store_event(sinsp_evt *evt) { m_inspector->m_sinsp_stats_v2.m_n_store_evts_drops++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_store_drops++; -#endif return; } @@ -929,9 +923,6 @@ void sinsp_parser::store_event(sinsp_evt *evt) { m_inspector->m_sinsp_stats_v2.m_n_stored_evts++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_stored_evts++; -#endif } bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_evt) @@ -957,9 +948,6 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev { m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_retrieve_drops++; -#endif return false; } @@ -983,9 +971,6 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev { m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_retrieved_evts++; -#endif return true; } @@ -1001,9 +986,6 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev { m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_retrieve_drops++; -#endif return false; } if (m_inspector != nullptr) @@ -1011,9 +993,6 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_retrieved_evts++; -#endif return true; } @@ -3810,18 +3789,12 @@ void sinsp_parser::parse_close_exit(sinsp_evt *evt) { m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_failed_fd_lookups--; -#endif if(evt->m_tinfo && evt->m_tinfo->is_lastevent_data_valid()) { if (m_inspector != nullptr) { m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; } -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_failed_fd_lookups--; -#endif } } } diff --git a/userspace/libsinsp/settings.h b/userspace/libsinsp/settings.h index f3f3f57d67..cee277128c 100644 --- a/userspace/libsinsp/settings.h +++ b/userspace/libsinsp/settings.h @@ -29,11 +29,6 @@ limitations under the License. // #define ASSERT_TO_LOG -// -// Controls if the library collects internal performance stats. -// -#undef GATHER_INTERNAL_STATS - // // Max size that the FD table of a process can reach // diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index d663fc686c..d351a44efa 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -49,12 +49,6 @@ limitations under the License. #include #endif -#ifdef GATHER_INTERNAL_STATS -#include "stats.h" -#else // GATHER_INTERNAL_STATS -struct sinsp_stats{}; // note: makes the unique_ptr static asserts happy -#endif // GATHER_INTERNAL_STATS - #ifdef HAS_ANALYZER #include "analyzer_int.h" #include "analyzer.h" @@ -86,7 +80,6 @@ sinsp::sinsp(bool static_container, const std::string &static_id, const std::str m_host_root(scap_get_host_root()), m_container_manager(this, static_container, static_id, static_name, static_image), m_usergroup_manager(this), - m_stats(nullptr), m_async_events_queue(DEFAULT_ASYNC_EVENT_QUEUE_SIZE), m_suppressed_comms(), m_inited(false) @@ -143,10 +136,6 @@ sinsp::sinsp(bool static_container, const std::string &static_id, const std::str m_self_pid = getpid(); #endif -#ifdef GATHER_INTERNAL_STATS - m_stats = std::make_unique; -#endif - m_proc_scan_timeout_ms = SCAP_PROC_SCAN_TIMEOUT_NONE; m_proc_scan_log_interval_ms = SCAP_PROC_SCAN_LOG_NONE; @@ -350,9 +339,6 @@ void sinsp::init() // // Basic inits // -#ifdef GATHER_INTERNAL_STATS - m_stats->clear(); -#endif m_sinsp_stats_v2.m_n_added_fds = 0; m_sinsp_stats_v2.m_n_cached_fd_lookups = 0; @@ -2138,46 +2124,6 @@ const struct scap_stats_v2* sinsp::get_capture_stats_v2(uint32_t flags, uint32_t return stats_v2; } -#ifdef GATHER_INTERNAL_STATS -sinsp_stats sinsp::get_stats() -{ - scap_stats stats; - - // - // Get capture stats from scap - // - if(m_h) - { - scap_get_stats(m_h, &stats); - - m_stats->m_n_seen_evts = stats.n_evts; - m_stats->m_n_drops = stats.n_drops; - m_stats->m_n_preemptions = stats.n_preemptions; - } - else - { - m_stats->m_n_seen_evts = 0; - m_stats->m_n_drops = 0; - m_stats->m_n_preemptions = 0; - } - - // - // Count the number of threads and fds by scanning the tables, - // and update the thread-related stats. - // - if(m_thread_manager) - { - m_thread_manager->update_statistics(); - } - - // - // Return the result - // - - return *m_stats.get(); -} -#endif // GATHER_INTERNAL_STATS - void sinsp::set_log_callback(sinsp_logger_callback cb) { if(cb) diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index 1a7e739e6f..7069d654ab 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -108,7 +108,6 @@ class sinsp_partial_tracer; class sinsp_plugin; class sinsp_plugin_manager; class sinsp_observer; -class sinsp_stats; std::vector sinsp_split(const std::string &s, char delim); @@ -554,11 +553,6 @@ class SINSP_PUBLIC sinsp : public capture_stats_source */ const struct scap_stats_v2* get_capture_stats_v2(uint32_t flags, uint32_t* nstats, int32_t* rc) const override; - -#ifdef GATHER_INTERNAL_STATS - sinsp_stats get_stats(); -#endif - libsinsp::event_processor* m_external_event_processor; sinsp_threadinfo* build_threadinfo() @@ -1139,11 +1133,6 @@ VISIBILITY_PRIVATE std::string m_filterstring; std::shared_ptr m_internal_flt_ast; - // - // Internal stats - // - std::unique_ptr m_stats; - std::vector m_tid_collisions; // diff --git a/userspace/libsinsp/sinsp_int.h b/userspace/libsinsp/sinsp_int.h index 50df22d665..a98b7dbe6a 100644 --- a/userspace/libsinsp/sinsp_int.h +++ b/userspace/libsinsp/sinsp_int.h @@ -48,7 +48,6 @@ limitations under the License. #include "scap.h" #include "parsers.h" #include "ifinfo.h" -#include "internal_metrics.h" #include "sinsp_public.h" #ifndef MIN diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index 4b24c35087..9eb04df509 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -419,60 +419,3 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s return stats; } - - -#ifdef GATHER_INTERNAL_STATS - -void sinsp_stats::clear() -{ - m_n_seen_evts = 0; - m_n_drops = 0; - m_n_preemptions = 0; - m_n_noncached_fd_lookups = 0; - m_n_cached_fd_lookups = 0; - m_n_failed_fd_lookups = 0; - m_n_threads = 0; - m_n_fds = 0; - m_n_added_fds = 0; - m_n_removed_fds = 0; - m_n_stored_evts = 0; - m_n_store_drops = 0; - m_n_retrieved_evts = 0; - m_n_retrieve_drops = 0; - m_metrics_registry.clear_all_metrics(); -} - -void sinsp_stats::emit(FILE* f) -{ - m_output_target = f; - - fprintf(f, "evts seen by driver: %" PRIu64 "\n", m_n_seen_evts); - fprintf(f, "drops: %" PRIu64 "\n", m_n_drops); - fprintf(f, "preemptions: %" PRIu64 "\n", m_n_preemptions); - fprintf(f, "fd lookups: %" PRIu64 "(%" PRIu64 " cached %" PRIu64 " noncached)\n", - m_n_noncached_fd_lookups + m_n_cached_fd_lookups, - m_n_cached_fd_lookups, - m_n_noncached_fd_lookups); - fprintf(f, "failed fd lookups: %" PRIu64 "\n", m_n_failed_fd_lookups); - fprintf(f, "n. threads: %" PRIu64 "\n", m_n_threads); - fprintf(f, "n. fds: %" PRIu64 "\n", m_n_fds); - fprintf(f, "added fds: %" PRIu64 "\n", m_n_added_fds); - fprintf(f, "removed fds: %" PRIu64 "\n", m_n_removed_fds); - fprintf(f, "stored evts: %" PRIu64 "\n", m_n_stored_evts); - fprintf(f, "store drops: %" PRIu64 "\n", m_n_store_drops); - fprintf(f, "retrieved evts: %" PRIu64 "\n", m_n_retrieved_evts); - fprintf(f, "retrieve drops: %" PRIu64 "\n", m_n_retrieve_drops); - - for(internal_metrics::registry::metric_map_iterator_t it = m_metrics_registry.get_metrics().begin(); it != m_metrics_registry.get_metrics().end(); it++) - { - fprintf(f, "%s: ", it->first.get_description().c_str()); - it->second->process(*this); - } -} - -void sinsp_stats::process(internal_metrics::counter& metric) -{ - fprintf(m_output_target, "%" PRIu64 "\n", metric.get_value()); -} - -#endif // GATHER_INTERNAL_STATS diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index 903a48e398..d292588d8f 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -19,7 +19,6 @@ limitations under the License. #pragma once #include #include -#include "internal_metrics.h" #include "threadinfo.h" typedef struct sinsp_stats_v2 @@ -90,43 +89,3 @@ namespace stats { } } - -#ifdef GATHER_INTERNAL_STATS - -// -// Processing stats class. -// Keeps a bunch of counters with key library performance metrics. -// -class SINSP_PUBLIC sinsp_stats : public internal_metrics::processor -{ -public: - void clear(); - void emit(FILE* f); - internal_metrics::registry& get_metrics_registry() - { - return m_metrics_registry; - } - - void process(internal_metrics::counter& metric); - - uint64_t m_n_seen_evts; - uint64_t m_n_drops; - uint64_t m_n_preemptions; - uint64_t m_n_noncached_fd_lookups; - uint64_t m_n_cached_fd_lookups; - uint64_t m_n_failed_fd_lookups; - uint64_t m_n_threads; - uint64_t m_n_fds; - uint64_t m_n_added_fds; - uint64_t m_n_removed_fds; - uint64_t m_n_stored_evts; - uint64_t m_n_store_drops; - uint64_t m_n_retrieved_evts; - uint64_t m_n_retrieve_drops; - -private: - internal_metrics::registry m_metrics_registry; - FILE* m_output_target; -}; - -#endif // GATHER_INTERNAL_STATS diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index 33f2162d54..d3ab783e0b 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -1424,14 +1424,6 @@ void sinsp_thread_manager::clear() m_last_tid = 0; m_last_flush_time_ns = 0; m_n_drops = 0; - -#ifdef GATHER_INTERNAL_STATS - m_failed_lookups = &m_inspector->m_stats->get_metrics_registry().register_counter(internal_metrics::metric_name("thread_failed_lookups","Failed thread lookups")); - m_cached_lookups = &m_inspector->m_stats->get_metrics_registry().register_counter(internal_metrics::metric_name("thread_cached_lookups","Cached thread lookups")); - m_non_cached_lookups = &m_inspector->m_stats->get_metrics_registry().register_counter(internal_metrics::metric_name("thread_non_cached_lookups","Non cached thread lookups")); - m_added_threads = &m_inspector->m_stats->get_metrics_registry().register_counter(internal_metrics::metric_name("thread_added","Number of added threads")); - m_removed_threads = &m_inspector->m_stats->get_metrics_registry().register_counter(internal_metrics::metric_name("thread_removed","Removed threads")); -#endif } /* This is called on the table after the `/proc` scan */ @@ -1521,9 +1513,6 @@ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_sc { m_inspector->m_sinsp_stats_v2.m_n_added_threads++; } -#ifdef GATHER_INTERNAL_STATS - m_added_threads->increment(); -#endif /* We have no more space */ if(m_threadtable.size() >= m_max_thread_table_size @@ -1709,9 +1698,6 @@ void sinsp_thread_manager::remove_thread(int64_t tid) { m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; } -#ifdef GATHER_INTERNAL_STATS - m_failed_lookups->increment(); -#endif return; } @@ -1834,9 +1820,6 @@ void sinsp_thread_manager::remove_thread(int64_t tid) { m_inspector->m_sinsp_stats_v2.m_n_removed_threads++; } -#ifdef GATHER_INTERNAL_STATS - m_removed_threads->increment(); -#endif } void sinsp_thread_manager::fix_sockets_coming_from_proc() @@ -1881,25 +1864,6 @@ void sinsp_thread_manager::create_thread_dependencies_after_proc_scan() }); } -void sinsp_thread_manager::update_statistics() -{ -#ifdef GATHER_INTERNAL_STATS - m_inspector->m_stats->m_n_threads = get_thread_count(); - - m_inspector->m_stats->m_n_fds = 0; - for(threadinfo_map_iterator_t it = m_threadtable.begin(); it != m_threadtable.end(); it++) - { - sinsp_fdtable* fd_table_ptr = it->second.get_fd_table(); - if(fd_table_ptr == NULL) - { - ASSERT(false); - return; - } - m_inspector->m_stats->m_n_fds += fd_table_ptr->size(); - } -#endif -} - void sinsp_thread_manager::free_dump_fdinfos(std::vector* fdinfos_to_free) { for(uint32_t j = 0; j < fdinfos_to_free->size(); j++) @@ -2195,9 +2159,6 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look { m_inspector->m_sinsp_stats_v2.m_n_cached_thread_lookups++; } -#ifdef GATHER_INTERNAL_STATS - m_cached_lookups->increment(); -#endif // This allows us to avoid performing an actual timestamp lookup // for something that may not need to be precise thr->m_lastaccess_ts = m_inspector->get_lastevent_ts(); @@ -2216,9 +2177,6 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look { m_inspector->m_sinsp_stats_v2.m_n_noncached_thread_lookups++; } -#ifdef GATHER_INTERNAL_STATS - m_non_cached_lookups->increment(); -#endif if(!lookup_only) { m_last_tinfo.reset(); @@ -2234,9 +2192,6 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look { m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; } -#ifdef GATHER_INTERNAL_STATS - m_failed_lookups->increment(); -#endif return NULL; } } diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 7fa20479d0..d457d82fef 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -37,7 +37,6 @@ struct iovec { #include #include #include "fdinfo.h" -#include "internal_metrics.h" #include "state/table.h" #include "thread_group_info.h" @@ -783,8 +782,6 @@ class SINSP_PUBLIC sinsp_thread_manager: public libsinsp::state::table return (uint32_t)m_threadtable.size(); } - void update_statistics(); - threadinfo_map_t* get_threads() { return &m_threadtable; @@ -903,12 +900,6 @@ VISIBILITY_PRIVATE int32_t m_max_n_proc_lookups = -1; int32_t m_max_n_proc_socket_lookups = -1; - INTERNAL_COUNTER(m_failed_lookups); - INTERNAL_COUNTER(m_cached_lookups); - INTERNAL_COUNTER(m_non_cached_lookups); - INTERNAL_COUNTER(m_added_threads); - INTERNAL_COUNTER(m_removed_threads); - friend class sinsp_parser; friend class sinsp_analyzer; friend class sinsp; From de9863aabf800fe076e64f31c08de561b57ba4e7 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Fri, 20 Oct 2023 22:19:05 +0000 Subject: [PATCH 05/10] new(libsinsp): expose full threadtable drops to libsinsp stats Signed-off-by: Melissa Kilby --- userspace/libsinsp/stats.cpp | 3 +++ userspace/libsinsp/stats.h | 1 + userspace/libsinsp/threadinfo.h | 1 + 3 files changed, 5 insertions(+) diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index 9eb04df509..b8b5a40418 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -56,6 +56,7 @@ static const char *const sinsp_stats_v2_resource_utilization_names[] = { [SINSP_STATS_V2_FAILED_THREAD_LOOKUPS] = "n_failed_thread_lookups", [SINSP_STATS_V2_ADDED_THREADS] = "n_added_threads", [SINSP_STATS_V2_REMOVED_THREADS] = "n_removed_threads", + [SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE] = "n_drops_full_threadtable", [SINSP_STATS_V2_N_CONTAINERS] = "n_containers", }; @@ -383,6 +384,7 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; stats[SINSP_STATS_V2_ADDED_THREADS].type = STATS_VALUE_TYPE_U64; stats[SINSP_STATS_V2_REMOVED_THREADS].type = STATS_VALUE_TYPE_U64; + stats[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].type = STATS_VALUE_TYPE_U32; stats[SINSP_STATS_V2_N_CONTAINERS].type = STATS_VALUE_TYPE_U32; } @@ -410,6 +412,7 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_failed_thread_lookups; stats[SINSP_STATS_V2_ADDED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_added_threads; stats[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_removed_threads; + stats[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = thread_manager->get_m_n_drops(); stats[SINSP_STATS_V2_N_CONTAINERS].value.u32 = n_containers; *nstats = SINSP_MAX_STATS_V2; diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index d292588d8f..2dd95d9d66 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -65,6 +65,7 @@ typedef enum sinsp_stats_v2_resource_utilization { SINSP_STATS_V2_FAILED_THREAD_LOOKUPS, ///< threadtable state related counters, unit: count. SINSP_STATS_V2_ADDED_THREADS, ///< threadtable state related counters, unit: count. SINSP_STATS_V2_REMOVED_THREADS, ///< threadtable state related counters, unit: count. + SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE, ///< Number of drops due to full threadtable, unit: count. SINSP_STATS_V2_N_CONTAINERS, ///< Number of containers currently cached by sinsp_container_manager, unit: count. SINSP_MAX_STATS_V2 }sinsp_stats_v2_resource_utilization; diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index d457d82fef..14a738b5c8 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -794,6 +794,7 @@ class SINSP_PUBLIC sinsp_thread_manager: public libsinsp::state::table int32_t get_m_n_proc_lookups() const { return m_n_proc_lookups; } int32_t get_m_n_main_thread_lookups() const { return m_n_main_thread_lookups; } uint64_t get_m_n_proc_lookups_duration_ns() const { return m_n_proc_lookups_duration_ns; } + uint32_t get_m_n_drops() const { return m_n_drops; } void reset_thread_counters() { m_n_proc_lookups = 0; m_n_main_thread_lookups = 0; m_n_proc_lookups_duration_ns = 0; } void set_m_max_n_proc_lookups(int32_t val) { m_max_n_proc_lookups = val; } From 68da9ff5252a343ff117aa35199a746c24040da2 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Wed, 25 Oct 2023 19:05:12 +0000 Subject: [PATCH 06/10] new(libsinsp): expose crucial container cache stats - Get container cache stats via hijacking the periodic container cache flush functionality - Includes some general cleanup Signed-off-by: Melissa Kilby --- userspace/libsinsp/container.cpp | 15 ++- userspace/libsinsp/sinsp.cpp | 2 + userspace/libsinsp/stats.cpp | 145 ++++++++++----------- userspace/libsinsp/stats.h | 14 +- userspace/libsinsp/test/sinsp_stats.ut.cpp | 9 +- 5 files changed, 100 insertions(+), 85 deletions(-) diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp index 09a667f9d7..90a742a089 100644 --- a/userspace/libsinsp/container.cpp +++ b/userspace/libsinsp/container.cpp @@ -88,11 +88,24 @@ bool sinsp_container_manager::remove_inactive_containers() }); auto containers = m_containers.lock(); + if (m_inspector != nullptr) + { + m_inspector->m_sinsp_stats_v2.m_n_missing_container_images = 0; + m_inspector->m_sinsp_stats_v2.m_n_containers = containers->size(); + } for(auto it = containers->begin(); it != containers->end();) { + sinsp_container_info::ptr_t container = it->second; + if (m_inspector != nullptr) + { + auto container_info = container.get(); + if (!container_info || (container_info && container_info->m_image.empty())) + { + m_inspector->m_sinsp_stats_v2.m_n_missing_container_images++; + } + } if(containers_in_use.find(it->first) == containers_in_use.end()) { - sinsp_container_info::ptr_t container = it->second; for(const auto &remove_cb : m_remove_callbacks) { remove_cb(*container); diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index d351a44efa..ff1f48c0d7 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -354,6 +354,8 @@ void sinsp::init() m_sinsp_stats_v2.m_n_failed_thread_lookups = 0; m_sinsp_stats_v2.m_n_added_threads = 0; m_sinsp_stats_v2.m_n_removed_threads = 0; + m_sinsp_stats_v2.m_n_missing_container_images = 0; + m_sinsp_stats_v2.m_n_containers= 0; m_nevts = 0; m_tid_to_remove = -1; diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index b8b5a40418..7232598b1b 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -16,15 +16,10 @@ limitations under the License. */ -//////////////////////////////////////////////////////////////////////////// -// Public definitions for the scap library -//////////////////////////////////////////////////////////////////////////// -#include "sinsp.h" +#ifdef __linux__ + #include "sinsp_int.h" -#include "sinsp_public.h" #include -#include -#include #include #include #include "stats.h" @@ -57,6 +52,7 @@ static const char *const sinsp_stats_v2_resource_utilization_names[] = { [SINSP_STATS_V2_ADDED_THREADS] = "n_added_threads", [SINSP_STATS_V2_REMOVED_THREADS] = "n_removed_threads", [SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE] = "n_drops_full_threadtable", + [SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES] = "n_missing_container_images", [SINSP_STATS_V2_N_CONTAINERS] = "n_containers", }; @@ -162,7 +158,6 @@ void get_rss_vsz_pss_total_memory_and_open_fds(uint32_t &rss, uint32_t &vsz, uin ASSERT(false); return; } - } void get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, double &cpu_usage_perc_total_host, uint32_t &procs_running_host) @@ -261,7 +256,6 @@ void get_cpu_usage_and_total_procs(double start_time, double &cpu_usage_perc, do cpu_usage_perc_total_host = 100.0 - ((idle * 100.0) / sum); cpu_usage_perc_total_host = std::round(cpu_usage_perc_total_host * 10.0) / 10.0; // round to 1 decimal } - } uint64_t get_container_memory_usage() @@ -274,7 +268,7 @@ uint64_t get_container_memory_usage() * typically libs clients (e.g. Falco) pods contain sidekick containers that use memory as well. * This metric accounts only for the container with the security monitoring agent running. */ - + unsigned long long memory_used = 0; const char* filepath = getenv(SINSP_AGENT_CGROUP_MEM_PATH_ENV_VAR); if (filepath == nullptr) { @@ -287,7 +281,6 @@ uint64_t get_container_memory_usage() ASSERT(false); return 0; } - unsigned long long memory_used = 0; /* memory size returned in bytes */ int fscanf_matched = fscanf(f, "%llu", &memory_used); @@ -296,13 +289,12 @@ uint64_t get_container_memory_usage() if (fscanf_matched != 1) { return 0; } - return memory_used; } -const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 sinsp_stats_v2_counters, scap_stats_v2* stats, uint32_t n_containers, uint32_t* nstats, int32_t* rc) +const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc) { - if (!stats) + if (!buffer) { *nstats = 0; *rc = SCAP_FAILURE; @@ -320,24 +312,24 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s double cpu_usage_perc_total_host = 0.0; uint32_t procs_running_host = 0; - if(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name != nullptr && strncmp(stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CPU_PERC], 15) != 0) + if(buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name != nullptr && strncmp(buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC].name, sinsp_stats_v2_resource_utilization_names[SINSP_RESOURCE_UTILIZATION_CPU_PERC], 15) != 0) { // Init for(uint32_t i = SINSP_RESOURCE_UTILIZATION_CPU_PERC; i < SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1; i++) { - stats[i].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - strlcpy(stats[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); + buffer[i].flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + strlcpy(buffer[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); } - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].type = STATS_VALUE_TYPE_D; - stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].type = STATS_VALUE_TYPE_U32; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; - stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC].type = STATS_VALUE_TYPE_D; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].type = STATS_VALUE_TYPE_D; + buffer[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].type = STATS_VALUE_TYPE_U64; } // Get stats / metrics snapshot @@ -345,15 +337,15 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s get_cpu_usage_and_total_procs(agent_info->start_time, cpu_usage_perc, cpu_usage_perc_total_host, procs_running_host); get_rss_vsz_pss_total_memory_and_open_fds(rss, vsz, pss, memory_used_host, open_fds_host); - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d = cpu_usage_perc; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32 = rss; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32 = vsz; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].value.u32 = pss; - stats[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].value.u64 = get_container_memory_usage(); - stats[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].value.d = cpu_usage_perc_total_host; - stats[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].value.u32 = procs_running_host; - stats[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].value.u64 = memory_used_host; - stats[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].value.u64 = open_fds_host; + buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC].value.d = cpu_usage_perc; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_RSS].value.u32 = rss; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_VSZ].value.u32 = vsz; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_PSS].value.u32 = pss; + buffer[SINSP_RESOURCE_UTILIZATION_CONTAINER_MEMORY].value.u64 = get_container_memory_usage(); + buffer[SINSP_RESOURCE_UTILIZATION_CPU_PERC_TOTAL_HOST].value.d = cpu_usage_perc_total_host; + buffer[SINSP_RESOURCE_UTILIZATION_PROCS_HOST].value.u32 = procs_running_host; + buffer[SINSP_RESOURCE_UTILIZATION_MEMORY_TOTAL_HOST].value.u64 = memory_used_host; + buffer[SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST].value.u64 = open_fds_host; *nstats = SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1; @@ -361,64 +353,69 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s if((flags & PPM_SCAP_STATS_STATE_COUNTERS)) { - if(stats[SINSP_STATS_V2_N_THREADS].name != nullptr && strncmp(stats[SINSP_STATS_V2_N_THREADS].name, sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], 10) != 0) + if(buffer[SINSP_STATS_V2_N_THREADS].name != nullptr && strncmp(buffer[SINSP_STATS_V2_N_THREADS].name, sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], 10) != 0) { // Init for(uint32_t i = SINSP_STATS_V2_N_THREADS; i < SINSP_MAX_STATS_V2; i++) { - stats[i].flags = PPM_SCAP_STATS_STATE_COUNTERS; - strlcpy(stats[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); + buffer[i].flags = PPM_SCAP_STATS_STATE_COUNTERS; + strlcpy(buffer[i].name, sinsp_stats_v2_resource_utilization_names[i], STATS_NAME_MAX); } - stats[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_CACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_FAILED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_ADDED_FDS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_REMOVED_FDS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_STORED_EVTS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_STORE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_RETRIEVED_EVTS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_ADDED_THREADS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_REMOVED_THREADS].type = STATS_VALUE_TYPE_U64; - stats[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].type = STATS_VALUE_TYPE_U32; - stats[SINSP_STATS_V2_N_CONTAINERS].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_CACHED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_FAILED_FD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_ADDED_FDS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_REMOVED_FDS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_STORED_EVTS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_STORE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_RETRIEVED_EVTS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_ADDED_THREADS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_REMOVED_THREADS].type = STATS_VALUE_TYPE_U64; + buffer[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES].type = STATS_VALUE_TYPE_U32; + buffer[SINSP_STATS_V2_N_CONTAINERS].type = STATS_VALUE_TYPE_U32; } // Get stats / metrics snapshot - stats[SINSP_STATS_V2_N_THREADS].value.u64 = thread_manager->get_thread_count(); - stats[SINSP_STATS_V2_N_FDS].value.u64 = 0; + + buffer[SINSP_STATS_V2_N_THREADS].value.u64 = thread_manager->get_thread_count(); + buffer[SINSP_STATS_V2_N_FDS].value.u64 = 0; threadinfo_map_t* threadtable = thread_manager->get_threads(); threadtable->loop([&] (sinsp_threadinfo& tinfo) { sinsp_fdtable* fdtable = tinfo.get_fd_table(); - stats[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); + buffer[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); return true; }); - stats[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_noncached_fd_lookups; - stats[SINSP_STATS_V2_CACHED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_cached_fd_lookups; - stats[SINSP_STATS_V2_FAILED_FD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_failed_fd_lookups; - stats[SINSP_STATS_V2_ADDED_FDS].value.u64 = sinsp_stats_v2_counters.m_n_added_fds; - stats[SINSP_STATS_V2_REMOVED_FDS].value.u64 = sinsp_stats_v2_counters.m_n_removed_fds; - stats[SINSP_STATS_V2_STORED_EVTS].value.u64 = sinsp_stats_v2_counters.m_n_stored_evts; - stats[SINSP_STATS_V2_STORE_EVTS_DROPS].value.u64 = sinsp_stats_v2_counters.m_n_store_evts_drops; - stats[SINSP_STATS_V2_RETRIEVED_EVTS].value.u64 = sinsp_stats_v2_counters.m_n_retrieved_evts; - stats[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].value.u64 = sinsp_stats_v2_counters.m_n_retrieve_evts_drops; - stats[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_noncached_thread_lookups; - stats[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_cached_thread_lookups; - stats[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = sinsp_stats_v2_counters.m_n_failed_thread_lookups; - stats[SINSP_STATS_V2_ADDED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_added_threads; - stats[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = sinsp_stats_v2_counters.m_n_removed_threads; - stats[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = thread_manager->get_m_n_drops(); - stats[SINSP_STATS_V2_N_CONTAINERS].value.u32 = n_containers; + buffer[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = stats_v2.m_n_noncached_fd_lookups; + buffer[SINSP_STATS_V2_CACHED_FD_LOOKUPS].value.u64 = stats_v2.m_n_cached_fd_lookups; + buffer[SINSP_STATS_V2_FAILED_FD_LOOKUPS].value.u64 = stats_v2.m_n_failed_fd_lookups; + buffer[SINSP_STATS_V2_ADDED_FDS].value.u64 = stats_v2.m_n_added_fds; + buffer[SINSP_STATS_V2_REMOVED_FDS].value.u64 = stats_v2.m_n_removed_fds; + buffer[SINSP_STATS_V2_STORED_EVTS].value.u64 = stats_v2.m_n_stored_evts; + buffer[SINSP_STATS_V2_STORE_EVTS_DROPS].value.u64 = stats_v2.m_n_store_evts_drops; + buffer[SINSP_STATS_V2_RETRIEVED_EVTS].value.u64 = stats_v2.m_n_retrieved_evts; + buffer[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].value.u64 = stats_v2.m_n_retrieve_evts_drops; + buffer[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_noncached_thread_lookups; + buffer[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_cached_thread_lookups; + buffer[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_failed_thread_lookups; + buffer[SINSP_STATS_V2_ADDED_THREADS].value.u64 = stats_v2.m_n_added_threads; + buffer[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = stats_v2.m_n_removed_threads; + buffer[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = thread_manager->get_m_n_drops(); + buffer[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES].value.u32 = stats_v2.m_n_missing_container_images; + buffer[SINSP_STATS_V2_N_CONTAINERS].value.u32 = stats_v2.m_n_containers; *nstats = SINSP_MAX_STATS_V2; } *rc = SCAP_SUCCESS; - return stats; + return buffer; } + +#endif diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index 2dd95d9d66..ccc0a7406e 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -37,6 +37,8 @@ typedef struct sinsp_stats_v2 uint64_t m_n_failed_thread_lookups; uint64_t m_n_added_threads; uint64_t m_n_removed_threads; + uint32_t m_n_missing_container_images; + uint32_t m_n_containers; }sinsp_stats_v2; typedef enum sinsp_stats_v2_resource_utilization { @@ -66,10 +68,12 @@ typedef enum sinsp_stats_v2_resource_utilization { SINSP_STATS_V2_ADDED_THREADS, ///< threadtable state related counters, unit: count. SINSP_STATS_V2_REMOVED_THREADS, ///< threadtable state related counters, unit: count. SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE, ///< Number of drops due to full threadtable, unit: count. - SINSP_STATS_V2_N_CONTAINERS, ///< Number of containers currently cached by sinsp_container_manager, unit: count. + SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES, ///< Number of cached containers (cgroups) without container info such as image, hijacked sinsp_container_manager::remove_inactive_containers() -> every flush snapshot update, unit: count. + SINSP_STATS_V2_N_CONTAINERS, ///< Number of containers (cgroups) currently cached by sinsp_container_manager, hijacked sinsp_container_manager::remove_inactive_containers() -> every flush snapshot update, unit: count. SINSP_MAX_STATS_V2 }sinsp_stats_v2_resource_utilization; +#ifdef __linux__ namespace libsinsp { namespace stats { @@ -77,16 +81,16 @@ namespace stats { \brief Retrieve current sinsp stats v2 including resource utilization metrics. \param agent_info Pointer to a \ref scap_agent_info containing relevant constants from the agent start up moment. \param thread_manager Pointer to a \ref thread_manager to access threadtable properties. - \param sinsp_stats_v2_counters sinsp_stats_v2 struct containing counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities). - \param stats Pointer to a \ref scap_stats_v2 pre-allocated sinsp_stats_v2_buffer (aka scap_stats_v2 schema). - \param n_containers Number of containers currently cached by sinsp_container_manager. + \param stats_v2 sinsp_stats_v2 struct containing counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities). + \param buffer Pointer to a \ref scap_stats_v2 pre-allocated sinsp_stats_v2_buffer (aka scap_stats_v2 schema). \param nstats Pointer reflecting number of statistics in returned buffer \param rc Pointer to return code \note Intended to be called once every x hours. \return Pointer to a \ref scap_stats_v2 buffer filled with the current sinsp stats v2 including resource utilization metrics. */ - const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 sinsp_stats_v2_counters, scap_stats_v2* stats, uint32_t n_containers, uint32_t* nstats, int32_t* rc); + const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc); } } +#endif diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp index 635d3536d5..662725591b 100644 --- a/userspace/libsinsp/test/sinsp_stats.ut.cpp +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -42,16 +42,15 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) auto buffer = m_inspector.get_sinsp_stats_v2_buffer(); sinsp_stats_v2 sinsp_stats_v2_counters = m_inspector.get_sinsp_stats_v2(); sinsp_thread_manager* thread_manager = m_inspector.m_thread_manager; - uint32_t n_containers = m_inspector.m_container_manager.get_containers()->size(); uint32_t flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); /* Extra call */ - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); ASSERT_EQ(nstats, SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1); ASSERT_EQ(rc, SCAP_SUCCESS); flags = (PPM_SCAP_STATS_RESOURCE_UTILIZATION | PPM_SCAP_STATS_STATE_COUNTERS); - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); ASSERT_EQ(nstats, SINSP_MAX_STATS_V2); ASSERT_EQ(rc, SCAP_SUCCESS); @@ -90,7 +89,7 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) /* Empty call */ nstats = 0; flags = 0; - sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, n_containers, &nstats, &rc); + sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); ASSERT_EQ(nstats, 0); ASSERT_EQ(rc, SCAP_SUCCESS); From 042db53ad13fd965b686aae53b5b0edf03b15a08 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Wed, 25 Oct 2023 19:07:29 +0000 Subject: [PATCH 07/10] fix(cmake): adjust setup re sinsp stats Co-authored-by: Federico Di Pierro Signed-off-by: Melissa Kilby --- userspace/libsinsp/CMakeLists.txt | 2 +- userspace/libsinsp/test/sinsp_stats.ut.cpp | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/userspace/libsinsp/CMakeLists.txt b/userspace/libsinsp/CMakeLists.txt index 91faf419d6..60ab68a001 100644 --- a/userspace/libsinsp/CMakeLists.txt +++ b/userspace/libsinsp/CMakeLists.txt @@ -123,6 +123,7 @@ set(SINSP_SOURCES threadinfo.cpp tuples.cpp sinsp.cpp + stats.cpp token_bucket.cpp stopwatch.cpp utils.cpp @@ -168,7 +169,6 @@ if(NOT MINIMAL_BUILD AND NOT EMSCRIPTEN) list(APPEND SINSP_SOURCES cgroup_limits.cpp container_engine/cri.cpp - stats.cpp grpc_channel_registry.cpp ) endif() diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp index 662725591b..0c9f177446 100644 --- a/userspace/libsinsp/test/sinsp_stats.ut.cpp +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -15,22 +15,16 @@ See the License for the specific language governing permissions and limitations under the License. */ +#ifdef __linux__ #include -#include "sinsp.h" -#include "utils.h" #include "sinsp_with_test_input.h" -#include "test_utils.h" -#include "scap_stats_v2.h" #include TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) { -#ifdef __linux__ - - open_inspector(SCAP_MODE_LIVE); - // Adopted from test: TEST_F(sinsp_with_test_input, PROC_FILTER_nthreads) - DEFAULT_TREE + // Adopted from test: TEST_F(sinsp_with_test_input, PROC_FILTER_nthreads) + DEFAULT_TREE /* we call a random event to obtain an event associated with this thread info */ auto evt = generate_random_event(p2_t1_tid); ASSERT_EQ(get_field_as_string(evt, "proc.nthreads"), "3"); @@ -92,6 +86,5 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); ASSERT_EQ(nstats, 0); ASSERT_EQ(rc, SCAP_SUCCESS); - -#endif } +#endif From 88beedb65659cca1841ccdb1fba5c1d0668adbb7 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Mon, 30 Oct 2023 22:08:01 +0000 Subject: [PATCH 08/10] cleanup(libsinsp): add enable option for m_sinsp_stats_v2 * make m_sinsp_stats_v2 a smart pointer initialized only when enabling sinsp_stats_v2 counters * only keep m_inspector nullptr check for `sinsp_thread_manager::remove_thread(int64_t tid)` Co-authored-by: Federico Di Pierro Signed-off-by: Melissa Kilby --- userspace/libsinsp/container.cpp | 10 ++--- userspace/libsinsp/fdinfo.cpp | 26 +++++------ userspace/libsinsp/parsers.cpp | 36 ++++++++-------- userspace/libsinsp/sinsp.cpp | 43 +++++++++++-------- userspace/libsinsp/sinsp.h | 9 +++- userspace/libsinsp/stats.cpp | 38 ++++++++-------- userspace/libsinsp/stats.h | 4 +- userspace/libsinsp/test/sinsp_stats.ut.cpp | 9 ++-- .../libsinsp/test/sinsp_with_test_input.h | 3 ++ userspace/libsinsp/threadinfo.cpp | 25 +++++------ 10 files changed, 111 insertions(+), 92 deletions(-) diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp index 90a742a089..f17165d062 100644 --- a/userspace/libsinsp/container.cpp +++ b/userspace/libsinsp/container.cpp @@ -88,20 +88,20 @@ bool sinsp_container_manager::remove_inactive_containers() }); auto containers = m_containers.lock(); - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_missing_container_images = 0; - m_inspector->m_sinsp_stats_v2.m_n_containers = containers->size(); + m_inspector->m_sinsp_stats_v2->m_n_missing_container_images = 0; + m_inspector->m_sinsp_stats_v2->m_n_containers = containers->size(); } for(auto it = containers->begin(); it != containers->end();) { sinsp_container_info::ptr_t container = it->second; - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { auto container_info = container.get(); if (!container_info || (container_info && container_info->m_image.empty())) { - m_inspector->m_sinsp_stats_v2.m_n_missing_container_images++; + m_inspector->m_sinsp_stats_v2->m_n_missing_container_images++; } } if(containers_in_use.find(it->first) == containers_in_use.end()) diff --git a/userspace/libsinsp/fdinfo.cpp b/userspace/libsinsp/fdinfo.cpp index 911f039dea..d9f60b718f 100644 --- a/userspace/libsinsp/fdinfo.cpp +++ b/userspace/libsinsp/fdinfo.cpp @@ -354,9 +354,9 @@ sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) // if(m_last_accessed_fd != -1 && fd == m_last_accessed_fd) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_cached_fd_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_cached_fd_lookups++; } return m_last_accessed_fdinfo; } @@ -368,17 +368,17 @@ sinsp_fdinfo_t* sinsp_fdtable::find(int64_t fd) if(fdit == m_table.end()) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_failed_fd_lookups++; } return NULL; } else { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_noncached_fd_lookups++; } m_last_accessed_fd = fd; @@ -408,9 +408,9 @@ sinsp_fdinfo_t* sinsp_fdtable::add(int64_t fd, sinsp_fdinfo_t* fdinfo) // No entry in the table, this is the normal case // m_last_accessed_fd = -1; - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_added_fds++; + m_inspector->m_sinsp_stats_v2->m_n_added_fds++; } std::pair::iterator, bool> insert_res = m_table.emplace(fd, *fdinfo); @@ -480,18 +480,18 @@ void sinsp_fdtable::erase(int64_t fd) // keep going. // ASSERT(false); - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_failed_fd_lookups++; } } else { m_table.erase(fdit); - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_noncached_fd_lookups++; - m_inspector->m_sinsp_stats_v2.m_n_removed_fds++; + m_inspector->m_sinsp_stats_v2->m_n_noncached_fd_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_removed_fds++; } } } diff --git a/userspace/libsinsp/parsers.cpp b/userspace/libsinsp/parsers.cpp index f28914c990..09052fdabf 100644 --- a/userspace/libsinsp/parsers.cpp +++ b/userspace/libsinsp/parsers.cpp @@ -734,9 +734,9 @@ bool sinsp_parser::reset(sinsp_evt *evt) etype == PPME_SYSCALL_VFORK_20_X || etype == PPME_SYSCALL_CLONE3_X) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups--; + m_inspector->m_sinsp_stats_v2->m_n_failed_thread_lookups--; } } return false; @@ -883,9 +883,9 @@ void sinsp_parser::store_event(sinsp_evt *evt) // we won't be able to parse the corresponding exit event and we'll have // to drop the information it carries. // - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_store_evts_drops++; + m_inspector->m_sinsp_stats_v2->m_n_store_evts_drops++; } return; } @@ -919,9 +919,9 @@ void sinsp_parser::store_event(sinsp_evt *evt) memcpy(tinfo->m_lastevent_data, evt->m_pevt, elen); tinfo->m_lastevent_cpuid = evt->get_cpuid(); - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_stored_evts++; + m_inspector->m_sinsp_stats_v2->m_n_stored_evts++; } } @@ -944,9 +944,9 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev // This happen especially at the beginning of trace files, where events // can be truncated // - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; + m_inspector->m_sinsp_stats_v2->m_n_retrieve_evts_drops++; } return false; } @@ -967,9 +967,9 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev && enter_evt->get_type() == PPME_SYSCALL_EXECVEAT_E) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; + m_inspector->m_sinsp_stats_v2->m_n_retrieved_evts++; } return true; } @@ -982,15 +982,15 @@ bool sinsp_parser::retrieve_enter_event(sinsp_evt *enter_evt, sinsp_evt *exit_ev { //ASSERT(false); exit_evt->m_tinfo->set_lastevent_data_validity(false); - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_retrieve_evts_drops++; + m_inspector->m_sinsp_stats_v2->m_n_retrieve_evts_drops++; } return false; } - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_retrieved_evts++; + m_inspector->m_sinsp_stats_v2->m_n_retrieved_evts++; } return true; @@ -3785,15 +3785,15 @@ void sinsp_parser::parse_close_exit(sinsp_evt *evt) // It is normal when a close fails that the fd lookup failed, so we revert the // increment of m_n_failed_fd_lookups (for the enter event too if there's one). // - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; + m_inspector->m_sinsp_stats_v2->m_n_failed_fd_lookups--; } if(evt->m_tinfo && evt->m_tinfo->is_lastevent_data_valid()) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_fd_lookups--; + m_inspector->m_sinsp_stats_v2->m_n_failed_fd_lookups--; } } } diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index ff1f48c0d7..231afa31ff 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -340,23 +340,6 @@ void sinsp::init() // Basic inits // - m_sinsp_stats_v2.m_n_added_fds = 0; - m_sinsp_stats_v2.m_n_cached_fd_lookups = 0; - m_sinsp_stats_v2.m_n_failed_fd_lookups = 0; - m_sinsp_stats_v2.m_n_added_fds = 0; - m_sinsp_stats_v2.m_n_removed_fds = 0; - m_sinsp_stats_v2.m_n_stored_evts = 0; - m_sinsp_stats_v2.m_n_store_evts_drops = 0; - m_sinsp_stats_v2.m_n_retrieved_evts = 0; - m_sinsp_stats_v2.m_n_retrieve_evts_drops = 0; - m_sinsp_stats_v2.m_n_noncached_thread_lookups = 0; - m_sinsp_stats_v2.m_n_cached_thread_lookups = 0; - m_sinsp_stats_v2.m_n_failed_thread_lookups = 0; - m_sinsp_stats_v2.m_n_added_threads = 0; - m_sinsp_stats_v2.m_n_removed_threads = 0; - m_sinsp_stats_v2.m_n_missing_container_images = 0; - m_sinsp_stats_v2.m_n_containers= 0; - m_nevts = 0; m_tid_to_remove = -1; m_lastevent_ts = 0; @@ -2047,7 +2030,7 @@ scap_stats_v2* sinsp::get_sinsp_stats_v2_buffer() return m_sinsp_stats_v2_buffer; } -sinsp_stats_v2 sinsp::get_sinsp_stats_v2() +std::shared_ptr sinsp::get_sinsp_stats_v2() { return m_sinsp_stats_v2; } @@ -2338,6 +2321,30 @@ void sinsp::set_proc_scan_log_interval_ms(uint64_t val) m_proc_scan_log_interval_ms = val; } +void sinsp::set_sinsp_stats_v2_enabled() +{ + if (m_sinsp_stats_v2 == nullptr) + { + m_sinsp_stats_v2 = std::make_unique(); + m_sinsp_stats_v2->m_n_noncached_fd_lookups = 0; + m_sinsp_stats_v2->m_n_cached_fd_lookups = 0; + m_sinsp_stats_v2->m_n_failed_fd_lookups = 0; + m_sinsp_stats_v2->m_n_added_fds = 0; + m_sinsp_stats_v2->m_n_removed_fds = 0; + m_sinsp_stats_v2->m_n_stored_evts = 0; + m_sinsp_stats_v2->m_n_store_evts_drops = 0; + m_sinsp_stats_v2->m_n_retrieved_evts = 0; + m_sinsp_stats_v2->m_n_retrieve_evts_drops = 0; + m_sinsp_stats_v2->m_n_noncached_thread_lookups = 0; + m_sinsp_stats_v2->m_n_cached_thread_lookups = 0; + m_sinsp_stats_v2->m_n_failed_thread_lookups = 0; + m_sinsp_stats_v2->m_n_added_threads = 0; + m_sinsp_stats_v2->m_n_removed_threads = 0; + m_sinsp_stats_v2->m_n_missing_container_images = 0; + m_sinsp_stats_v2->m_n_containers= 0; + } +} + /////////////////////////////////////////////////////////////////////////////// // Note: this is defined here so we can inline it in sinso::next /////////////////////////////////////////////////////////////////////////////// diff --git a/userspace/libsinsp/sinsp.h b/userspace/libsinsp/sinsp.h index 7069d654ab..9e0695d76e 100644 --- a/userspace/libsinsp/sinsp.h +++ b/userspace/libsinsp/sinsp.h @@ -438,6 +438,11 @@ class SINSP_PUBLIC sinsp : public capture_stats_source */ void set_proc_scan_log_interval_ms(uint64_t val); + /*! + * \brief enabling sinsp state counters on the hot path via initializing the respective smart pointer. + */ + void set_sinsp_stats_v2_enabled(); + /*! \brief Start writing the captured events to file. @@ -507,7 +512,7 @@ class SINSP_PUBLIC sinsp : public capture_stats_source \note sinsp stats may be refactored near-term. */ - sinsp_stats_v2 get_sinsp_stats_v2(); + std::shared_ptr get_sinsp_stats_v2(); /*! \brief Look up a thread given its tid and return its information, @@ -1103,7 +1108,7 @@ VISIBILITY_PRIVATE bool m_is_dumping; const scap_machine_info* m_machine_info; const scap_agent_info* m_agent_info; - sinsp_stats_v2 m_sinsp_stats_v2; + std::shared_ptr m_sinsp_stats_v2; scap_stats_v2 m_sinsp_stats_v2_buffer[SINSP_MAX_STATS_V2]; uint32_t m_num_cpus; bool m_is_tracers_capture_enabled; diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index 7232598b1b..d6358c06d4 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -292,7 +292,7 @@ uint64_t get_container_memory_usage() return memory_used; } -const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc) +const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, std::shared_ptr stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc) { if (!buffer) { @@ -301,6 +301,7 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s return NULL; } + *nstats = 0; if((flags & PPM_SCAP_STATS_RESOURCE_UTILIZATION)) { uint32_t rss = 0; @@ -351,7 +352,7 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s } - if((flags & PPM_SCAP_STATS_STATE_COUNTERS)) + if((flags & PPM_SCAP_STATS_STATE_COUNTERS) && stats_v2) { if(buffer[SINSP_STATS_V2_N_THREADS].name != nullptr && strncmp(buffer[SINSP_STATS_V2_N_THREADS].name, sinsp_stats_v2_resource_utilization_names[SINSP_STATS_V2_N_THREADS], 10) != 0) { @@ -392,29 +393,28 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s buffer[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); return true; }); - buffer[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = stats_v2.m_n_noncached_fd_lookups; - buffer[SINSP_STATS_V2_CACHED_FD_LOOKUPS].value.u64 = stats_v2.m_n_cached_fd_lookups; - buffer[SINSP_STATS_V2_FAILED_FD_LOOKUPS].value.u64 = stats_v2.m_n_failed_fd_lookups; - buffer[SINSP_STATS_V2_ADDED_FDS].value.u64 = stats_v2.m_n_added_fds; - buffer[SINSP_STATS_V2_REMOVED_FDS].value.u64 = stats_v2.m_n_removed_fds; - buffer[SINSP_STATS_V2_STORED_EVTS].value.u64 = stats_v2.m_n_stored_evts; - buffer[SINSP_STATS_V2_STORE_EVTS_DROPS].value.u64 = stats_v2.m_n_store_evts_drops; - buffer[SINSP_STATS_V2_RETRIEVED_EVTS].value.u64 = stats_v2.m_n_retrieved_evts; - buffer[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].value.u64 = stats_v2.m_n_retrieve_evts_drops; - buffer[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_noncached_thread_lookups; - buffer[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_cached_thread_lookups; - buffer[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = stats_v2.m_n_failed_thread_lookups; - buffer[SINSP_STATS_V2_ADDED_THREADS].value.u64 = stats_v2.m_n_added_threads; - buffer[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = stats_v2.m_n_removed_threads; + buffer[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = stats_v2->m_n_noncached_fd_lookups; + buffer[SINSP_STATS_V2_CACHED_FD_LOOKUPS].value.u64 = stats_v2->m_n_cached_fd_lookups; + buffer[SINSP_STATS_V2_FAILED_FD_LOOKUPS].value.u64 = stats_v2->m_n_failed_fd_lookups; + buffer[SINSP_STATS_V2_ADDED_FDS].value.u64 = stats_v2->m_n_added_fds; + buffer[SINSP_STATS_V2_REMOVED_FDS].value.u64 = stats_v2->m_n_removed_fds; + buffer[SINSP_STATS_V2_STORED_EVTS].value.u64 = stats_v2->m_n_stored_evts; + buffer[SINSP_STATS_V2_STORE_EVTS_DROPS].value.u64 = stats_v2->m_n_store_evts_drops; + buffer[SINSP_STATS_V2_RETRIEVED_EVTS].value.u64 = stats_v2->m_n_retrieved_evts; + buffer[SINSP_STATS_V2_RETRIEVE_EVTS_DROPS].value.u64 = stats_v2->m_n_retrieve_evts_drops; + buffer[SINSP_STATS_V2_NONCACHED_THREAD_LOOKUPS].value.u64 = stats_v2->m_n_noncached_thread_lookups; + buffer[SINSP_STATS_V2_CACHED_THREAD_LOOKUPS].value.u64 = stats_v2->m_n_cached_thread_lookups; + buffer[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = stats_v2->m_n_failed_thread_lookups; + buffer[SINSP_STATS_V2_ADDED_THREADS].value.u64 = stats_v2->m_n_added_threads; + buffer[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = stats_v2->m_n_removed_threads; buffer[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = thread_manager->get_m_n_drops(); - buffer[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES].value.u32 = stats_v2.m_n_missing_container_images; - buffer[SINSP_STATS_V2_N_CONTAINERS].value.u32 = stats_v2.m_n_containers; + buffer[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES].value.u32 = stats_v2->m_n_missing_container_images; + buffer[SINSP_STATS_V2_N_CONTAINERS].value.u32 = stats_v2->m_n_containers; *nstats = SINSP_MAX_STATS_V2; } *rc = SCAP_SUCCESS; - return buffer; } diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index ccc0a7406e..c65d604fb3 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -81,7 +81,7 @@ namespace stats { \brief Retrieve current sinsp stats v2 including resource utilization metrics. \param agent_info Pointer to a \ref scap_agent_info containing relevant constants from the agent start up moment. \param thread_manager Pointer to a \ref thread_manager to access threadtable properties. - \param stats_v2 sinsp_stats_v2 struct containing counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities). + \param stats_v2 Pointer to a \ref sinsp_stats_v2 containing counters related to the sinsp state tables (e.g. adding, removing, storing, failed lookup activities). \param buffer Pointer to a \ref scap_stats_v2 pre-allocated sinsp_stats_v2_buffer (aka scap_stats_v2 schema). \param nstats Pointer reflecting number of statistics in returned buffer \param rc Pointer to return code @@ -89,7 +89,7 @@ namespace stats { \return Pointer to a \ref scap_stats_v2 buffer filled with the current sinsp stats v2 including resource utilization metrics. */ - const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, sinsp_stats_v2 stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc); + const scap_stats_v2* get_sinsp_stats_v2(uint32_t flags, const scap_agent_info* agent_info, sinsp_thread_manager* thread_manager, std::shared_ptr stats_v2, scap_stats_v2* buffer, uint32_t* nstats, int32_t* rc); } } diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp index 0c9f177446..c229bbed0f 100644 --- a/userspace/libsinsp/test/sinsp_stats.ut.cpp +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -34,11 +34,14 @@ TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) int32_t rc; const scap_stats_v2* sinsp_stats_v2_snapshot; auto buffer = m_inspector.get_sinsp_stats_v2_buffer(); - sinsp_stats_v2 sinsp_stats_v2_counters = m_inspector.get_sinsp_stats_v2(); + auto sinsp_stats_v2_counters = m_inspector.get_sinsp_stats_v2(); sinsp_thread_manager* thread_manager = m_inspector.m_thread_manager; - uint32_t flags = PPM_SCAP_STATS_RESOURCE_UTILIZATION; + uint32_t flags = 0; sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); - /* Extra call */ + ASSERT_EQ(nstats, 0); + ASSERT_EQ(rc, SCAP_SUCCESS); + /* Extra call */ + flags |= PPM_SCAP_STATS_RESOURCE_UTILIZATION; sinsp_stats_v2_snapshot = libsinsp::stats::get_sinsp_stats_v2(flags, agent_info, thread_manager, sinsp_stats_v2_counters, buffer, &nstats, &rc); ASSERT_EQ(nstats, SINSP_RESOURCE_UTILIZATION_FDS_TOTAL_HOST + 1); ASSERT_EQ(rc, SCAP_SUCCESS); diff --git a/userspace/libsinsp/test/sinsp_with_test_input.h b/userspace/libsinsp/test/sinsp_with_test_input.h index d69a6bf8e7..a3c32ac82a 100644 --- a/userspace/libsinsp/test/sinsp_with_test_input.h +++ b/userspace/libsinsp/test/sinsp_with_test_input.h @@ -60,6 +60,9 @@ class sinsp_with_test_input : public ::testing::Test { void open_inspector(sinsp_mode_t mode = SINSP_MODE_TEST) { m_inspector.open_test_input(m_test_data.get(), mode); + m_inspector.set_sinsp_stats_v2_enabled(); + // Extra call to see we don;t fail + m_inspector.set_sinsp_stats_v2_enabled(); } scap_evt* add_event(uint64_t ts, uint64_t tid, ppm_event_code event_type, uint32_t n, ...) diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index d3ab783e0b..a9947e83dc 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -1509,9 +1509,9 @@ std::unique_ptr sinsp_thread_manager::new_threadinfo() const */ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_scap_proctable) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_added_threads++; + m_inspector->m_sinsp_stats_v2->m_n_added_threads++; } /* We have no more space */ @@ -1694,9 +1694,10 @@ void sinsp_thread_manager::remove_thread(int64_t tid) /* This should never happen but just to be sure. */ if(thread_to_remove == nullptr) { - if (m_inspector != nullptr) + // Extra m_inspector nullptr check + if (m_inspector != nullptr && m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_failed_thread_lookups++; } return; } @@ -1816,9 +1817,9 @@ void sinsp_thread_manager::remove_thread(int64_t tid) * the cache just to be sure. */ m_last_tid = -1; - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_removed_threads++; + m_inspector->m_sinsp_stats_v2->m_n_removed_threads++; } } @@ -2155,9 +2156,9 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look thr = m_last_tinfo.lock(); if (thr) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_cached_thread_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_cached_thread_lookups++; } // This allows us to avoid performing an actual timestamp lookup // for something that may not need to be precise @@ -2173,9 +2174,9 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look if(thr) { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_noncached_thread_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_noncached_thread_lookups++; } if(!lookup_only) { @@ -2188,9 +2189,9 @@ threadinfo_map_t::ptr_t sinsp_thread_manager::find_thread(int64_t tid, bool look } else { - if (m_inspector != nullptr) + if (m_inspector->m_sinsp_stats_v2) { - m_inspector->m_sinsp_stats_v2.m_n_failed_thread_lookups++; + m_inspector->m_sinsp_stats_v2->m_n_failed_thread_lookups++; } return NULL; } From a4ccc7c37de7e06666f18dbce26621059444b56e Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Mon, 30 Oct 2023 22:23:53 +0000 Subject: [PATCH 09/10] cleanup(libsinsp): don't count sanboxes as missing_container_images Signed-off-by: Melissa Kilby --- userspace/libsinsp/container.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/userspace/libsinsp/container.cpp b/userspace/libsinsp/container.cpp index f17165d062..df4f357d39 100644 --- a/userspace/libsinsp/container.cpp +++ b/userspace/libsinsp/container.cpp @@ -91,6 +91,7 @@ bool sinsp_container_manager::remove_inactive_containers() if (m_inspector->m_sinsp_stats_v2) { m_inspector->m_sinsp_stats_v2->m_n_missing_container_images = 0; + // Will include pod sanboxes, but that's ok m_inspector->m_sinsp_stats_v2->m_n_containers = containers->size(); } for(auto it = containers->begin(); it != containers->end();) @@ -99,8 +100,9 @@ bool sinsp_container_manager::remove_inactive_containers() if (m_inspector->m_sinsp_stats_v2) { auto container_info = container.get(); - if (!container_info || (container_info && container_info->m_image.empty())) + if (!container_info || (container_info && !container_info->m_is_pod_sandbox && container_info->m_image.empty())) { + // Only count missing container images and exclude sandboxes m_inspector->m_sinsp_stats_v2->m_n_missing_container_images++; } } From 0c856e8e58c01e8eaa025047208a6a415e4f1dd4 Mon Sep 17 00:00:00 2001 From: Melissa Kilby Date: Mon, 13 Nov 2023 23:45:45 +0000 Subject: [PATCH 10/10] cleanup(libsinsp): apply reviewers suggestions re sinsp stats v2 Co-authored-by: Andrea Terzolo Signed-off-by: Melissa Kilby --- userspace/libsinsp/sinsp.cpp | 2 +- userspace/libsinsp/stats.cpp | 7 ++++-- userspace/libsinsp/stats.h | 1 + userspace/libsinsp/test/sinsp_stats.ut.cpp | 3 +++ .../libsinsp/test/sinsp_with_test_input.h | 3 --- userspace/libsinsp/threadinfo.cpp | 23 +++++++++++-------- userspace/libsinsp/threadinfo.h | 2 -- 7 files changed, 23 insertions(+), 18 deletions(-) diff --git a/userspace/libsinsp/sinsp.cpp b/userspace/libsinsp/sinsp.cpp index 231afa31ff..8e3196cd2b 100644 --- a/userspace/libsinsp/sinsp.cpp +++ b/userspace/libsinsp/sinsp.cpp @@ -43,7 +43,6 @@ limitations under the License. #include "plugin_filtercheck.h" #include "strl.h" #include "scap-int.h" -#include "stats.h" #if defined(HAS_CAPTURE) && !defined(CYGWING_AGENT) && !defined(MINIMAL_BUILD) && !defined(__EMSCRIPTEN__) #include @@ -2340,6 +2339,7 @@ void sinsp::set_sinsp_stats_v2_enabled() m_sinsp_stats_v2->m_n_failed_thread_lookups = 0; m_sinsp_stats_v2->m_n_added_threads = 0; m_sinsp_stats_v2->m_n_removed_threads = 0; + m_sinsp_stats_v2->m_n_drops_full_threadtable = 0; m_sinsp_stats_v2->m_n_missing_container_images = 0; m_sinsp_stats_v2->m_n_containers= 0; } diff --git a/userspace/libsinsp/stats.cpp b/userspace/libsinsp/stats.cpp index d6358c06d4..82e8fcc0ef 100644 --- a/userspace/libsinsp/stats.cpp +++ b/userspace/libsinsp/stats.cpp @@ -390,7 +390,10 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s threadinfo_map_t* threadtable = thread_manager->get_threads(); threadtable->loop([&] (sinsp_threadinfo& tinfo) { sinsp_fdtable* fdtable = tinfo.get_fd_table(); - buffer[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); + if (fdtable != nullptr) + { + buffer[SINSP_STATS_V2_N_FDS].value.u64 += fdtable->size(); + } return true; }); buffer[SINSP_STATS_V2_NONCACHED_FD_LOOKUPS].value.u64 = stats_v2->m_n_noncached_fd_lookups; @@ -407,7 +410,7 @@ const scap_stats_v2* libsinsp::stats::get_sinsp_stats_v2(uint32_t flags, const s buffer[SINSP_STATS_V2_FAILED_THREAD_LOOKUPS].value.u64 = stats_v2->m_n_failed_thread_lookups; buffer[SINSP_STATS_V2_ADDED_THREADS].value.u64 = stats_v2->m_n_added_threads; buffer[SINSP_STATS_V2_REMOVED_THREADS].value.u64 = stats_v2->m_n_removed_threads; - buffer[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = thread_manager->get_m_n_drops(); + buffer[SINSP_STATS_V2_N_DROPS_FULL_THREADTABLE].value.u32 = stats_v2->m_n_drops_full_threadtable; buffer[SINSP_STATS_V2_N_MISSING_CONTAINER_IMAGES].value.u32 = stats_v2->m_n_missing_container_images; buffer[SINSP_STATS_V2_N_CONTAINERS].value.u32 = stats_v2->m_n_containers; diff --git a/userspace/libsinsp/stats.h b/userspace/libsinsp/stats.h index c65d604fb3..6aea8e509e 100644 --- a/userspace/libsinsp/stats.h +++ b/userspace/libsinsp/stats.h @@ -37,6 +37,7 @@ typedef struct sinsp_stats_v2 uint64_t m_n_failed_thread_lookups; uint64_t m_n_added_threads; uint64_t m_n_removed_threads; + uint32_t m_n_drops_full_threadtable; uint32_t m_n_missing_container_images; uint32_t m_n_containers; }sinsp_stats_v2; diff --git a/userspace/libsinsp/test/sinsp_stats.ut.cpp b/userspace/libsinsp/test/sinsp_stats.ut.cpp index c229bbed0f..34e5faede9 100644 --- a/userspace/libsinsp/test/sinsp_stats.ut.cpp +++ b/userspace/libsinsp/test/sinsp_stats.ut.cpp @@ -23,6 +23,9 @@ limitations under the License. TEST_F(sinsp_with_test_input, sinsp_stats_v2_resource_utilization) { + m_inspector.set_sinsp_stats_v2_enabled(); + // Extra call to see we don't fail + m_inspector.set_sinsp_stats_v2_enabled(); // Adopted from test: TEST_F(sinsp_with_test_input, PROC_FILTER_nthreads) DEFAULT_TREE /* we call a random event to obtain an event associated with this thread info */ diff --git a/userspace/libsinsp/test/sinsp_with_test_input.h b/userspace/libsinsp/test/sinsp_with_test_input.h index a3c32ac82a..d69a6bf8e7 100644 --- a/userspace/libsinsp/test/sinsp_with_test_input.h +++ b/userspace/libsinsp/test/sinsp_with_test_input.h @@ -60,9 +60,6 @@ class sinsp_with_test_input : public ::testing::Test { void open_inspector(sinsp_mode_t mode = SINSP_MODE_TEST) { m_inspector.open_test_input(m_test_data.get(), mode); - m_inspector.set_sinsp_stats_v2_enabled(); - // Extra call to see we don;t fail - m_inspector.set_sinsp_stats_v2_enabled(); } scap_evt* add_event(uint64_t ts, uint64_t tid, ppm_event_code event_type, uint32_t n, ...) diff --git a/userspace/libsinsp/threadinfo.cpp b/userspace/libsinsp/threadinfo.cpp index a9947e83dc..f672c2410b 100644 --- a/userspace/libsinsp/threadinfo.cpp +++ b/userspace/libsinsp/threadinfo.cpp @@ -1423,7 +1423,6 @@ void sinsp_thread_manager::clear() m_thread_groups.clear(); m_last_tid = 0; m_last_flush_time_ns = 0; - m_n_drops = 0; } /* This is called on the table after the `/proc` scan */ @@ -1509,10 +1508,6 @@ std::unique_ptr sinsp_thread_manager::new_threadinfo() const */ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_scap_proctable) { - if (m_inspector->m_sinsp_stats_v2) - { - m_inspector->m_sinsp_stats_v2->m_n_added_threads++; - } /* We have no more space */ if(m_threadtable.size() >= m_max_thread_table_size @@ -1521,13 +1516,16 @@ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_sc #endif ) { - // rate limit messages to avoid spamming the logs - if (m_n_drops % m_max_thread_table_size == 0) + if (m_inspector->m_sinsp_stats_v2) { - g_logger.format(sinsp_logger::SEV_INFO, "Thread table full, dropping tid %lu (pid %lu, comm \"%s\")", - threadinfo->m_tid, threadinfo->m_pid, threadinfo->m_comm.c_str()); + // rate limit messages to avoid spamming the logs + if (m_inspector->m_sinsp_stats_v2->m_n_drops_full_threadtable % m_max_thread_table_size == 0) + { + g_logger.format(sinsp_logger::SEV_INFO, "Thread table full, dropping tid %lu (pid %lu, comm \"%s\")", + threadinfo->m_tid, threadinfo->m_pid, threadinfo->m_comm.c_str()); + } + m_inspector->m_sinsp_stats_v2->m_n_drops_full_threadtable++; } - m_n_drops++; return false; } @@ -1550,6 +1548,11 @@ bool sinsp_thread_manager::add_thread(sinsp_threadinfo *threadinfo, bool from_sc tinfo_shared_ptr->compute_program_hash(); m_threadtable.put(std::move(tinfo_shared_ptr)); + if (m_inspector->m_sinsp_stats_v2) + { + m_inspector->m_sinsp_stats_v2->m_n_added_threads++; + } + return true; } diff --git a/userspace/libsinsp/threadinfo.h b/userspace/libsinsp/threadinfo.h index 14a738b5c8..0df1d08200 100644 --- a/userspace/libsinsp/threadinfo.h +++ b/userspace/libsinsp/threadinfo.h @@ -794,7 +794,6 @@ class SINSP_PUBLIC sinsp_thread_manager: public libsinsp::state::table int32_t get_m_n_proc_lookups() const { return m_n_proc_lookups; } int32_t get_m_n_main_thread_lookups() const { return m_n_main_thread_lookups; } uint64_t get_m_n_proc_lookups_duration_ns() const { return m_n_proc_lookups_duration_ns; } - uint32_t get_m_n_drops() const { return m_n_drops; } void reset_thread_counters() { m_n_proc_lookups = 0; m_n_main_thread_lookups = 0; m_n_proc_lookups_duration_ns = 0; } void set_m_max_n_proc_lookups(int32_t val) { m_max_n_proc_lookups = val; } @@ -892,7 +891,6 @@ VISIBILITY_PRIVATE int64_t m_last_tid; std::weak_ptr m_last_tinfo; uint64_t m_last_flush_time_ns; - uint32_t m_n_drops; const uint32_t m_thread_table_absolute_max_size = 131072; uint32_t m_max_thread_table_size; int32_t m_n_proc_lookups = 0;