diff --git a/.github/actions/composite-perf/action.yml b/.github/actions/composite-perf/action.yml index 5ef2106aac..79890e4eab 100644 --- a/.github/actions/composite-perf/action.yml +++ b/.github/actions/composite-perf/action.yml @@ -4,10 +4,16 @@ description: 'Run multiple perf tests on libs.' outputs: perf_tests: description: "Unit tests perf.data" - value: ${{ steps.store-outputs.outputs.tests }} + value: ${{ steps.store-outputs.outputs.perf_tests }} perf_scap: description: "Scap file perf.data" - value: ${{ steps.store-outputs.outputs.scap }} + value: ${{ steps.store-outputs.outputs.perf_scap }} + massif_tests: + description: "Unit tests heaptrack data" + value: ${{ steps.store-outputs.outputs.heaptrack_tests }} + massif_scap: + description: "Scap file heaptrack data" + value: ${{ steps.store-outputs.outputs.heaptrack_scap }} runs: using: "composite" @@ -15,35 +21,54 @@ runs: - name: Install deps ⛓️ shell: bash run: | - sudo apt update && sudo apt install -y --no-install-recommends ca-certificates cmake build-essential git clang llvm pkg-config autoconf automake libtool libelf-dev wget libc-ares-dev libcurl4-openssl-dev libssl-dev libtbb-dev libjq-dev libjsoncpp-dev libgrpc++-dev protobuf-compiler-grpc libgtest-dev libprotobuf-dev linux-tools-common linux-tools-generic linux-tools-`uname -r` + sudo apt update && sudo apt install -y --no-install-recommends ca-certificates cmake build-essential git clang llvm pkg-config autoconf automake libtool libelf-dev wget libc-ares-dev libcurl4-openssl-dev libssl-dev libtbb-dev libjq-dev libjsoncpp-dev libgrpc++-dev protobuf-compiler-grpc libgtest-dev libprotobuf-dev linux-tools-common linux-tools-generic linux-tools-`uname -r` heaptrack coreutils util-linux sudo .github/install-deps.sh - name: Build shell: bash run: | mkdir -p build - cd build && cmake -DUSE_BUNDLED_DEPS=False ../ + cd build && cmake -DUSE_BUNDLED_DEPS=False -DCMAKE_BUILD_TYPE=Debug ../ make unit-test-libsinsp -j4 make sinsp-example -j4 - - name: Run Perf - unit tests + - name: Download scap file + shell: bash + run: | + cd build + wget https://download.falco.org/fixtures/trace-files/traces-positive.zip + unzip traces-positive.zip + + - name: Run - perf unit tests shell: bash run: | cd build sudo perf record --call-graph dwarf -o perf_tests.data -q libsinsp/test/unit-test-libsinsp - - name: Run Perf - scap file + - name: Run - perf scap file shell: bash run: | cd build - wget https://download.falco.org/fixtures/trace-files/traces-positive.zip - unzip traces-positive.zip - sudo perf record --call-graph dwarf -o perf_scap.data -q ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap + sudo nice ionice -c 1 -n 0 perf record --call-graph dwarf -o perf_scap.data -q ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap + + - name: Run - heaptrack unit tests + shell: bash + run: | + cd build + sudo heaptrack -o heaptrack_tests.data libsinsp/test/unit-test-libsinsp + + - name: Run - heaptrack scap file + shell: bash + run: | + cd build + sudo nice ionice -c 1 -n 0 heaptrack -o heaptrack_scap.data ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap - name: Set Outputs id: store-outputs shell: bash run: | cd build - echo "tests=$(realpath perf_tests.data)" >> $GITHUB_OUTPUT - echo "scap=$(realpath perf_scap.data)" >> $GITHUB_OUTPUT \ No newline at end of file + echo "perf_tests=$(realpath perf_tests.data)" >> $GITHUB_OUTPUT + echo "perf_scap=$(realpath perf_scap.data)" >> $GITHUB_OUTPUT + echo "heaptrack_tests=$(realpath heaptrack_tests.data)" >> $GITHUB_OUTPUT + echo "heaptrack_scap=$(realpath heaptrack_scap.data)" >> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 6a0c34feba..09b956322b 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -35,6 +35,8 @@ jobs: path: | ${{ steps.perf.outputs.perf_tests }} ${{ steps.perf.outputs.perf_scap }} + ${{ steps.perf.outputs.heaptrack_tests }} + ${{ steps.perf.outputs.heaptrack_scap }} if-no-files-found: error - name: Checkout Flamegraph ⤵️ @@ -44,23 +46,35 @@ jobs: path: flamegraph ref: master - - name: Generate svg file - unit tests + - name: Generate svg file - perf unit tests run: | - sudo perf script --dsos unit-test-libsinsp -i ${{ steps.perf.outputs.perf_tests }} > trace_tests.perf - ./flamegraph/stackcollapse-perf.pl trace_tests.perf > trace_tests.folded - ./flamegraph/flamegraph.pl trace_tests.folded > trace_tests.svg + sudo perf script --dsos unit-test-libsinsp -i ${{ steps.perf.outputs.perf_tests }} > perf_tests.perf + ./flamegraph/stackcollapse-perf.pl perf_tests.perf > perf_tests.folded + ./flamegraph/flamegraph.pl perf_tests.folded > perf_tests.svg - - name: Generate svg file - scap file + - name: Generate svg file - perf scap file run: | - sudo perf script --dsos sinsp-example -i ${{ steps.perf.outputs.perf_scap }} > trace_scap.perf - ./flamegraph/stackcollapse-perf.pl trace_scap.perf > trace_scap.folded - ./flamegraph/flamegraph.pl trace_scap.folded > trace_scap.svg + sudo perf script --dsos sinsp-example -i ${{ steps.perf.outputs.perf_scap }} > perf_scap.perf + ./flamegraph/stackcollapse-perf.pl perf_scap.perf > perf_scap.folded + ./flamegraph/flamegraph.pl perf_scap.folded > perf_scap.svg + + - name: Generate svg file - heaptrack unit tests + run: | + heaptrack_print ${{ steps.perf.outputs.heaptrack_tests }} -F stacks.txt + ./flamegraph/flamegraph.pl stacks.txt > heaptrack_tests.svg + rm -rf stacks.txt + + - name: Generate svg file - heaptrack scap file + run: | + heaptrack_print ${{ steps.perf.outputs.heaptrack_scap }} -F stacks.txt + ./flamegraph/flamegraph.pl stacks.txt > heaptrack_scap.svg + rm -rf stacks.txt - name: Upload svg files uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: perf_svg - path: trace_*.svg + path: '*.svg' if-no-files-found: error deploy-pages: @@ -100,12 +114,18 @@ jobs: - name: Generate perf pages run: | - mv trace_tests.svg docs/trace_tests.svg - echo '' > docs/unit_tests.md - sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/unit_tests.md - mv trace_scap.svg docs/trace_scap.svg - echo '' > docs/scap_file.md - sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/scap_file.md + mv perf_tests.svg docs/ + echo '' > docs/perf_unit_tests.md + sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/perf_unit_tests.md + mv perf_scap.svg docs/ + echo '' > docs/perf_scap_file.md + sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/perf_scap_file.md + mv heaptrack_tests.svg docs/ + echo '' > docs/heaptrack_unit_tests.md + sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/heaptrack_unit_tests.md + mv heaptrack_scap.svg docs/ + echo '' > docs/heaptrack_scap_file.md + sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/heaptrack_scap_file.md - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1 with: diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index af0fb55112..e6f2c18628 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -27,19 +27,27 @@ jobs: event: push name: perf_report - - name: Diff from master - unit tests + - name: Diff from master - perf unit tests run: | sudo perf diff perf_tests.data ${{ steps.perf.outputs.perf_tests }} -d unit-test-libsinsp -b -o 1 --percentage relative -q &> perf_tests_diff.txt - - name: Diff from master - scap file + - name: Diff from master - perf scap file run: | sudo perf diff perf_scap.data ${{ steps.perf.outputs.perf_scap }} -d sinsp-example -b -o 1 --percentage relative -q &> perf_scap_diff.txt + - name: Diff from master - heaptrack unit tests + run: | + sudo heaptrack_print heaptrack_tests.data -d ${{ steps.perf.outputs.heaptrack_tests }} &> heaptrack_tests_diff.txt + + - name: Diff from master - heaptrack scap file + run: | + sudo heaptrack_print heaptrack_scap.data -d ${{ steps.perf.outputs.heaptrack_scap }} &> heaptrack_scap_diff.txt + - name: Archive perf diff uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: perf_diff - path: perf_*_diff.txt + path: '*_diff.txt' if-no-files-found: error - name: Save PR info @@ -56,6 +64,16 @@ jobs: echo "\`\`\`" >> ./pr/COMMENT head -n10 "perf_scap_diff.txt" >> ./pr/COMMENT echo "\`\`\`" >> ./pr/COMMENT + echo "" >> ./pr/COMMENT + echo "# Heap diff from master - unit tests" >> ./pr/COMMENT + echo "\`\`\`" >> ./pr/COMMENT + tail -n6 "heaptrack_tests_diff.txt" >> ./pr/COMMENT + echo "\`\`\`" >> ./pr/COMMENT + echo "" >> ./pr/COMMENT + echo "# Heap diff from master - scap file" >> ./pr/COMMENT + echo "\`\`\`" >> ./pr/COMMENT + tail -n6 "heaptrack_scap_diff.txt" >> ./pr/COMMENT + echo "\`\`\`" >> ./pr/COMMENT echo Uploading PR info... cat ./pr/COMMENT echo "" @@ -68,22 +86,50 @@ jobs: retention-days: 1 if-no-files-found: warn - # Check will fail if there is any function slowed down >=3% (relative). + # Check will fail if sum of all differences is >= 1%. # But we will always comment with the perf diff from master - - name: Check >= 3% threshold - unit tests + - name: Check >= 1% threshold - perf unit tests + if: always() run: | - awk '{if (substr($2,RSTART+RLENGTH)+0 >= 3) print }' perf_tests_diff.txt &> perf_diff_above_thresh.txt - if [ -s perf_diff_above_thresh.txt ]; then + sum=$(awk '{sum+=sprintf("%f",$2)}END{printf "%.6f\n",sum}' perf_tests_diff.txt | tr ',' '.') + if (( $(echo "$sum >= 1.0" | bc -l) )); then exit 1 fi - # Check will fail if there is any function slowed down >=10% (relative). - # Larger threshold since scap file perf seems much more unstable. + # Check will fail if sum of all differences is >= 1%. # But we will always comment with the perf diff from master - - name: Check >= 10% threshold - scap file - if: always() # Even if unit tests threshold check failed + - name: Check >= 1% threshold - perf scap file + if: always() # Even if other threshold checks failed run: | - awk '{if (substr($2,RSTART+RLENGTH)+0 >= 10) print }' perf_scap_diff.txt &> perf_diff_above_thresh.txt - if [ -s perf_diff_above_thresh.txt ]; then + sum=$(awk '{sum+=sprintf("%f",$2)}END{printf "%.6f\n",sum}' perf_scap_diff.txt | tr ',' '.') + if (( $(echo "$sum >= 1.0" | bc -l) )); then exit 1 + fi + + # Check will fail if there is any heap memory usage difference >= 1M, + # or if there is new memory leaked. + - name: Check >= 1M threshold or new leaked mem - heaptrack unit tests + if: always() # Even if other threshold checks failed + run: | + tail -n 6 heaptrack_tests_diff.txt | grep "peak heap memory consumption" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 >= 1048576) print }' &> heaptrack_tests_diff_above_thresh.txt + if [ -s heaptrack_tests_diff_above_thresh.txt ]; then + exit 1 + fi + tail -n 6 heaptrack_scap_diff.txt | grep "total memory leaked" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 > 0) print }' &> heaptrack_scap_diff_above_thresh.txt + if [ -s heaptrack_scap_diff_above_thresh.txt ]; then + exit 2 + fi + + # Check will fail if there is any heap memory usage difference >= 1M, + # or if there is new memory leaked. + - name: Check >= 1M threshold or new leaked mem - heaptrack scap file + if: always() # Even if other threshold checks failed + run: | + tail -n 6 heaptrack_scap_diff.txt | grep "peak heap memory consumption" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 >= 1048576) print }' &> heaptrack_scap_diff_above_thresh.txt + if [ -s heaptrack_scap_diff_above_thresh.txt ]; then + exit 1 + fi + tail -n 6 heaptrack_scap_diff.txt | grep "total memory leaked" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 > 0) print }' &> heaptrack_scap_diff_above_thresh.txt + if [ -s heaptrack_scap_diff_above_thresh.txt ]; then + exit 2 fi \ No newline at end of file diff --git a/docs/perf.md b/docs/perf.md index 46c20812a5..06850bff66 100644 --- a/docs/perf.md +++ b/docs/perf.md @@ -1,9 +1,11 @@ # Home of Falco Perf Monitoring -Our CI is capable of continuously benchmarking performance of our userspace code. +Our CI is capable of continuously benchmarking performance of our userspace code, both CPU and memory. Every PR will have a comment with the perf diff from master for multiple aspects, while on master the flamegraph are pushed to this github pages. Navigate to the perf reports on the left, or click these links: -* [unit tests perf](unit_tests.md) -* [scap file reading perf](scap_file.md) \ No newline at end of file +* [unit tests cpu perf](perf_unit_tests.md) +* [scap file reading cpu perf](perf_scap_file.md) +* [unit tests memory profile](heaptrack_unit_tests.md) +* [scap file reading memory profile](heaptrack_scap_file.md) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index c83bfe1a96..cb4913964f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,7 +11,9 @@ nav: - Report: report.md - Perf: - perf.md - - Unit tests: unit_tests.md - - Scap file: scap_file.md + - Perf Unit Tests: perf_unit_tests.md + - Perf Scap File: perf_scap_file.md + - Memory Unit Tests: heaptrack_unit_tests.md + - Memory Scap File: heaptrack_scap_file.md theme: material diff --git a/userspace/libsinsp/test/sinsp_utils.ut.cpp b/userspace/libsinsp/test/sinsp_utils.ut.cpp index 83bd6b06c1..021ebfb799 100644 --- a/userspace/libsinsp/test/sinsp_utils.ut.cpp +++ b/userspace/libsinsp/test/sinsp_utils.ut.cpp @@ -18,6 +18,7 @@ limitations under the License. #include #include +#include TEST(sinsp_utils_test, concatenate_paths) { @@ -214,9 +215,12 @@ TEST(sinsp_utils_test, sinsp_split_check_terminator) // check that the null terminator is enforced const char *in = "hello\0worlddd"; size_t len = 13; +#ifdef _DEBUG + EXPECT_THROW(sinsp_split(in, len, '\0'), sinsp_exception); +#else auto split = sinsp_split(in, len, '\0'); - EXPECT_EQ(split.size(), 2); EXPECT_EQ(split[0], "hello"); EXPECT_EQ(split[1], "worldd"); +#endif }