new(ci,docs): added heaptrack to our new perf related CI.

.github/actions/composite-perf/action.yml

@@ -4,46 +4,71 @@ description: 'Run multiple perf tests on libs.'
 outputs:
   perf_tests:
     description: "Unit tests perf.data"
-    value: ${{ steps.store-outputs.outputs.tests }}
+    value: ${{ steps.store-outputs.outputs.perf_tests }}
   perf_scap:
     description: "Scap file perf.data"
-    value: ${{ steps.store-outputs.outputs.scap }}
+    value: ${{ steps.store-outputs.outputs.perf_scap }}
+  massif_tests:
+    description: "Unit tests heaptrack data"
+    value: ${{ steps.store-outputs.outputs.heaptrack_tests }}
+  massif_scap:
+    description: "Scap file heaptrack data"
+    value: ${{ steps.store-outputs.outputs.heaptrack_scap }}
 
 runs:
   using: "composite"
   steps:
     - name: Install deps ⛓️
       shell: bash
       run: |
-        sudo apt update && sudo apt install -y --no-install-recommends ca-certificates cmake build-essential git clang llvm pkg-config autoconf automake libtool libelf-dev wget libc-ares-dev libcurl4-openssl-dev libssl-dev libtbb-dev libjq-dev libjsoncpp-dev libgrpc++-dev protobuf-compiler-grpc libgtest-dev libprotobuf-dev linux-tools-common linux-tools-generic linux-tools-`uname -r`
+        sudo apt update && sudo apt install -y --no-install-recommends ca-certificates cmake build-essential git clang llvm pkg-config autoconf automake libtool libelf-dev wget libc-ares-dev libcurl4-openssl-dev libssl-dev libtbb-dev libjq-dev libjsoncpp-dev libgrpc++-dev protobuf-compiler-grpc libgtest-dev libprotobuf-dev linux-tools-common linux-tools-generic linux-tools-`uname -r` heaptrack coreutils util-linux
         sudo .github/install-deps.sh
 
     - name: Build
       shell: bash
       run: |
         mkdir -p build
-        cd build && cmake -DUSE_BUNDLED_DEPS=False ../
+        cd build && cmake -DUSE_BUNDLED_DEPS=False -DCMAKE_BUILD_TYPE=Debug ../
         make unit-test-libsinsp -j4
         make sinsp-example -j4
 
-    - name: Run Perf - unit tests
+    - name: Download scap file
+      shell: bash
+      run: |
+        cd build
+        wget https://download.falco.org/fixtures/trace-files/traces-positive.zip
+        unzip traces-positive.zip
+
+    - name: Run - perf unit tests
       shell: bash
       run: |
         cd build
         sudo perf record --call-graph dwarf -o perf_tests.data -q libsinsp/test/unit-test-libsinsp
 
-    - name: Run Perf - scap file
+    - name: Run - perf scap file
       shell: bash
       run: |
         cd build
-        wget https://download.falco.org/fixtures/trace-files/traces-positive.zip
-        unzip traces-positive.zip
-        sudo perf record --call-graph dwarf -o perf_scap.data -q ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap   
+        sudo nice ionice -c 1 -n 0 perf record --call-graph dwarf -o perf_scap.data -q ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap
+
+    - name: Run - heaptrack unit tests
+      shell: bash
+      run: |
+        cd build
+        sudo heaptrack -o heaptrack_tests.data libsinsp/test/unit-test-libsinsp
+
+    - name: Run - heaptrack scap file
+      shell: bash
+      run: |
+        cd build
+        sudo nice ionice -c 1 -n 0 heaptrack -o heaptrack_scap.data ./libsinsp/examples/sinsp-example -s traces-positive/falco-event-generator.scap
 
     - name: Set Outputs
       id: store-outputs
       shell: bash
       run: |
         cd build
-        echo "tests=$(realpath perf_tests.data)" >> $GITHUB_OUTPUT
-        echo "scap=$(realpath perf_scap.data)" >> $GITHUB_OUTPUT
+        echo "perf_tests=$(realpath perf_tests.data)" >> $GITHUB_OUTPUT
+        echo "perf_scap=$(realpath perf_scap.data)" >> $GITHUB_OUTPUT
+        echo "heaptrack_tests=$(realpath heaptrack_tests.data)" >> $GITHUB_OUTPUT
+        echo "heaptrack_scap=$(realpath heaptrack_scap.data)" >> $GITHUB_OUTPUT

.github/workflows/pages.yml

@@ -35,6 +35,8 @@ jobs:
           path: |
             ${{ steps.perf.outputs.perf_tests }}
             ${{ steps.perf.outputs.perf_scap }}
+            ${{ steps.perf.outputs.heaptrack_tests }}
+            ${{ steps.perf.outputs.heaptrack_scap }}
           if-no-files-found: error
 
       - name: Checkout Flamegraph ⤵️
@@ -44,23 +46,35 @@ jobs:
           path: flamegraph
           ref: master
 
-      - name: Generate svg file - unit tests
+      - name: Generate svg file - perf unit tests
         run: |
-          sudo perf script --dsos unit-test-libsinsp -i ${{ steps.perf.outputs.perf_tests }} > trace_tests.perf
-          ./flamegraph/stackcollapse-perf.pl trace_tests.perf > trace_tests.folded
-          ./flamegraph/flamegraph.pl trace_tests.folded > trace_tests.svg
+          sudo perf script --dsos unit-test-libsinsp -i ${{ steps.perf.outputs.perf_tests }} > perf_tests.perf
+          ./flamegraph/stackcollapse-perf.pl perf_tests.perf > perf_tests.folded
+          ./flamegraph/flamegraph.pl perf_tests.folded > perf_tests.svg
 
-      - name: Generate svg file - scap file
+      - name: Generate svg file - perf scap file
         run: |
-          sudo perf script --dsos sinsp-example -i ${{ steps.perf.outputs.perf_scap }} > trace_scap.perf
-          ./flamegraph/stackcollapse-perf.pl trace_scap.perf > trace_scap.folded
-          ./flamegraph/flamegraph.pl trace_scap.folded > trace_scap.svg 
+          sudo perf script --dsos sinsp-example -i ${{ steps.perf.outputs.perf_scap }} > perf_scap.perf
+          ./flamegraph/stackcollapse-perf.pl perf_scap.perf > perf_scap.folded
+          ./flamegraph/flamegraph.pl perf_scap.folded > perf_scap.svg
+
+      - name: Generate svg file - heaptrack unit tests
+        run: |
+          heaptrack_print ${{ steps.perf.outputs.heaptrack_tests }} -F stacks.txt
+          ./flamegraph/flamegraph.pl stacks.txt > heaptrack_tests.svg
+          rm -rf stacks.txt
+
+      - name: Generate svg file - heaptrack scap file
+        run: |
+          heaptrack_print ${{ steps.perf.outputs.heaptrack_scap }} -F stacks.txt
+          ./flamegraph/flamegraph.pl stacks.txt > heaptrack_scap.svg
+          rm -rf stacks.txt
 
       - name: Upload svg files
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
           name: perf_svg
-          path: trace_*.svg
+          path: '*.svg'
           if-no-files-found: error
 
   deploy-pages:
@@ -100,12 +114,18 @@ jobs:
 
       - name: Generate perf pages
         run: |
-          mv trace_tests.svg docs/trace_tests.svg
-          echo '<object data="../trace_tests.svg" type="image/svg+xml" id="unit_tests"></object>' > docs/unit_tests.md
-          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/unit_tests.md
-          mv trace_scap.svg docs/trace_scap.svg
-          echo '<object data="../trace_scap.svg" type="image/svg+xml" id="scap_file"></object>' > docs/scap_file.md
-          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/scap_file.md
+          mv perf_tests.svg docs/
+          echo '<object data="../perf_tests.svg" type="image/svg+xml" id="perf_unit_tests"></object>' > docs/perf_unit_tests.md
+          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/perf_unit_tests.md
+          mv perf_scap.svg docs/
+          echo '<object data="../perf_scap.svg" type="image/svg+xml" id="perf_scap_file"></object>' > docs/perf_scap_file.md
+          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/perf_scap_file.md
+          mv heaptrack_tests.svg docs/
+          echo '<object data="../heaptrack_tests.svg" type="image/svg+xml" id="heaptrack_unit_tests"></object>' > docs/heaptrack_unit_tests.md
+          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/heaptrack_unit_tests.md
+          mv heaptrack_scap.svg docs/
+          echo '<object data="../heaptrack_scap.svg" type="image/svg+xml" id="heaptrack_scap_file"></object>' > docs/heaptrack_scap_file.md
+          sed -i '1s/^/---\nhide:\n- toc\n---\n\n/' docs/heaptrack_scap_file.md
 
       - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
         with:

.github/workflows/perf.yml

@@ -27,19 +27,27 @@ jobs:
           event: push
           name: perf_report
 
-      - name: Diff from master - unit tests
+      - name: Diff from master - perf unit tests
         run: |
           sudo perf diff perf_tests.data ${{ steps.perf.outputs.perf_tests }} -d unit-test-libsinsp -b -o 1 --percentage relative -q &> perf_tests_diff.txt
 
-      - name: Diff from master - scap file
+      - name: Diff from master - perf scap file
         run: |
           sudo perf diff perf_scap.data ${{ steps.perf.outputs.perf_scap }} -d sinsp-example -b -o 1 --percentage relative -q &> perf_scap_diff.txt    
 
+      - name: Diff from master - heaptrack unit tests
+        run: |
+          sudo heaptrack_print heaptrack_tests.data -d ${{ steps.perf.outputs.heaptrack_tests }} &> heaptrack_tests_diff.txt
+
+      - name: Diff from master - heaptrack scap file
+        run: |
+          sudo heaptrack_print heaptrack_scap.data -d ${{ steps.perf.outputs.heaptrack_scap }} &> heaptrack_scap_diff.txt
+
       - name: Archive perf diff
         uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
         with:
           name: perf_diff
-          path: perf_*_diff.txt
+          path: '*_diff.txt'
           if-no-files-found: error
 
       - name: Save PR info
@@ -56,6 +64,16 @@ jobs:
           echo "\`\`\`" >> ./pr/COMMENT
           head -n10 "perf_scap_diff.txt" >> ./pr/COMMENT
           echo "\`\`\`" >> ./pr/COMMENT
+          echo "" >> ./pr/COMMENT
+          echo "# Heap diff from master - unit tests" >> ./pr/COMMENT
+          echo "\`\`\`" >> ./pr/COMMENT
+          tail -n6 "heaptrack_tests_diff.txt" >> ./pr/COMMENT
+          echo "\`\`\`" >> ./pr/COMMENT
+          echo "" >> ./pr/COMMENT
+          echo "# Heap diff from master - scap file" >> ./pr/COMMENT
+          echo "\`\`\`" >> ./pr/COMMENT
+          tail -n6 "heaptrack_scap_diff.txt" >> ./pr/COMMENT
+          echo "\`\`\`" >> ./pr/COMMENT
           echo Uploading PR info...
           cat ./pr/COMMENT
           echo ""
@@ -68,22 +86,50 @@ jobs:
           retention-days: 1
           if-no-files-found: warn
 
-      # Check will fail if there is any function slowed down >=3% (relative).
+      # Check will fail if sum of all differences is >= 1%.
       # But we will always comment with the perf diff from master
-      - name: Check >= 3% threshold - unit tests
+      - name: Check >= 1% threshold - perf unit tests
+        if: always()
         run: |
-          awk '{if (substr($2,RSTART+RLENGTH)+0 >= 3) print }' perf_tests_diff.txt &> perf_diff_above_thresh.txt
-          if [ -s perf_diff_above_thresh.txt ]; then
+          sum=$(awk '{sum+=sprintf("%f",$2)}END{printf "%.6f\n",sum}' perf_tests_diff.txt | tr ',' '.')
+          if (( $(echo "$sum >= 1.0" | bc -l) )); then
             exit 1
           fi
 
-      # Check will fail if there is any function slowed down >=10% (relative).
-      # Larger threshold since scap file perf seems much more unstable.
+      # Check will fail if sum of all differences is >= 1%.
       # But we will always comment with the perf diff from master
-      - name: Check >= 10% threshold - scap file
-        if: always() # Even if unit tests threshold check failed
+      - name: Check >= 1% threshold - perf scap file
+        if: always() # Even if other threshold checks failed
         run: |
-          awk '{if (substr($2,RSTART+RLENGTH)+0 >= 10) print }' perf_scap_diff.txt &> perf_diff_above_thresh.txt
-          if [ -s perf_diff_above_thresh.txt ]; then
+          sum=$(awk '{sum+=sprintf("%f",$2)}END{printf "%.6f\n",sum}' perf_scap_diff.txt | tr ',' '.')
+          if (( $(echo "$sum >= 1.0" | bc -l) )); then
             exit 1
+          fi
+
+      # Check will fail if there is any heap memory usage difference >= 1M,
+      # or if there is new memory leaked.
+      - name: Check >= 1M threshold or new leaked mem - heaptrack unit tests
+        if: always() # Even if other threshold checks failed
+        run: |
+          tail -n 6 heaptrack_tests_diff.txt | grep "peak heap memory consumption" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 >= 1048576) print }' &> heaptrack_tests_diff_above_thresh.txt
+          if [ -s heaptrack_tests_diff_above_thresh.txt ]; then
+            exit 1
+          fi
+          tail -n 6 heaptrack_scap_diff.txt | grep "total memory leaked" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 > 0) print }' &> heaptrack_scap_diff_above_thresh.txt
+          if [ -s heaptrack_scap_diff_above_thresh.txt ]; then
+            exit 2
+          fi
+
+      # Check will fail if there is any heap memory usage difference >= 1M,
+      # or if there is new memory leaked.
+      - name: Check >= 1M threshold or new leaked mem - heaptrack scap file
+        if: always() # Even if other threshold checks failed
+        run: |
+          tail -n 6 heaptrack_scap_diff.txt | grep "peak heap memory consumption" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 >= 1048576) print }' &> heaptrack_scap_diff_above_thresh.txt
+          if [ -s heaptrack_scap_diff_above_thresh.txt ]; then
+            exit 1
+          fi
+          tail -n 6 heaptrack_scap_diff.txt | grep "total memory leaked" | awk -F': ' '{print $2 }' | tr '.' ',' | numfmt --from=iec | awk '{if (substr($1,RSTART+RLENGTH)+0 > 0) print }' &> heaptrack_scap_diff_above_thresh.txt
+          if [ -s heaptrack_scap_diff_above_thresh.txt ]; then
+            exit 2
           fi

docs/perf.md

@@ -1,9 +1,11 @@
 # Home of Falco Perf Monitoring
 
-Our CI is capable of continuously benchmarking performance of our userspace code.  
+Our CI is capable of continuously benchmarking performance of our userspace code, both CPU and memory.   
 Every PR will have a comment with the perf diff from master for multiple aspects, while on master the flamegraph are pushed to this github pages.
 
 Navigate to the perf reports on the left, or click these links:
 
-* [unit tests perf](unit_tests.md)
-* [scap file reading perf](scap_file.md)
+* [unit tests cpu perf](perf_unit_tests.md)
+* [scap file reading cpu perf](perf_scap_file.md)
+* [unit tests memory profile](heaptrack_unit_tests.md)
+* [scap file reading memory profile](heaptrack_scap_file.md)

mkdocs.yml

@@ -11,7 +11,9 @@ nav:
       - Report: report.md
     - Perf:
         - perf.md
-        - Unit tests: unit_tests.md
-        - Scap file: scap_file.md
+        - Perf Unit Tests: perf_unit_tests.md
+        - Perf Scap File: perf_scap_file.md
+        - Memory Unit Tests: heaptrack_unit_tests.md
+        - Memory Scap File: heaptrack_scap_file.md
 
 theme: material

userspace/libsinsp/test/sinsp_utils.ut.cpp

@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <gtest/gtest.h>
 #include <libsinsp/utils.h>
+#include <libsinsp/sinsp_exception.h>
 
 TEST(sinsp_utils_test, concatenate_paths)
 {
@@ -214,9 +215,12 @@ TEST(sinsp_utils_test, sinsp_split_check_terminator)
 	// check that the null terminator is enforced
 	const char *in = "hello\0worlddd";
 	size_t len = 13;
+#ifdef _DEBUG
+	EXPECT_THROW(sinsp_split(in, len, '\0'), sinsp_exception);
+#else
 	auto split = sinsp_split(in, len, '\0');
-
 	EXPECT_EQ(split.size(), 2);
 	EXPECT_EQ(split[0], "hello");
 	EXPECT_EQ(split[1], "worldd");
+#endif
 }