fix(CI): correctly extract dependencies for alternatives plugins

When running the CI for a plugin which is set as an alternative in the rules files, it is correctly set as a dependency. Furthermore, the dependency extraction logic from rulesfiles has been unified for all the use cases and supports multiple rulesfiles at once. Signed-off-by: Aldo Lacuku <[email protected]>
falcosecurity · Mar 13, 2024 · 110e665 · 110e665
1 parent 9920d35
commit 110e665
Show file tree

Hide file tree

Showing 4 changed files with 131 additions and 63 deletions.
diff --git a/.github/extract-plugins-deps-from-rulesfile.sh b/.github/extract-plugins-deps-from-rulesfile.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+set -o errexit
+set -o nounset
+set -o pipefail
+# Plugins for which we need to check if there exist as alternative plugin.
+# If so, then we set them as a dependency. This is a must for rulesfiles
+# that have multiple plugins that satisfy their requirements and the plugin we are
+# checking is an alternative.
+# It accepts a single value or coma separated values.
+PLUGINS=$1
+
+filtered_entries=()
+
+# Extract plugins requirement from all files and save in a local file.
+# Combine the sections from multiple files and save the output to file.
+yq eval-all --no-doc '.[].required_plugin_versions | select(. != null and . != "")' ${@:2} > combined_requirements.yaml
+# Remove duplicates from the top level.
+yq eval-all --inplace 'unique_by(.name)' combined_requirements.yaml
+
+#echo $(cat combined_requirements.yaml)
+
+for YAML_FILE in "combined_requirements.yaml"; do
+  #echo "Processing file $YAML_FILE"
+  # Get the length of the entries list
+  length=$(yq eval '. | length' "$YAML_FILE")
+  # Iterate over each index in the entries list
+  for ((i = 0; i < length; i++)); do
+      # Access the entry by index using yq
+      entry=$(yq eval '.['"$i"']' "$YAML_FILE")
+
+      # Extract name and version from the entry
+      name=$(echo "$entry" | yq eval '.name' -)
+      version=$(echo "$entry" | yq eval '.version' -)
+      # If a plugin we are considering exists as an alternative of another one, then we just skip.
+      # This case could happen when we are processing multiple files and one of them overrides the
+      # plugin since it has some specific rules for that plugin.
+      to_be_skipped=false
+      for alternative in $(yq eval  '.[].alternatives[].name' combined_requirements.yaml);do
+        if [[  "$alternative" == "$name" ]]; then
+          to_be_skipped=true
+
+          break
+        fi
+      done
+
+      if [ "$to_be_skipped" = true ];then
+        #echo "skipping plugin ${name} because already an alternative"
+        continue
+      fi
+
+      # Check if alternatives exist
+      alternatives=$(echo "$entry" | yq eval '.alternatives[]?')
+      if [ -n "$alternatives" ]; then
+          is_alternative=false
+          # Get the length of the alternatives list
+          alt_length=$(echo "$entry" | yq eval '.alternatives | length' -)
+          # Iterate over each alternative
+          for ((j = 0; j < alt_length; j++)); do
+              alt_entry=$(echo "$entry" | yq eval '.alternatives['"$j"']?' -)
+              alt_name=$(echo "$alt_entry" | yq eval '.name' -)
+              alt_version=$(echo "$alt_entry" | yq eval '.version' -)
+              # If our plugin is set as an alternative then we use it as a dependency.
+              if [[ " ${PLUGINS//,/ } " =~ " $alt_name " ]]; then
+                  #echo "Preferring alternative plugin ${alt_name} over ${name}"
+                  is_alternative=true
+                  name=$alt_name
+                  version=$alt_version
+                  break
+              fi
+          done
+      fi
+    filtered_entries+=("$name:$version")
+  done
+done
+
+# Output the filtered entries
+printf "%s\n" "${filtered_entries[@]}"
diff --git a/.github/setup-plugin-config-rules.sh b/.github/setup-plugin-config-rules.sh
@@ -18,24 +18,17 @@ if [ ! -f "$config_file" ]; then
   # we collect all plugin dependencies across all plugin rulesets
   # todo(jasondellaluce): find a way to avoid ignoring alternatives
   if [ -d "$rules_dir" ]; then
-    echo Extracting plugin dependencies from rules files...
     rules_files=$(ls $rules_dir/*)
-    for rules_file in "$rules_files"; do
-      echo Extracting plugin dependencies from rules file "${rules_file}"...
-      rules_deps=$(cat $rules_file | yq -r '.[].required_plugin_versions | select(. != null and . != "")[] | [.name + ":" + .version] | @csv')
-      for dep in $rules_deps; do
-        plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
-        if [[ ${deps} != *"$plugin_name"* ]]; then
-          deps="${deps} "${plugin_name}
-        fi
-      done
-    done
+    echo Extracting plugin dependencies from rules file "${rules_files}"...
+    rules_deps=$($GITHUB_WORKSPACE/.github/extract-plugins-deps-from-rulesfile.sh $PLUGIN $rules_files)
+    echo "${rules_deps}"
   fi
 
   mkdir -p $(echo $config_file | sed 's:[^/]*$::')
   touch $config_file
   echo "plugins:" >> $config_file
-  for dep in $deps; do
+  for dep in $rules_deps; do
+    dep=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
     echo "  - name: ${dep}" >> $config_file
     echo "    library_path: lib${dep}.so" >> $config_file
   done

diff --git a/.github/workflows/reusable_suggest_rules_version.yaml b/.github/workflows/reusable_suggest_rules_version.yaml
@@ -38,7 +38,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install system dependencies
-        run: pip install yq
+        run: sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
 
       - name: Setup plugin config and rules
         id: get-config

diff --git a/.github/workflows/reusable_validate_plugins.yaml b/.github/workflows/reusable_validate_plugins.yaml
@@ -40,10 +40,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install system dependencies
-        run: |
-          apt update -y
-          apt install -y --no-install-recommends pip git jq
-          pip install yq
+        run: wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
 
       - name: Setup plugin config and rules
         id: get-config
@@ -125,7 +122,7 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Install system dependencies
-        run: pip install yq
+        run:  sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq
 
       - name: Setup plugin config and rules
         id: get-config
@@ -181,48 +178,49 @@ jobs:
           sudo mkdir -p /usr/share/falco/plugins
 
           rules_files=$(ls ${{ steps.get-config.outputs.rules_dir }}/*)
-          for rules_file in $rules_files; do
-            deps=$(cat $rules_file | yq -r '.[].required_plugin_versions | select(. != null and . != "")[] | [.name + ":" + .version] | @csv')
-
-            ver_diff=0
-            has_updates=1
-            while [ "$has_updates" -eq 1 ]; do
-              has_updates=0
-              for dep in $deps; do
-                plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
-
-                # forcing zero patch version to forbid patch-like dependencies
-                # bumping minor version at every iteration
-                plugin_ver=$(echo $dep | tr -d '"' | cut -d ':' -f 2)
-                plugin_ver_major=$(echo $plugin_ver | cut -d '.' -f 1)
-                plugin_ver_minor=$(expr $(echo $plugin_ver | cut -d '.' -f 2) + $ver_diff)
-                plugin_ver_patch=0 
-                plugin_ver="${plugin_ver_major}.${plugin_ver_minor}.${plugin_ver_patch}"
-
-                set +e pipefail
-                sudo falcoctl artifact install ${plugin_name}:${plugin_ver}
-                if [ $? -eq 0 ]; then
-                  echo Installed plugin "${plugin_name}" at version "${plugin_ver}"
-                  has_updates=1
-                else
-                  echo Can\'t pull plugin "${plugin_name}" at version "${plugin_ver}"
-                  echo Attempt installing locally-built plugin "${plugin_name}"...
-                  for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
-                    echo Extracting archive "$archive"...
-                    mkdir -p tmpdir && pushd tmpdir
-                    tar -xvf $archive
-                    sudo cp -r *.so /usr/share/falco/plugins || true
-                    popd && rm -fr tmpdir
-                  done
-                fi
-                set -e pipefail
-              done
-              ver_diff=$(expr $ver_diff + 1)
-
-              ./.github/validate-rules.sh \
-                "${{ inputs.falco-image }}" \
-                "${{ inputs.rules-checker }}" \
-                "${{ steps.get-config.outputs.config_file }}" \
-                "$rules_file"
-            done 
-          done
+          deps=$(./.github/extract-plugins-deps-from-rulesfile.sh \
+              "${{ inputs.plugin }}" \
+              "$rules_files")
+          echo "Deps: ${deps}"
+          ver_diff=0
+          has_updates=1
+          while [ "$has_updates" -eq 1 ]; do
+            has_updates=0
+            for dep in $deps; do
+              echo "Plugin: ${dep}"
+              plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
+
+              # forcing zero patch version to forbid patch-like dependencies
+              # bumping minor version at every iteration
+              plugin_ver=$(echo $dep | tr -d '"' | cut -d ':' -f 2)
+              plugin_ver_major=$(echo $plugin_ver | cut -d '.' -f 1)
+              plugin_ver_minor=$(expr $(echo $plugin_ver | cut -d '.' -f 2) + $ver_diff)
+              plugin_ver_patch=0 
+              plugin_ver="${plugin_ver_major}.${plugin_ver_minor}.${plugin_ver_patch}"
+
+              set +e pipefail
+              sudo falcoctl artifact install ${plugin_name}:${plugin_ver}
+              if [ $? -eq 0 ]; then
+                echo Installed plugin "${plugin_name}" at version "${plugin_ver}"
+                has_updates=1
+              else
+                echo Can\'t pull plugin "${plugin_name}" at version "${plugin_ver}"
+                echo Attempt installing locally-built plugin "${plugin_name}"...
+                for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
+                  echo Extracting archive "$archive"...
+                  mkdir -p tmpdir && pushd tmpdir
+                  tar -xvf $archive
+                  sudo cp -r *.so /usr/share/falco/plugins || true
+                  popd && rm -fr tmpdir
+                done
+              fi
+              set -e pipefail
+            done
+            ver_diff=$(expr $ver_diff + 1)
+
+            ./.github/validate-rules.sh \
+              "${{ inputs.falco-image }}" \
+              "${{ inputs.rules-checker }}" \
+              "${{ steps.get-config.outputs.config_file }}" \
+              "$rules_files"
+          done