Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(CI): correctly extract dependencies for alternatives plugins #432

Merged
merged 1 commit into from
Mar 13, 2024
Merged

fix(CI): correctly extract dependencies for alternatives plugins #432

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions .github/extract-plugins-deps-from-rulesfile.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/bin/bash
set -o errexit
set -o nounset
set -o pipefail
# Plugins for which we need to check if there exist as alternative plugin.
# If so, then we set them as a dependency. This is a must for rulesfiles
# that have multiple plugins that satisfy their requirements and the plugin we are
# checking is an alternative.
# It accepts a single value or coma separated values.
PLUGINS=$1

filtered_entries=()

# Extract plugins requirement from all files and save in a local file.
# Combine the sections from multiple files and save the output to file.
yq eval-all --no-doc '.[].required_plugin_versions | select(. != null and . != "")' ${@:2} > combined_requirements.yaml
# Remove duplicates from the top level.
yq eval-all --inplace 'unique_by(.name)' combined_requirements.yaml

#echo $(cat combined_requirements.yaml)

for YAML_FILE in "combined_requirements.yaml"; do
#echo "Processing file $YAML_FILE"
# Get the length of the entries list
length=$(yq eval '. | length' "$YAML_FILE")
# Iterate over each index in the entries list
for ((i = 0; i < length; i++)); do
# Access the entry by index using yq
entry=$(yq eval '.['"$i"']' "$YAML_FILE")

# Extract name and version from the entry
name=$(echo "$entry" | yq eval '.name' -)
version=$(echo "$entry" | yq eval '.version' -)
# If a plugin we are considering exists as an alternative of another one, then we just skip.
# This case could happen when we are processing multiple files and one of them overrides the
# plugin since it has some specific rules for that plugin.
to_be_skipped=false
for alternative in $(yq eval '.[].alternatives[].name' combined_requirements.yaml);do
if [[ "$alternative" == "$name" ]]; then
to_be_skipped=true

break
fi
done

if [ "$to_be_skipped" = true ];then
#echo "skipping plugin ${name} because already an alternative"
continue
fi

# Check if alternatives exist
alternatives=$(echo "$entry" | yq eval '.alternatives[]?')
if [ -n "$alternatives" ]; then
is_alternative=false
# Get the length of the alternatives list
alt_length=$(echo "$entry" | yq eval '.alternatives | length' -)
# Iterate over each alternative
for ((j = 0; j < alt_length; j++)); do
alt_entry=$(echo "$entry" | yq eval '.alternatives['"$j"']?' -)
alt_name=$(echo "$alt_entry" | yq eval '.name' -)
alt_version=$(echo "$alt_entry" | yq eval '.version' -)
# If our plugin is set as an alternative then we use it as a dependency.
if [[ " ${PLUGINS//,/ } " =~ " $alt_name " ]]; then
#echo "Preferring alternative plugin ${alt_name} over ${name}"
is_alternative=true
name=$alt_name
version=$alt_version
break
fi
done
fi
filtered_entries+=("$name:$version")
done
done

# Output the filtered entries
printf "%s\n" "${filtered_entries[@]}"
17 changes: 5 additions & 12 deletions .github/setup-plugin-config-rules.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,24 +18,17 @@ if [ ! -f "$config_file" ]; then
# we collect all plugin dependencies across all plugin rulesets
# todo(jasondellaluce): find a way to avoid ignoring alternatives
if [ -d "$rules_dir" ]; then
echo Extracting plugin dependencies from rules files...
rules_files=$(ls $rules_dir/*)
for rules_file in "$rules_files"; do
echo Extracting plugin dependencies from rules file "${rules_file}"...
rules_deps=$(cat $rules_file | yq -r '.[].required_plugin_versions | select(. != null and . != "")[] | [.name + ":" + .version] | @csv')
for dep in $rules_deps; do
plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
if [[ ${deps} != *"$plugin_name"* ]]; then
deps="${deps} "${plugin_name}
fi
done
done
echo Extracting plugin dependencies from rules file "${rules_files}"...
rules_deps=$($GITHUB_WORKSPACE/.github/extract-plugins-deps-from-rulesfile.sh $PLUGIN $rules_files)
echo "${rules_deps}"
fi

mkdir -p $(echo $config_file | sed 's:[^/]*$::')
touch $config_file
echo "plugins:" >> $config_file
for dep in $deps; do
for dep in $rules_deps; do
dep=$(echo $dep | tr -d '"' | cut -d ':' -f 1)
echo " - name: ${dep}" >> $config_file
echo " library_path: lib${dep}.so" >> $config_file
done
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/reusable_suggest_rules_version.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
uses: actions/checkout@v3

- name: Install system dependencies
run: pip install yq
run: sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq

- name: Setup plugin config and rules
id: get-config
Expand Down
98 changes: 48 additions & 50 deletions .github/workflows/reusable_validate_plugins.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,7 @@ jobs:
uses: actions/checkout@v3

- name: Install system dependencies
run: |
apt update -y
apt install -y --no-install-recommends pip git jq
pip install yq
run: wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq

- name: Setup plugin config and rules
id: get-config
Expand Down Expand Up @@ -125,7 +122,7 @@ jobs:
uses: actions/checkout@v3

- name: Install system dependencies
run: pip install yq
run: sudo wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq

- name: Setup plugin config and rules
id: get-config
Expand Down Expand Up @@ -181,48 +178,49 @@ jobs:
sudo mkdir -p /usr/share/falco/plugins

rules_files=$(ls ${{ steps.get-config.outputs.rules_dir }}/*)
for rules_file in $rules_files; do
deps=$(cat $rules_file | yq -r '.[].required_plugin_versions | select(. != null and . != "")[] | [.name + ":" + .version] | @csv')

ver_diff=0
has_updates=1
while [ "$has_updates" -eq 1 ]; do
has_updates=0
for dep in $deps; do
plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)

# forcing zero patch version to forbid patch-like dependencies
# bumping minor version at every iteration
plugin_ver=$(echo $dep | tr -d '"' | cut -d ':' -f 2)
plugin_ver_major=$(echo $plugin_ver | cut -d '.' -f 1)
plugin_ver_minor=$(expr $(echo $plugin_ver | cut -d '.' -f 2) + $ver_diff)
plugin_ver_patch=0
plugin_ver="${plugin_ver_major}.${plugin_ver_minor}.${plugin_ver_patch}"

set +e pipefail
sudo falcoctl artifact install ${plugin_name}:${plugin_ver}
if [ $? -eq 0 ]; then
echo Installed plugin "${plugin_name}" at version "${plugin_ver}"
has_updates=1
else
echo Can\'t pull plugin "${plugin_name}" at version "${plugin_ver}"
echo Attempt installing locally-built plugin "${plugin_name}"...
for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
echo Extracting archive "$archive"...
mkdir -p tmpdir && pushd tmpdir
tar -xvf $archive
sudo cp -r *.so /usr/share/falco/plugins || true
popd && rm -fr tmpdir
done
fi
set -e pipefail
done
ver_diff=$(expr $ver_diff + 1)

./.github/validate-rules.sh \
"${{ inputs.falco-image }}" \
"${{ inputs.rules-checker }}" \
"${{ steps.get-config.outputs.config_file }}" \
"$rules_file"
done
done
deps=$(./.github/extract-plugins-deps-from-rulesfile.sh \
"${{ inputs.plugin }}" \
"$rules_files")
echo "Deps: ${deps}"
ver_diff=0
has_updates=1
while [ "$has_updates" -eq 1 ]; do
has_updates=0
for dep in $deps; do
echo "Plugin: ${dep}"
plugin_name=$(echo $dep | tr -d '"' | cut -d ':' -f 1)

# forcing zero patch version to forbid patch-like dependencies
# bumping minor version at every iteration
plugin_ver=$(echo $dep | tr -d '"' | cut -d ':' -f 2)
plugin_ver_major=$(echo $plugin_ver | cut -d '.' -f 1)
plugin_ver_minor=$(expr $(echo $plugin_ver | cut -d '.' -f 2) + $ver_diff)
plugin_ver_patch=0
plugin_ver="${plugin_ver_major}.${plugin_ver_minor}.${plugin_ver_patch}"

set +e pipefail
sudo falcoctl artifact install ${plugin_name}:${plugin_ver}
if [ $? -eq 0 ]; then
echo Installed plugin "${plugin_name}" at version "${plugin_ver}"
has_updates=1
else
echo Can\'t pull plugin "${plugin_name}" at version "${plugin_ver}"
echo Attempt installing locally-built plugin "${plugin_name}"...
for archive in $(ls /tmp/plugins-${{ inputs.arch }}/${plugin_name}-*); do
echo Extracting archive "$archive"...
mkdir -p tmpdir && pushd tmpdir
tar -xvf $archive
sudo cp -r *.so /usr/share/falco/plugins || true
popd && rm -fr tmpdir
done
fi
set -e pipefail
done
ver_diff=$(expr $ver_diff + 1)

./.github/validate-rules.sh \
"${{ inputs.falco-image }}" \
"${{ inputs.rules-checker }}" \
"${{ steps.get-config.outputs.config_file }}" \
"$rules_files"
done
Loading