Skip to content

Commit

Permalink
[DRAFT] CI experiments
Browse files Browse the repository at this point in the history
  • Loading branch information
alliepiper committed Apr 21, 2024
1 parent 29a24f4 commit 366bb99
Show file tree
Hide file tree
Showing 8 changed files with 889 additions and 298 deletions.
64 changes: 51 additions & 13 deletions .github/actions/compute-matrix/compute-matrix.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@

set -euo pipefail

# Ensure the script is being executed in its containing directory
cd "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

write_output() {
local key="$1"
local value="$2"
Expand All @@ -13,7 +16,7 @@ explode_std_versions() {
}

explode_libs() {
jq -cr 'map(. as $o | {lib: $o.lib[]} + del($o.lib))'
jq -cr 'map(. as $o | {project: $o.project[]} + del($o.project))'
}

# Filter out the libraries that are dirty
Expand All @@ -34,7 +37,10 @@ filter_libs() {
# echo "Dirty libraries: ${dirty_libs[@]}" >> /dev/stderr

# Construct a regex to filter out the dirty libraries
dirty_lib_regex=$(IFS="|"; echo "${dirty_libs[*]}")
dirty_lib_regex=$(
IFS="|"
echo "${dirty_libs[*]}"
)
dirty_lib_regex="^(${dirty_lib_regex})\$"
jq_filter="map(select(.lib | test(\"$dirty_lib_regex\")))"
jq -cr "$jq_filter"
Expand All @@ -44,31 +50,63 @@ extract_matrix() {
local file="$1"
local type="$2"
local matrix=$(yq -o=json "$file" | jq -cr ".$type")
write_output "DEVCONTAINER_VERSION" "$(yq -o json "$file" | jq -cr '.devcontainer_version')"
local devcontainer_version=$(yq -o json "$file" | jq -cr '.devcontainer_version')

write_output "DEVCONTAINER_VERSION" "$devcontainer_version"

local nvcc_full_matrix="$(echo "$matrix" | jq -cr '.nvcc' | explode_std_versions )"
local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
write_output "PER_CUDA_COMPILER_MATRIX" "$per_cuda_compiler_matrix"
write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"
local full_matrix="$(echo "$matrix" | explode_std_versions | explode_libs)" # | filter_libs)"

write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"
# For each json object in the nvcc matrix array, accumulate the output json object from `generate-ci-dispatch-job.sh`
# into a single json object. If duplicate keys are found, merge the matching value arrays.
local wf_json="{}"
local input_objects=$(echo "$full_matrix" | jq -cr '.[]')
for input_object in $input_objects; do
local output_object=$(./generate-ci-dispatch-job.sh "$devcontainer_version" "${input_object}")
wf_json=$(
jq --argjson accum "$wf_json" --argjson new "$output_object" '
$new | . as $input |
reduce keys[] as $key (
$accum;
if .[$key] then
.[$key] |= (
reduce ($input[$key] | keys_unsorted[]) as $nestedKey (.;
.[$nestedKey] += $input[$key][$nestedKey]
)
)
else
.[$key] = $input[$key]
end
)
' <<<'{}')
done
wf_keys=$(echo "$wf_json" | jq -r 'keys | @json')

local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
write_output "WORKFLOW" "$wf_json"
write_output "WORKFLOW_KEYS" "$wf_keys"
}

# local per_cuda_compiler_matrix="$(echo "$nvcc_full_matrix" | jq -cr ' group_by(.cuda + .compiler.name) | map({(.[0].cuda + "-" + .[0].compiler.name): .}) | add')"
# write_output "PER_CUDA_COMPILER_MATRIX" "$per_cuda_compiler_matrix"
# write_output "PER_CUDA_COMPILER_KEYS" "$(echo "$per_cuda_compiler_matrix" | jq -r 'keys | @json')"

# write_output "NVRTC_MATRIX" "$(echo "$matrix" | jq '.nvrtc' | explode_std_versions)"

# local clang_cuda_matrix="$(echo "$matrix" | jq -cr '.["clang-cuda"]' | explode_std_versions | explode_libs | filter_libs)"
# write_output "CLANG_CUDA_MATRIX" "$clang_cuda_matrix"
# write_output "CCCL_INFRA_MATRIX" "$(echo "$matrix" | jq -cr '.["cccl-infra"]' )"
# }

main() {
if [ "$1" == "-v" ]; then
set -x
shift
fi

if [ $# -ne 2 ] || [ "$2" != "pull_request" ]; then
if [ $# -ne 2 ]; then
echo "Usage: $0 [-v] MATRIX_FILE MATRIX_TYPE"
echo " -v : Enable verbose output"
echo " MATRIX_FILE : The path to the matrix file."
echo " MATRIX_TYPE : The desired matrix. Supported values: 'pull_request'"
echo " MATRIX_TYPE : The desired matrix."
exit 1
fi

Expand Down
Loading

0 comments on commit 366bb99

Please sign in to comment.