From 52ea9a1551ae3a2792ff020303aa4583e8937520 Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Mon, 4 Nov 2024 17:04:43 +0200 Subject: [PATCH] Only evaluate the targets we will build and retire hydraJobs Signed-off-by: Joonas Rautiola --- ghaf-main-pipeline.groovy | 45 ++++++++++++++-------------------- ghaf-pre-merge-pipeline.groovy | 37 ++++++++++++---------------- utils.groovy | 23 +++++++++++++++++ 3 files changed, 57 insertions(+), 48 deletions(-) diff --git a/ghaf-main-pipeline.groovy b/ghaf-main-pipeline.groovy index 116cfc0..b26a2a7 100644 --- a/ghaf-main-pipeline.groovy +++ b/ghaf-main-pipeline.groovy @@ -15,36 +15,32 @@ properties([ githubProjectProperty(displayName: '', projectUrlStr: REPO_URL), ]) -// Which attribute of the flake to evaluate for building -def flakeAttr = ".#hydraJobs" - -// Target names must be direct children of the above def targets = [ - [ target: "docs.aarch64-linux", + [ system: "aarch64-linux", target: "doc", archive: false, hwtest_device: null ], - [ target: "docs.x86_64-linux", + [ system: "x86_64-linux", target: "doc", archive: false, hwtest_device: null ], - [ target: "generic-x86_64-debug.x86_64-linux", + [ system: "x86_64-linux", target: "generic-x86_64-debug", archive: true, hwtest_device: "nuc" ], - [ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux", + [ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug", archive: true, hwtest_device: "lenovo-x1" ], - [ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux", + [ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64", archive: true, hwtest_device: "riscv" ], - [ target: "nvidia-jetson-orin-agx-debug.aarch64-linux", + [ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug", archive: true, hwtest_device: "orin-agx" ], - [ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux", + [ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64", archive: true, hwtest_device: "orin-agx" ], - [ target: "nvidia-jetson-orin-nx-debug.aarch64-linux", + [ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug", archive: true, hwtest_device: "orin-nx" ], - [ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux", + [ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64", archive: true, hwtest_device: "orin-nx" ], ] @@ -83,22 +79,19 @@ pipeline { steps { dir(WORKDIR) { script { - // nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json - sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json" - - // jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt - sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt" + // Creates jobs.txt in working directory to use later + utils.nix_eval_jobs(targets) targets.each { - def target = it['target'] + def target = it.system + "." + it.target // row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim() - target_jobs[target] = { + target_jobs["${it.target} (${it.system})"] = { stage("Build ${target}") { def opts = "" - if (it['archive']) { + if (it.archive) { opts = "--out-link archive/${target}" } else { opts = "--no-link" @@ -108,12 +101,12 @@ pipeline { sh "nix build -L ${drvPath}\\^* ${opts}" // only attempt signing if there is something to sign - if (it['archive']) { + if (it.archive) { def img_relpath = utils.find_img_relpath(target, "archive") utils.sign_file("archive/${img_relpath}", "sig/${img_relpath}.sig") } } else { - error("Target \"${target}\" was not found in ${flakeAttr}") + error("Target \"${target}\" was not found in packages") } } catch (InterruptedException e) { throw e @@ -124,7 +117,7 @@ pipeline { } } - if (it['archive']) { + if (it.archive) { stage("Archive ${target}") { script { utils.archive_artifacts("archive", target) @@ -133,9 +126,9 @@ pipeline { } } - if (it['hwtest_device'] != null) { + if (it.hwtest_device != null) { stage("Test ${target}") { - utils.ghaf_parallel_hw_test(target, it['hwtest_device'], '_boot_bat_') + utils.ghaf_parallel_hw_test(target, it.hwtest_device, '_boot_bat_') } } } diff --git a/ghaf-pre-merge-pipeline.groovy b/ghaf-pre-merge-pipeline.groovy index e03825a..0c23975 100644 --- a/ghaf-pre-merge-pipeline.groovy +++ b/ghaf-pre-merge-pipeline.groovy @@ -8,20 +8,16 @@ def REPO_URL = 'https://github.com/tiiuae/ghaf/' def WORKDIR = 'ghaf' -// Which attribute of the flake to evaluate for building -def flakeAttr = ".#hydraJobs" - -// Target names must be direct children of the above def targets = [ - [ target: "docs.aarch64-linux" ], - [ target: "docs.x86_64-linux" ], - [ target: "generic-x86_64-debug.x86_64-linux" ], - [ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux" ], - [ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux" ], - [ target: "nvidia-jetson-orin-agx-debug.aarch64-linux" ], - [ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux" ], - [ target: "nvidia-jetson-orin-nx-debug.aarch64-linux" ], - [ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux" ], + [ system: "aarch64-linux", target: "doc", ], + [ system: "x86_64-linux", target: "doc", ], + [ system: "x86_64-linux", target: "generic-x86_64-debug", ], + [ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug", ], + [ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64", ], + [ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug", ], + [ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64", ], + [ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug", ], + [ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64", ], ] target_jobs = [:] @@ -140,25 +136,22 @@ pipeline { steps { dir(WORKDIR) { script { - // nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json - sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json" - - // jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt - sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt" + // Creates jobs.txt in working directory to use later + utils.nix_eval_jobs(targets) targets.each { - def target = it['target'] + def target = it.system + "." + it.target // row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim() - target_jobs[target] = { - stage("${target}") { + target_jobs["${it.target} (${it.system})"] = { + stage("Build ${target}") { try { if (drvPath) { sh "nix build --no-link -L ${drvPath}\\^*" } else { - error("Target \"${target}\" was not found in ${flakeAttr}") + error("Target \"${target}\" was not found in packages") } } catch (InterruptedException e) { throw e diff --git a/utils.groovy b/utils.groovy index 63dbcf6..0a598bb 100644 --- a/utils.groovy +++ b/utils.groovy @@ -304,6 +304,29 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset= archive_artifacts("ghaf-parallel-hw-test", flakeref_trimmed) } +def nix_eval_jobs(List targets) { + // transform target names into valid nix arrays to be plugged into the expression below + def x86_targets = "\"${targets.findAll { it.system == "x86_64-linux" }.target.join('" "')}\"" + def aarch64_targets = "\"${targets.findAll { it.system == "aarch64-linux" }.target.join('" "')}\"" + + // nix-eval-jobs is used to evaluate the targets in parallel and compute derivation paths. + // nix expression is used to create an attset on the fly which is a subset of #packages, + // but only includes the targets we want to build, to save time + sh """ + nix-eval-jobs --gc-roots-dir gcroots --force-recurse --expr ' \ + let \ + flake = builtins.getFlake ("git+file://" + toString ./.); \ + lib = (import flake.inputs.nixpkgs { }).lib; \ + in { \ + x86_64-linux = lib.getAttrs [ ${x86_targets} ] flake.packages.x86_64-linux; \ + aarch64-linux = lib.getAttrs [ ${aarch64_targets} ] flake.packages.aarch64-linux; \ + }' > jobs.json + """ + + // target's name and derivation path are read from jobs.json and written into into jobs.txt + sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt" +} + return this ////////////////////////////////////////////////////////////////////////////////