Skip to content

Commit

Permalink
Merge branch 'main' into feature/sign-cert-split-pi
Browse files Browse the repository at this point in the history 
Signed-off-by: Aleksandr Tserepov-Savolainen <[email protected]>
  • Loading branch information
@alextserepov
alextserepov committed Nov 6, 2024
2 parents 7868e77 + 97e20dc commit 67adc3d
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 48 deletions.
45 changes: 19 additions & 26 deletions ghaf-main-pipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,32 @@ properties([
githubProjectProperty(displayName: '', projectUrlStr: REPO_URL),
])

// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux",
[ system: "aarch64-linux", target: "doc",
archive: false, hwtest_device: null
],
[ target: "docs.x86_64-linux",
[ system: "x86_64-linux", target: "doc",
archive: false, hwtest_device: null
],
[ target: "generic-x86_64-debug.x86_64-linux",
[ system: "x86_64-linux", target: "generic-x86_64-debug",
archive: true, hwtest_device: "nuc"
],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux",
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug",
archive: true, hwtest_device: "lenovo-x1"
],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64",
archive: true, hwtest_device: "riscv"
],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux",
[ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug",
archive: true, hwtest_device: "orin-agx"
],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64",
archive: true, hwtest_device: "orin-agx"
],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux",
[ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug",
archive: true, hwtest_device: "orin-nx"
],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64",
archive: true, hwtest_device: "orin-nx"
],
]
Expand Down Expand Up @@ -83,22 +79,19 @@ pipeline {
steps {
dir(WORKDIR) {
script {
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
// Creates jobs.txt in working directory to use later
utils.nix_eval_jobs(targets)

targets.each {
def target = it['target']
def target = it.system + "." + it.target

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
target_jobs["${it.target} (${it.system})"] = {
stage("Build ${target}") {
def opts = ""
if (it['archive']) {
if (it.archive) {
opts = "--out-link archive/${target}"
} else {
opts = "--no-link"
Expand All @@ -108,12 +101,12 @@ pipeline {
sh "nix build -L ${drvPath}\\^* ${opts}"

// only attempt signing if there is something to sign
if (it['archive']) {
if (it.archive) {
def img_relpath = utils.find_img_relpath(target, "archive")
utils.sign_file("archive/${img_relpath}", "sig/${img_relpath}.sig", "INT-Ghaf-Devenv-Image")
}
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
error("Target \"${target}\" was not found in packages")
}
} catch (InterruptedException e) {
throw e
Expand All @@ -124,7 +117,7 @@ pipeline {
}
}

if (it['archive']) {
if (it.archive) {
stage("Archive ${target}") {
script {
utils.archive_artifacts("archive", target)
Expand All @@ -133,9 +126,9 @@ pipeline {
}
}

if (it['hwtest_device'] != null) {
if (it.hwtest_device != null) {
stage("Test ${target}") {
utils.ghaf_parallel_hw_test(target, it['hwtest_device'], '_boot_bat_')
utils.ghaf_parallel_hw_test(target, it.hwtest_device, '_boot_bat_')
}
}
}
Expand Down
37 changes: 15 additions & 22 deletions ghaf-pre-merge-pipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,16 @@
def REPO_URL = 'https://github.com/tiiuae/ghaf/'
def WORKDIR = 'ghaf'

// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux" ],
[ target: "docs.x86_64-linux" ],
[ target: "generic-x86_64-debug.x86_64-linux" ],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux" ],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux" ],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux" ],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux" ],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux" ],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux" ],
[ system: "aarch64-linux", target: "doc", ],
[ system: "x86_64-linux", target: "doc", ],
[ system: "x86_64-linux", target: "generic-x86_64-debug", ],
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug", ],
[ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64", ],
[ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug", ],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64", ],
[ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug", ],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64", ],
]

target_jobs = [:]
Expand Down Expand Up @@ -140,25 +136,22 @@ pipeline {
steps {
dir(WORKDIR) {
script {
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
// Creates jobs.txt in working directory to use later
utils.nix_eval_jobs(targets)

targets.each {
def target = it['target']
def target = it.system + "." + it.target

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
stage("${target}") {
target_jobs["${it.target} (${it.system})"] = {
stage("Build ${target}") {
try {
if (drvPath) {
sh "nix build --no-link -L ${drvPath}\\^*"
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
error("Target \"${target}\" was not found in packages")
}
} catch (InterruptedException e) {
throw e
Expand Down
23 changes: 23 additions & 0 deletions utils.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,29 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset=
archive_artifacts("ghaf-parallel-hw-test", flakeref_trimmed)
}

def nix_eval_jobs(List<Map> targets) {
// transform target names into valid nix arrays to be plugged into the expression below
def x86_targets = "\"${targets.findAll { it.system == "x86_64-linux" }.target.join('" "')}\""
def aarch64_targets = "\"${targets.findAll { it.system == "aarch64-linux" }.target.join('" "')}\""

// nix-eval-jobs is used to evaluate the targets in parallel and compute derivation paths.
// nix expression is used to create an attset on the fly which is a subset of #packages,
// but only includes the targets we want to build, to save time
sh """
nix-eval-jobs --gc-roots-dir gcroots --force-recurse --expr ' \
let \
flake = builtins.getFlake ("git+file://" + toString ./.); \
lib = (import flake.inputs.nixpkgs { }).lib; \
in { \
x86_64-linux = lib.getAttrs [ ${x86_targets} ] flake.packages.x86_64-linux; \
aarch64-linux = lib.getAttrs [ ${aarch64_targets} ] flake.packages.aarch64-linux; \
}' > jobs.json
"""

// target's name and derivation path are read from jobs.json and written into into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
}

return this

////////////////////////////////////////////////////////////////////////////////

0 comments on commit 67adc3d

Please sign in to comment.