Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only evaluate what is needed to build #84

Merged
merged 1 commit into from
Nov 6, 2024
Merged

Only evaluate what is needed to build #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 19 additions & 26 deletions ghaf-main-pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,36 +15,32 @@ properties([
githubProjectProperty(displayName: '', projectUrlStr: REPO_URL),
])

// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux",
[ system: "aarch64-linux", target: "doc",
archive: false, hwtest_device: null
],
[ target: "docs.x86_64-linux",
[ system: "x86_64-linux", target: "doc",
archive: false, hwtest_device: null
],
[ target: "generic-x86_64-debug.x86_64-linux",
[ system: "x86_64-linux", target: "generic-x86_64-debug",
archive: true, hwtest_device: "nuc"
],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux",
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug",
archive: true, hwtest_device: "lenovo-x1"
],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64",
archive: true, hwtest_device: "riscv"
],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux",
[ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug",
archive: true, hwtest_device: "orin-agx"
],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64",
archive: true, hwtest_device: "orin-agx"
],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux",
[ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug",
archive: true, hwtest_device: "orin-nx"
],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux",
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64",
archive: true, hwtest_device: "orin-nx"
],
]
Expand Down Expand Up @@ -83,22 +79,19 @@ pipeline {
steps {
dir(WORKDIR) {
script {
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
// Creates jobs.txt in working directory to use later
utils.nix_eval_jobs(targets)

targets.each {
def target = it['target']
def target = it.system + "." + it.target

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
target_jobs["${it.target} (${it.system})"] = {
stage("Build ${target}") {
def opts = ""
if (it['archive']) {
if (it.archive) {
opts = "--out-link archive/${target}"
} else {
opts = "--no-link"
Expand All @@ -108,12 +101,12 @@ pipeline {
sh "nix build -L ${drvPath}\\^* ${opts}"

// only attempt signing if there is something to sign
if (it['archive']) {
if (it.archive) {
def img_relpath = utils.find_img_relpath(target, "archive")
utils.sign_file("archive/${img_relpath}", "sig/${img_relpath}.sig")
}
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
error("Target \"${target}\" was not found in packages")
}
} catch (InterruptedException e) {
throw e
Expand All @@ -124,7 +117,7 @@ pipeline {
}
}

if (it['archive']) {
if (it.archive) {
stage("Archive ${target}") {
script {
utils.archive_artifacts("archive", target)
Expand All @@ -133,9 +126,9 @@ pipeline {
}
}

if (it['hwtest_device'] != null) {
if (it.hwtest_device != null) {
stage("Test ${target}") {
utils.ghaf_parallel_hw_test(target, it['hwtest_device'], '_boot_bat_')
utils.ghaf_parallel_hw_test(target, it.hwtest_device, '_boot_bat_')
}
}
}
Expand Down
37 changes: 15 additions & 22 deletions ghaf-pre-merge-pipeline.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,20 +8,16 @@
def REPO_URL = 'https://github.com/tiiuae/ghaf/'
def WORKDIR = 'ghaf'

// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux" ],
[ target: "docs.x86_64-linux" ],
[ target: "generic-x86_64-debug.x86_64-linux" ],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux" ],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux" ],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux" ],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux" ],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux" ],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux" ],
[ system: "aarch64-linux", target: "doc", ],
[ system: "x86_64-linux", target: "doc", ],
[ system: "x86_64-linux", target: "generic-x86_64-debug", ],
[ system: "x86_64-linux", target: "lenovo-x1-carbon-gen11-debug", ],
[ system: "x86_64-linux", target: "microchip-icicle-kit-debug-from-x86_64", ],
[ system: "aarch64-linux", target: "nvidia-jetson-orin-agx-debug", ],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-agx-debug-from-x86_64", ],
[ system: "aarch64-linux", target: "nvidia-jetson-orin-nx-debug", ],
[ system: "x86_64-linux", target: "nvidia-jetson-orin-nx-debug-from-x86_64", ],
]

target_jobs = [:]
Expand Down Expand Up @@ -140,25 +136,22 @@ pipeline {
steps {
dir(WORKDIR) {
script {
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
// Creates jobs.txt in working directory to use later
utils.nix_eval_jobs(targets)

targets.each {
def target = it['target']
def target = it.system + "." + it.target

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
stage("${target}") {
target_jobs["${it.target} (${it.system})"] = {
stage("Build ${target}") {
try {
if (drvPath) {
sh "nix build --no-link -L ${drvPath}\\^*"
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
error("Target \"${target}\" was not found in packages")
}
} catch (InterruptedException e) {
throw e
Expand Down
23 changes: 23 additions & 0 deletions utils.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,29 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset=
archive_artifacts("ghaf-parallel-hw-test", flakeref_trimmed)
}

def nix_eval_jobs(List<Map> targets) {
// transform target names into valid nix arrays to be plugged into the expression below
def x86_targets = "\"${targets.findAll { it.system == "x86_64-linux" }.target.join('" "')}\""
def aarch64_targets = "\"${targets.findAll { it.system == "aarch64-linux" }.target.join('" "')}\""

// nix-eval-jobs is used to evaluate the targets in parallel and compute derivation paths.
// nix expression is used to create an attset on the fly which is a subset of #packages,
// but only includes the targets we want to build, to save time
sh """
nix-eval-jobs --gc-roots-dir gcroots --force-recurse --expr ' \
let \
flake = builtins.getFlake ("git+file://" + toString ./.); \
lib = (import flake.inputs.nixpkgs { }).lib; \
in { \
x86_64-linux = lib.getAttrs [ ${x86_targets} ] flake.packages.x86_64-linux; \
aarch64-linux = lib.getAttrs [ ${aarch64_targets} ] flake.packages.aarch64-linux; \
}' > jobs.json
"""

// target's name and derivation path are read from jobs.json and written into into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"
}

return this

////////////////////////////////////////////////////////////////////////////////
Loading