Skip to content

Commit

Permalink
Merge pull request #77 from joinemm/pr-parallel-main
Browse files Browse the repository at this point in the history 
Build the main pipeline in parallel
  • Loading branch information
@vjuntunen
vjuntunen authored Oct 17, 2024
2 parents 59ca9be + 0fcebfe commit af25e7c
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 35 deletions.
111 changes: 80 additions & 31 deletions ghaf-main-pipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,39 @@ properties([
githubProjectProperty(displayName: '', projectUrlStr: REPO_URL),
])

////////////////////////////////////////////////////////////////////////////////
// Which attribute of the flake to evaluate for building
def flakeAttr = ".#hydraJobs"

// Target names must be direct children of the above
def targets = [
[ target: "docs.aarch64-linux",
hwtest_device: null ],
[ target: "docs.x86_64-linux",
hwtest_device: null ],
[ target: "generic-x86_64-debug.x86_64-linux",
hwtest_device: "nuc" ],
[ target: "lenovo-x1-carbon-gen11-debug.x86_64-linux",
hwtest_device: "lenovo-x1" ],
[ target: "microchip-icicle-kit-debug-from-x86_64.x86_64-linux",
hwtest_device: null ],
[ target: "nvidia-jetson-orin-agx-debug.aarch64-linux",
hwtest_device: "orin-agx" ],
[ target: "nvidia-jetson-orin-agx-debug-from-x86_64.x86_64-linux",
hwtest_device: "orin-agx" ],
[ target: "nvidia-jetson-orin-nx-debug.aarch64-linux",
hwtest_device: "orin-nx" ],
[ target: "nvidia-jetson-orin-nx-debug-from-x86_64.x86_64-linux",
hwtest_device: "orin-nx" ],
]

target_jobs = [:]

pipeline {
agent { label 'built-in' }
triggers {
pollSCM('* * * * *')
}
options {
disableConcurrentBuilds()
timestamps ()
buildDiscarder(logRotator(numToKeepStr: '100'))
}
Expand All @@ -44,43 +68,68 @@ pipeline {
}
}
}
stage('Build x86_64') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'archive')
utils.nix_build('.#packages.x86_64-linux.generic-x86_64-debug', 'archive')
utils.nix_build('.#packages.x86_64-linux.doc')
}
}
}
}
stage('Build aarch64') {

stage('Evaluate') {
steps {
dir(WORKDIR) {
script {
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'archive')
utils.nix_build('.#packages.aarch64-linux.doc')
// nix-eval-jobs is used to evaluate the given flake attribute, and output target information into jobs.json
sh "nix-eval-jobs --gc-roots-dir gcroots --flake ${flakeAttr} --force-recurse > jobs.json"

// jobs.json is parsed using jq. target's name and derivation path are appended as space separated row into jobs.txt
sh "jq -r '.attr + \" \" + .drvPath' < jobs.json > jobs.txt"

targets.each {
def target = it['target']

// row that matches this target is grepped from jobs.txt, extracting the pre-evaluated derivation path
def drvPath = sh (script: "cat jobs.txt | grep ${target} | cut -d ' ' -f 2", returnStdout: true).trim()

target_jobs[target] = {
stage("Build ${target}") {
def opts = ""
if (it['hwtest_device'] != null) {
opts = "--out-link archive/${target}"
} else {
opts = "--no-link"
}
try {
if (drvPath) {
sh "nix build -L ${drvPath}\\^* ${opts}"
} else {
error("Target \"${target}\" was not found in ${flakeAttr}")
}
} catch (InterruptedException e) {
throw e
} catch (Exception e) {
unstable("FAILED: ${target}")
currentBuild.result = "FAILURE"
println "Error: ${e.toString()}"
}
}

if (it['hwtest_device'] != null) {
stage("Archive ${target}") {
script {
utils.archive_artifacts("archive", target)
}
}

stage("Test ${target}") {
utils.ghaf_parallel_hw_test(target, it['hwtest_device'])
}
}
}
}
}
}
}
}
stage('HW test') {

stage('Build targets') {
steps {
dir(WORKDIR) {
script {
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-agx-debug-from-x86_64', 'orin-agx')
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-agx-debug', 'orin-agx')
utils.ghaf_hw_test('.#packages.x86_64-linux.nvidia-jetson-orin-nx-debug-from-x86_64', 'orin-nx')
utils.ghaf_hw_test('.#packages.aarch64-linux.nvidia-jetson-orin-nx-debug', 'orin-nx')
utils.ghaf_hw_test('.#packages.x86_64-linux.lenovo-x1-carbon-gen11-debug', 'lenovo-x1')
utils.ghaf_hw_test('.#packages.x86_64-linux.generic-x86_64-debug', 'nuc')
utils.ghaf_hw_test('.#packages.x86_64-linux.microchip-icicle-kit-debug-from-x86_64', 'riscv')
}
script {
parallel target_jobs
}
}
}
Expand Down
8 changes: 4 additions & 4 deletions utils.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def sign_file(String path, String sigfile, String cert="INT-Ghaf-Devenv-Common")
}

def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_') {
testagent_nodes = nodesByLabel(label: 'testagent', offline: false)
testagent_nodes = nodesByLabel(label: "$device_config", offline: false)
if (!testagent_nodes) {
println "Warning: Skipping HW test '$flakeref', no test agents online"
unstable("No test agents online")
Expand Down Expand Up @@ -217,7 +217,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
job: "ghaf-hw-test",
propagate: false,
parameters: [
string(name: "LABEL", value: "testagent"),
string(name: "LABEL", value: "$device_config"),
string(name: "DEVICE_CONFIG_NAME", value: "$device_config"),
string(name: "IMG_URL", value: "$img_url"),
string(name: "DESC", value: "$description"),
Expand Down Expand Up @@ -247,7 +247,7 @@ def ghaf_hw_test(String flakeref, String device_config, String testset='_boot_')
}

def ghaf_parallel_hw_test(String flakeref, String device_config, String testset='_boot_') {
testagent_nodes = nodesByLabel(label: "testagent_$device_config", offline: false)
testagent_nodes = nodesByLabel(label: "$device_config", offline: false)
if (!testagent_nodes) {
println "Warning: Skipping HW test '$flakeref', no test agents online"
unstable("No test agents online")
Expand Down Expand Up @@ -275,7 +275,7 @@ def ghaf_parallel_hw_test(String flakeref, String device_config, String testset=
job: "ghaf-parallel-hw-test",
propagate: false,
parameters: [
string(name: "LABEL", value: "testagent_$device_config"),
string(name: "LABEL", value: "$device_config"),
string(name: "DEVICE_CONFIG_NAME", value: "$device_config"),
string(name: "IMG_URL", value: "$img_url"),
string(name: "DESC", value: "$description"),
Expand Down

0 comments on commit af25e7c

Please sign in to comment.