Jenkinsfile.cycle

#!/usr/bin/env groovy

conda_packages = [
  "ataos",
  "atbuilding",
  "atdome",
  "atdometrajectory",
  "athexapod",
  "atmcs_sim",
  "atmonochromator",
  "atpneumatics_sim",
  "atspectrograph",
  "atwhitelight",
  "audio_broadcaster",
  "authorize",
  "cbp",
  "dimm",
  "dream",
  "dmocps",
  "dsm",
  "eas",
  "electrometer",
  "epm",
  "ess",
  "fiberspectrograph",
  "genericcamera",
  "gis",
  "headerservice",
  "hvac",
  "integrationtests",
  "lasertracker",
  "ledprojector",
  "linearstage",
  "love-commander",
  "love-csc",
  "love-frontend",
  "love-frontend-k8s",
  "love-manager",
  "love-manager-static",
  "love-producer",
  "love-view-backup",
  "mtaircompressor",
  "mthexapod",
  "mtdome",
  "mtdometrajectory",
  "mtm2",
  "mtmount",
  "mtrotator",
  "nightreport",
  "ospl-daemon",
  "pmd",
  "salkafka",
  "scheduler",
  "schedview",
  "test",
  "tunablelaser",
  "watcher",
  "weatherforecast"
]
pipeline {

  options {
    buildDiscarder(
      logRotator(
        artifactDaysToKeepStr: '',
        artifactNumToKeepStr: '',
        daysToKeepStr: '14',
        numToKeepStr: '10'
      ))
    disableConcurrentBuilds()
  }

  agent {
      label "${params.build_agent}"
  }

  environment {
    registryCredential = "dockerhub-lssttsadmin"
    pub_registryCredential = "dockerhub-lssttsadmin"
    CONDA_PACKAGES = conda_packages.join(" ")
    CONDA_PACKAGES_C = conda_packages.join("_c ").concat("_c")
    CAMSPEC_GENCAM = "genericcamera-canon genericcamera-vimba"
    CAMSPEC_GENCAM_C = "genericcamera-canon_c genericcamera-vimba_c"
  }
  parameters {
      booleanParam(
        defaultValue: false,
        description: "Set the CACHE_OPTION variable to '--no-cache'",
        name: 'set_cache_option'
      )
      choice choices: ['Node3_4CPU', 'CSC_Conda_Node', 'Node1_4CPU', 'Node2_8CPU'], description: 'Select the build agent', name: 'build_agent'
      booleanParam defaultValue: false, description: "Build/push Conda Package Builder arm64 and aarch64 images.", name: 'build_conda_package_builder'
      booleanParam defaultValue: false, description: "Build/push base deployment image (only run this if you are building revision 0).", name: 'deploy_conda'
      booleanParam defaultValue: false, description: "Build/push lsstsqre deployment image (only run this if you are building revision 0 or updating the DM stack version).", name: 'deploy_lsstsqre'
      booleanParam defaultValue: false, description: "Build/push ts-cycle conda package.", name: 'ts_cycle'
      booleanParam defaultValue: false, description: "Build/push all components based on base deployment image.", name: 'base_components'
      choice(name: 'component', choices: ["ALL"] + conda_packages, description: "Select one component to build. Building all components is only allowed when building revision 0.")
      booleanParam defaultValue: false, description: "Build/push ESS Controller image.", name: 'build_ess_controller'
      booleanParam defaultValue: false, description: "Build/push Vents Controller image.", name: 'build_vents_controller'
      booleanParam defaultValue: false, description: "Build/push AudioTrigger Controller image.", name: 'build_audiotrigger_controller'
      booleanParam defaultValue: false, description: "Build/push camera-specific GenericCamera images.", name: 'build_camspec_gencam'
      booleanParam defaultValue: false, description: "Build/push M1M3 simulator image.", name: 'build_m1m3_sim'
      booleanParam defaultValue: false, description: "Build/push pointing component deployment image.", name: 'build_ptg'
      booleanParam defaultValue: false, description: "Build/push MTAOS image.", name: 'build_mtaos'
      booleanParam defaultValue: false, description: "Build/push OODS image.", name: 'build_oods'
      booleanParam defaultValue: false, description: "Build/push scriptqueue image.", name: 'build_scriptqueue'
      booleanParam defaultValue: false, description: "Build/push sal-sciplat recommended image? This adds Telescope and Site layer to a lsstsqre/centos image in prepartion for building nublado images.", name: 'build_sciplat'
      booleanParam defaultValue: false, description: "Build/push sal-sciplat-lab recommended image? Adds SQuaRe layer to sal-sciplat to produce a nublado-ready image.", name: 'build_sciplat_lab_recommended'
      booleanParam defaultValue: false, description: "Build/push RubinTV Rapid Analysis.", name: 'rapid_analysis'
      booleanParam defaultValue: false, description: "Build/push development image.", name: 'develop_env'
  }

  stages {

    stage ('Build Conda Package Builder images') {
      when {
          expression { params.build_conda_package_builder }
      }
      steps {
          build job: "CondaPackageBuilder_amd64/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_conda_package_builder', value: true)], wait: false
          build job: "CondaPackageBuilder_aarch64/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_conda_package_builder_aarch64', value: true)], wait: false
      }
    }
    stage ('Build deploy-conda-private') {
      when {
          expression { params.deploy_conda }
      }
      steps {

        withEnv(["HOME=${env.WORKSPACE}"]) {
          sh """
          export \$(cat cycle/cycle.env | grep rev | sed  's/#.*//g' | xargs)
          export \$(cat cycle/cycle.env | grep CYCLE | sed  's/#.*//g' | xargs)
          if [ "\$rev" != ".000" ]; then echo "FAILED: base conda image can only be built when building revision 0."; exit 1; else echo "BUILDING CYCLE \${CYCLE}\${rev}";fi
          """
        }

        withCredentials([file(credentialsId: 'yum-nexus-private', variable: 'yum_nexus_private')]) {
          sh "cp \$yum_nexus_private ${env.WORKSPACE}/build/common/"
        }

      // When using the docker container, we need to change
      // the HOME path to WORKSPACE to have the authority
      // to install the packages.
        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {


            sh """
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} deploy-conda-private
            """
          }
        }

        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push deploy-conda-private
            """
          }
        }
      }
    }
    stage ('Build deploy-lsstsqre-private') {
      when {
          expression { params.deploy_lsstsqre }
      }
      steps {

        withCredentials([file(credentialsId: 'yum-nexus-private', variable: 'yum_nexus_private')]) {
          sh "cp \$yum_nexus_private ${env.WORKSPACE}/build/common/"
        }
        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {

            sh """
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} deploy-lsstsqre-private
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build deploy-lsstsqre-private_c
            """
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push deploy-lsstsqre-private deploy-lsstsqre-private_c
            """
          }
        }
      }
    }
    stage ('Build ts-cycle conda package') {
      when {
          expression { params.ts_cycle }
      }
      steps {
          build job: "Cycle_conda_package/" + env.BRANCH_NAME.replace("/", "%2F"), wait: false
      }
    }
    stage ('Build ALL base components') {
      when {
        allOf {
          expression { params.base_components }
          expression { params.component.equals("ALL")}
        }
      }
      steps {

        withEnv(["HOME=${env.WORKSPACE}"]) {
          sh """
          export \$(cat cycle/cycle.env | grep rev | sed  's/#.*//g' | xargs)
          export \$(cat cycle/cycle.env | grep CYCLE | sed  's/#.*//g' | xargs)
          if [ "\$rev" != ".000" ]; then echo "FAILED: Building ALL components only allowed when building revision 0."; exit 1 ; else echo "BUILDING CYCLE \${CYCLE}\${rev}";fi
          """
        }

        script{
          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              export DOCKER_BUILDKIT=0
              for comp in \${CONDA_PACKAGES}; do
                docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} \${comp}
              done
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build \${CONDA_PACKAGES_C}
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push \${CONDA_PACKAGES} \${CONDA_PACKAGES_C}
            """
          }
        }
      }
    }
    stage ('Build single base component') {
      when {
        allOf {
          expression { params.base_components }
          not{ expression { params.component.equals("ALL") } }
        }
      }
      steps {
        script{
          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} ${params.component}
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} ${params.component}_c
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push ${params.component} ${params.component}_c
            """
          }
        }
      }
    }
    stage ('Build ESS Controller Docker image') {
      when {
          expression { params.build_ess_controller }
      }
      steps {
          build job: "EssController/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_ess_controller', value: true)], wait: false
      }
    }
    stage ('Build Vents Controller Docker image') {
      when {
          expression { params.build_vents_controller }
      }
      steps {
          build job: "VentsController/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_vents_controller', value: true)], wait: false
      }
    }
    stage ('Build AudioTrigger Controller Docker image') {
      when {
          expression { params.build_audiotrigger_controller }
      }
      steps {
          build job: "AudioTriggerController/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_audiotrigger_controller', value: true)], wait: false
      }
    }
    stage ('Build camera specific GenericCamera Docker images') {
      when {
          expression { params.build_camspec_gencam }
      }
      steps {

        withCredentials([file(credentialsId: 'nexus-netrc', variable: 'nexus_netrc')]) {
            sh "cp \$nexus_netrc ${env.WORKSPACE}/build/common/"
        }
        script{
        def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} \${CAMSPEC_GENCAM}
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build \${CAMSPEC_GENCAM_C}
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push \${CAMSPEC_GENCAM} \${CAMSPEC_GENCAM_C}
            """
          }
        }
      }
    }
    stage ('Build M1M3 simulator') {
      when {
          expression { params.build_m1m3_sim }
      }
      steps {

        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} mtm1m3_sim
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build mtm1m3_sim_c
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push mtm1m3_sim mtm1m3_sim_c
            """
          }
        }
      }
    }
    stage ('Build PTG') {
      when {
          expression { params.build_ptg }
      }
      steps {

        script{

          // Create a variable based on user input
          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withCredentials([sshUserPrivateKey(credentialsId: 'RubinObsWorkerSSHKey', keyFileVariable: 'github_cred')]) {
            sh "cp \$github_cred ${env.WORKSPACE}/build/ptg/id_rsa"
          }
          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} ptg ptg_c
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ptg_c
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push ptg ptg_c
            """
          }
        }
      }
    }
    stage ('Build mtaos') {
      when {
          expression { params.build_mtaos }
      }
      steps {

        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              export DOCKER_BUILDKIT=0
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} mtaos
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build mtaos_c
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push mtaos mtaos_c
            """
          }
        }
      }
    }
    stage ('Build oods') {
      when {
        expression { params.build_oods }
      }
      steps {
        script {
          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh"""
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build oods oods_c
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build atoods atoods_c ccoods ccoods_c mtoods mtoods_c
              """
            }
          }
        }
        script {
          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh"""
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push oods oods_c atoods atoods_c ccoods ccoods_c mtoods mtoods_c
              """
            }
          }
        }
      }
    }
    stage ('Build scriptqueue') {
      when {
          expression { params.build_scriptqueue }
      }
      steps {

        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
              sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} scriptqueue
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build scriptqueue_c
              """
              }
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push scriptqueue scriptqueue_c
            """
          }
        }
      }
    }
    stage ('Build Sal Sciplat (pre-nublado) recommended image') {
      when {
          expression { params.build_sciplat }
      }
      steps {

        withCredentials([file(credentialsId: 'yum-nexus-private', variable: 'yum_nexus_private')]) {
          sh "cp \$yum_nexus_private ${env.WORKSPACE}/build/common/"
        }

        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            sh """
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} opensplice-lsstsqre-licensed
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} sal-sciplat-licensed
            """
          }
        }

        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push sal-sciplat-licensed
            """
          }
        }
      }
    }
    stage ('Build Sal Sciplat Lab (nublado) recommended image') {
      when {
          expression { params.build_sciplat_lab_recommended }
      }
      steps {
          build job: "SalSciplatLab/" + env.BRANCH_NAME.replace("/", "%2F"), parameters: [booleanParam(name: 'build_sciplat_lab_recommended', value: true)], wait: false
      }
    }
    stage ('Build RubinTV Rapid Analysis') {
      when {
          expression { params.rapid_analysis }
      }
      steps {

        script{

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          withEnv(["HOME=${env.WORKSPACE}"]) {
            sh """
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} rapid-analysis
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build rapid-analysis_c
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} rapid-analysis-usdf
            docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build rapid-analysis-usdf_c
            """
          }
        }
        script {
          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push rapid-analysis rapid-analysis_c
            """
          }
        }
        script {
          docker.withRegistry("", pub_registryCredential) {
            sh """
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push rapid-analysis-usdf rapid-analysis-usdf_c
            """
          }
        }
      }
    }
    stage ('Build development image') {
      when {
          expression { params.develop_env }
      }
      steps {
        withCredentials([file(credentialsId: 'yum-nexus-private', variable: 'yum_nexus_private')]) {
          sh "cp \$yum_nexus_private ${env.WORKSPACE}/build/common/"
        }
        script {

          def cacheOption = params.set_cache_option ? "--no-cache" : ""

          docker.withRegistry("https://ts-dockerhub.lsst.org/", "nexus3-lsst_jenkins") {
            sh """
              export DOCKER_BUILDKIT=0
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} --pull base-sqre-private
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push base-sqre-private

              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} --pull salobj-private
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push salobj-private

              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml build ${cacheOption} --pull develop-env-private
              docker compose --env-file cycle/cycle.env -f cycle/docker-compose.yaml push develop-env-private
            """
          }
        }
      }
    }
  }
}