Fix AMI builds

Use paths for driver validation Use different host devices for Docker Signed-off-by: Chris Yan <[email protected]>
samuel-lee-msft · Nov 22, 2021 · ba53c10 · ba53c10
1 parent 4059c08
commit ba53c10
Show file tree

Hide file tree

Showing 6 changed files with 77 additions and 28 deletions.
diff --git a/.jenkins/infrastructure/build_azure_managed_images.Jenkinsfile b/.jenkins/infrastructure/build_azure_managed_images.Jenkinsfile
@@ -5,7 +5,7 @@ import java.time.*
 import java.time.format.DateTimeFormatter
 
 OECI_LIB_VERSION = env.OECI_LIB_VERSION ?: "master"
-oe = library("OpenEnclaveCommon@${OECI_LIB_VERSION}").jenkins.common.Openenclave.new()
+library "OpenEnclaveJenkinsLibrary@${params.OECI_LIB_VERSION}"
 
 GLOBAL_TIMEOUT_MINUTES = 480
 
@@ -59,7 +59,7 @@ def buildLinuxManagedImage(String os_type, String version, String image_id, Stri
                         --gallery-image-definition ${os_type}-${version} \
                         --gallery-image-version ${gallery_image_version}
                 """
-                oe.azureEnvironment(az_cleanup_existing_image_version_script, params.OE_DEPLOY_IMAGE)
+                common.azureEnvironment(az_cleanup_existing_image_version_script, params.OE_DEPLOY_IMAGE)
             }
             stage("Run Packer Job") {
                 timeout(GLOBAL_TIMEOUT_MINUTES) {
@@ -69,11 +69,11 @@ def buildLinuxManagedImage(String os_type, String version, String image_id, Stri
                                     usernamePassword(credentialsId: JENKINS_USER_CREDS_ID,
                                                     usernameVariable: "SSH_USERNAME",
                                                     passwordVariable: "SSH_PASSWORD")]) {
-                        def cmd = ("packer build -force " +
-                                    "-var-file=${WORKSPACE}/.jenkins/infrastructure/provision/templates/packer/azure_managed_image/${os_type}-${version}-variables.json " +
-                                    "${WORKSPACE}/.jenkins/infrastructure/provision/templates/packer/azure_managed_image/packer-${os_type}.json")
-                        oe.exec_with_retry(10, 60) {
-                            oe.azureEnvironment(cmd, params.OE_DEPLOY_IMAGE)
+                        def cmd = ("""packer build -force \
+                                        -var-file=${WORKSPACE}/.jenkins/infrastructure/provision/templates/packer/azure_managed_image/${os_type}-${version}-variables.json \
+                                        ${WORKSPACE}/.jenkins/infrastructure/provision/templates/packer/azure_managed_image/packer-${os_type}.json""")
+                        common.exec_with_retry(10, 60) {
+                            common.azureEnvironment(cmd, params.OE_DEPLOY_IMAGE)
                         }
                     }
                 }
@@ -106,7 +106,7 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                     ${az_login_script}
                     az group create --name ${vm_rg_name} --location ${REGION}
                 """
-                oe.azureEnvironment(az_rg_create_script, params.OE_DEPLOY_IMAGE)
+                common.azureEnvironment(az_rg_create_script, params.OE_DEPLOY_IMAGE)
             }
 
             try {
@@ -130,7 +130,7 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                             --admin-password ${JENKINS_USER_PASSWORD} \
                             --image ${azure_image_id}
                     """
-                    oe.azureEnvironment(provision_script, params.OE_DEPLOY_IMAGE)
+                    common.azureEnvironment(provision_script, params.OE_DEPLOY_IMAGE)
                 }
 
                 stage("Deploy VM") {
@@ -165,8 +165,8 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                             --command-id RunPowerShellScript \
                             --scripts @${WORKSPACE}/.jenkins/infrastructure/provision/run-sysprep.ps1
                     """
-                    oe.exec_with_retry(10, 30) {
-                        oe.azureEnvironment(deploy_script, params.OE_DEPLOY_IMAGE)
+                    common.exec_with_retry(10, 30) {
+                        common.azureEnvironment(deploy_script, params.OE_DEPLOY_IMAGE)
                     }
                 }
 
@@ -179,8 +179,8 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                             az vm deallocate --resource-group ${vm_rg_name} --name ${vm_name}
                             az vm generalize --resource-group ${vm_rg_name} --name ${vm_name}
                         """
-                        oe.exec_with_retry(10, 30) {
-                            oe.azureEnvironment(generalize_script, params.OE_DEPLOY_IMAGE)
+                        common.exec_with_retry(10, 30) {
+                            common.azureEnvironment(generalize_script, params.OE_DEPLOY_IMAGE)
                         }
                     }
                 }
@@ -206,8 +206,8 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                                 --hyper-v-generation ${AZURE_IMAGES_MAP[os_series]["generation"]} \
                                 --source \$VM_ID
                             """
-                        oe.exec_with_retry(10, 30) {
-                            oe.azureEnvironment(capture_script, params.OE_DEPLOY_IMAGE)
+                        common.exec_with_retry(10, 30) {
+                            common.azureEnvironment(capture_script, params.OE_DEPLOY_IMAGE)
                         }
                     }
                 }
@@ -239,8 +239,8 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                                 --target-regions ${env.REPLICATION_REGIONS.split(',').join(' ')} \
                                 --replica-count 1
                         """
-                        oe.exec_with_retry(10, 30) {
-                            oe.azureEnvironment(upload_script, params.OE_DEPLOY_IMAGE)
+                        common.exec_with_retry(10, 30) {
+                            common.azureEnvironment(upload_script, params.OE_DEPLOY_IMAGE)
                         }
                     }
                 }
@@ -251,7 +251,7 @@ def buildWindowsManagedImage(String os_series, String img_name_suffix, String la
                         ${az_login_script}
                         az group delete --name ${vm_rg_name} --yes
                     """
-                    oe.azureEnvironment(az_rg_cleanup_script, params.OE_DEPLOY_IMAGE)
+                    common.azureEnvironment(az_rg_cleanup_script, params.OE_DEPLOY_IMAGE)
                 }
             }
         }

diff --git a/...infrastructure/provision/templates/packer/azure_managed_image/ubuntu-18.04-variables.json b/...infrastructure/provision/templates/packer/azure_managed_image/ubuntu-18.04-variables.json
@@ -6,6 +6,6 @@
     "ansible_group": "linux-agents",
     "playbook_file_name": "oe-linux-acc-setup.yml",
     "base_gallery_image_version": "latest",
-    "base_gallery_image_name": "Ubuntu_1804_LTS_Gen2",
+    "base_gallery_image_name": "Ubuntu_18.04_LTS_Gen2",
     "base_gallery_name": "Vanilla_Images"
 }
diff --git a/...infrastructure/provision/templates/packer/azure_managed_image/ubuntu-20.04-variables.json b/...infrastructure/provision/templates/packer/azure_managed_image/ubuntu-20.04-variables.json
@@ -6,6 +6,6 @@
     "ansible_group": "linux-agents",
     "playbook_file_name": "oe-linux-acc-setup.yml",
     "base_gallery_image_version": "latest",
-    "base_gallery_image_name": "Ubuntu_2004_LTS_Gen2",
+    "base_gallery_image_name": "Ubuntu_20.04_LTS_Gen2",
     "base_gallery_name": "Vanilla_Images"
 }
diff --git a/.jenkins/library/vars/helpers.groovy b/.jenkins/library/vars/helpers.groovy
@@ -87,6 +87,7 @@ def getWindowsCwd() {
             returnStdout: true
         ).trim()
 }
+
 /**
  * Tests Open Enclave samples on *nix systems
  *
@@ -513,3 +514,43 @@ def get_date(String delimiter = "") {
         return "Canonical:UbuntuServer:18_04-lts-gen2:latest"
     }
  }
+
+/*
+ * Determine correct Intel SGX devices to mount for Docker
+ * Returns in the format of --device=<DEVICE1> --device=<DEVICE2>...
+ *   Note: This is really only necessary as Ubuntu 20.04 has SGX 
+ *   driver 1.41 and Ubuntu 18.04 has an older version
+ *
+ * @param os_type     Host Operating System Distribution (e.g. Ubuntu)
+ * @param os_version  Host Operating System Version (e.g. 20.04)
+ */
+def getDockerSGXDevices(String os_type, String os_version) {
+    def devices = []
+    if ( os_type.equalsIgnoreCase('ubuntu') && os_version.equals('20.04') ) {
+        devices.add('/dev/sgx/provision')
+        devices.add('/dev/sgx/enclave')
+    }
+    else if ( os_type.equalsIgnoreCase('ubuntu') && os_version.equals('18.04') ) {
+        devices.add('/dev/sgx')
+    }
+    else {
+        error("getDockerSGXDevices(): Unknown OS (${os_type}) or version (${os_version})")
+    }
+    String returnDevices = ""
+    for (device in devices) {
+        if ( fileExists("${device}") ) {
+            returnDevices += " --device=${device}:${device} "
+        }
+    }
+    return returnDevices
+}
+
+/**
+ * Returns the Ubuntu release version (E.g. "18.04")
+ */
+def getUbuntuReleaseVer() {
+    sh(
+        returnStdout: true,
+        script: 'lsb_release -rs'
+    ).trim()
+}
diff --git a/.jenkins/library/vars/tests.groovy b/.jenkins/library/vars/tests.groovy
@@ -98,11 +98,13 @@ def ACCContainerTest(String label, String version, List extra_cmake_args = []) {
                 cleanWs()
                 checkout scm
                 def cmakeArgs = helpers.CmakeArgs("RelWithDebInfo","OFF","ON","-DLVI_MITIGATION_BINDIR=/usr/local/lvi-mitigation/bin",extra_cmake_args.join(' '))
+                def devices = helpers.getDockerSGXDevices("ubuntu", helpers.getUbuntuReleaseVer())
+                println("${label} running Docker container with ${devices}")
                 def task = """
                            ${helpers.ninjaBuildCommand(cmakeArgs)}
                            ${helpers.TestCommand()}
                            """
-                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE --device /dev/sgx:/dev/sgx --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
+                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE ${devices} --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
             }
         }
     }
@@ -115,6 +117,8 @@ def ACCPackageTest(String label, String version, List extra_cmake_args = []) {
                 cleanWs()
                 checkout scm
                 def cmakeArgs = helpers.CmakeArgs("RelWithDebInfo","OFF","ON","-DLVI_MITIGATION_BINDIR=/usr/local/lvi-mitigation/bin",extra_cmake_args.join(' '))
+                def devices = helpers.getDockerSGXDevices("ubuntu", helpers.getUbuntuReleaseVer())
+                println("${label} running Docker container with ${devices}")
                 common.ContainerTasks(
                     "oetools-${version}:${params.DOCKER_TAG}",
                     globalvars.COMPILER,
@@ -128,7 +132,7 @@ def ACCPackageTest(String label, String version, List extra_cmake_args = []) {
                     ),
                     helpers.TestSamplesCommand()
                     ],
-                    "--cap-add=SYS_PTRACE --device /dev/sgx:/dev/sgx --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket"
+                    "--cap-add=SYS_PTRACE ${devices} --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket"
                 )
             }
         }
@@ -145,6 +149,8 @@ def ACCHostVerificationTest(String version, String build_type) {
                 cleanWs()
                 checkout scm
                 def cmakeArgs = "-G Ninja -DCMAKE_BUILD_TYPE=${build_type} -Wdev"
+                def devices = helpers.getDockerSGXDevices("ubuntu", helpers.getUbuntuReleaseVer())
+                println("ACC-1804 running Docker container with ${devices}")
                 println("Generating certificates and reports ...")
                 def task = """
                            ${helpers.ninjaBuildCommand(cmakeArgs)}
@@ -161,7 +167,7 @@ def ACCHostVerificationTest(String version, String build_type) {
                            ../../../output/bin/oeutil gen --format sgx_ecdsa --quote-proc out --verify
                            popd
                            """
-                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE --device /dev/sgx:/dev/sgx --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
+                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE ${devices} --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
 
                 def ec_cert_created = fileExists 'build/tests/host_verify/host/sgx_cert_ec.der'
                 def rsa_cert_created = fileExists 'build/tests/host_verify/host/sgx_cert_rsa.der'
@@ -245,6 +251,8 @@ def ACCHostVerificationPackageTest(String version, String build_type) {
                 cleanWs()
                 checkout scm
                 def cmakeArgs = "-G Ninja -DCMAKE_BUILD_TYPE=${build_type} -Wdev"
+                def devices = helpers.getDockerSGXDevices("ubuntu", helpers.getUbuntuReleaseVer())
+                println("ACC-1804 running Docker container with ${devices}")
                 println("Generating certificates and reports ...")
                 def task = """
                            ${helpers.ninjaBuildCommand(cmakeArgs)}
@@ -261,7 +269,7 @@ def ACCHostVerificationPackageTest(String version, String build_type) {
                            ../../../output/bin/oeutil gen --format sgx_ecdsa --quote-proc out --verify
                            popd
                            """
-                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE --device /dev/sgx:/dev/sgx --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
+                common.ContainerRun("oetools-${version}:${params.DOCKER_TAG}", "clang-10", task, "--cap-add=SYS_PTRACE ${devices} --volume /var/run/aesmd/aesm.socket:/var/run/aesmd/aesm.socket")
 
                 def ec_cert_created = fileExists 'build/tests/host_verify/host/sgx_cert_ec.der'
                 def rsa_cert_created = fileExists 'build/tests/host_verify/host/sgx_cert_rsa.der'

diff --git a/scripts/ansible/roles/linux/intel/tasks/driver-validation.yml b/scripts/ansible/roles/linux/intel/tasks/driver-validation.yml
@@ -10,7 +10,7 @@
   include_vars:
     file: "{{ ansible_distribution | lower }}/{{ ansible_distribution_release | lower }}.yml"
 
-- name: Load default driver
-  modprobe:
-    name: intel_sgx
-    state: present
+- name: Check default driver files
+  stat:
+    path: "{{ item }}"
+  loop: "{{ intel_dcap_driver_files }}"