Merged PR 8235253: Enable OneBranch pipelines

This change rewrites our Azure DevOps pipelines to be compatible with OneBranch pipelines. It also adds new scripts to help with building, testing and packaging SymCrypt. These scripts replicate some of the functionality of `scbuild` but are also compatible with Linux builds. They can be used directly on the command line by developers, but the OneBranch pipeline also uses them to move as much as possible of the "business logic" of building SymCrypt out of the YAML templates and into Python scripts.

Also includes various reorganization and small fixes.

.artifactignore

@@ -1,4 +1,4 @@
 # Do not try to upload git submodules when publishing ADO artifacts
 
 **/SymCryptDependencies/
-**/jitterentropy-library/
+**/3rdparty/

.gitignore

@@ -69,7 +69,7 @@ release/
 scbuild.log
 
 # Cmake output
-bin/
+bin*/
 CMakeFiles
 CMakeCache.txt
 **/cmake_install.*
@@ -82,3 +82,6 @@ CMakeCache.txt
 
 # Makefiles from CMake
 Makefile
+
+# Local Guardian configuration
+.gdn

.gitmodules

@@ -1,8 +1,8 @@
 [submodule "SymCryptDependencies"]
-	path = SymCryptDependencies
+	path = unittest/SymCryptDependencies
 	url = https://microsoft.visualstudio.com/DefaultCollection/SymCryptDependencies/_git/SymCryptDependencies
 
 [submodule "jitterentropy-library"]
-	path = jitterentropy-library
+	path = 3rdparty/jitterentropy-library
 	url = https://github.com/smuellerDD/jitterentropy-library
 	ignore = untracked

.pipelines/OneBranch.PullRequest.yml

@@ -0,0 +1,143 @@
+#################################################################################
+#                               OneBranch Pipelines                             #
+# This pipeline was created by EasyStart from a sample located at:              #
+#   https://aka.ms/obpipelines/easystart/samples                                #
+# Documentation:  https://aka.ms/obpipelines                                    #
+# Yaml Schema:    https://aka.ms/obpipelines/yaml/schema                        #
+# Retail Tasks:   https://aka.ms/obpipelines/tasks                              #
+# Support:        https://aka.ms/onebranchsup                                   #
+#################################################################################
+
+trigger: none
+
+variables:
+  CDP_DEFINITION_BUILD_COUNT: $[counter('', 0)] # needed for onebranch.pipeline.version task https://aka.ms/obpipelines/versioning
+  LinuxContainerImage: 'onebranch.azurecr.io/linux/ubuntu-2004:latest' # Docker image which is used to build the project https://aka.ms/obpipelines/containers
+  WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2019/vse2022:latest'
+
+resources:
+  repositories: 
+    - repository: templates
+      type: git
+      name: OneBranch.Pipelines/GovernedTemplates
+      ref: refs/heads/main
+
+extends:
+  template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates # https://aka.ms/obpipelines/templates
+  parameters:
+    globalSdl: # https://aka.ms/obpipelines/sdl
+      policheck:
+        break: true # always break the build on policheck issues.
+      binskim:
+        enabled: false # Temporarily disable binskim until we sort out BA2018 errors
+
+    stages:
+    - stage: Build_Windows
+      displayName: Build Windows
+      jobs:
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Debug'
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'X86'
+          config: 'Debug'
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'X86'
+          config: 'Release'
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+          additionalArgs: '--no-asm'
+          identifier: 'NoAsm'
+      - template: .pipelines/templates/build-windows.yml@self
+        parameters:
+          arch: 'X86'
+          config: 'Release'
+          additionalArgs: '--no-asm'
+          identifier: 'NoAsm'
+
+    - stage: Build_Linux
+      displayName: Build Linux
+      jobs:
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Debug'
+          cc: 'gcc'
+          cxx: 'g++'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Sanitize'
+          cc: 'gcc'
+          cxx: 'g++'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+          cc: 'gcc'
+          cxx: 'g++'          
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Debug'
+          cc: 'clang'
+          cxx: 'clang++'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Sanitize'
+          cc: 'clang'
+          cxx: 'clang++'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+          cc: 'clang'
+          cxx: 'clang++'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+          cc: 'gcc'
+          cxx: 'g++'
+          additionalArgs: '--no-asm'
+          identifier: 'NoAsm'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'AMD64'
+          config: 'Release'
+          cc: 'clang'
+          cxx: 'clang++'
+          additionalArgs: '--no-asm'
+          identifier: 'NoAsm'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'X86'
+          config: 'Release'
+          cc: 'gcc'
+          cxx: 'g++'
+          additionalArgs: '--no-asm --no-fips'
+          identifier: 'NoAsm'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'ARM64'
+          config: 'Debug'
+          cc: 'clang'
+          cxx: 'clang++'
+          additionalArgs: '--toolchain=cmake-configs/Toolchain-Clang-ARM64.cmake'
+      - template: .pipelines/templates/build-linux.yml@self
+        parameters:
+          arch: 'ARM64'
+          config: 'Release'
+          cc: 'clang'
+          cxx: 'clang++'
+          additionalArgs: '--toolchain=cmake-configs/Toolchain-Clang-ARM64.cmake'

.pipelines/templates/build-linux.yml

@@ -0,0 +1,142 @@
+parameters:
+- name: arch
+  type: string
+  values:
+  - AMD64
+  - X86
+  - ARM64
+  - ARM
+- name: config
+  type: string
+  values:
+  - Debug
+  - Release
+  - Sanitize
+- name: cc
+  type: string
+  values:
+  - gcc
+  - clang
+- name: cxx
+  type: string
+  values:
+  - g++
+  - clang++
+- name: additionalArgs # Additional arguments to pass to the build script
+  type: string
+  default: ''
+- name: identifier # Additional identifier for job name
+  type: string
+  default: ''
+
+jobs:
+- job: Linux_${{ parameters.arch}}_${{ parameters.cc }}_${{parameters.config}}_${{parameters.identifier}}
+  displayName: Linux ${{ parameters.arch}} ${{ parameters.cc }} ${{parameters.config}} ${{ parameters.identifier }}
+
+  pool:
+    type: linux
+
+  variables:
+    ob_outputDirectory: $(Build.SourcesDirectory)/bin
+    ob_sdl_binskim_break: ${{ eq(parameters.config, 'Release') }} # Only break on binskim failures for Release binaries https://aka.ms/obpipelines/sdl
+
+  steps:
+
+    # TODO: Need to make this match the SymCrypt version (major.minor.patch). We also shouldn't have to update it manually.
+    # https://microsoft.visualstudio.com/DefaultCollection/OS/_workitems/edit/42880140/
+    - task: onebranch.pipeline.version@1
+      displayName: 'Setup BuildNumber'
+      inputs:
+        system: 'RevisionCounter'
+        major: '103'
+        minor: '0'
+        exclude_commit: false
+
+    - ${{ if eq(parameters.cc, 'clang') }}:
+      # Install clang-11 and manually add symlinks so that it can be run via just "clang" rather
+      # than clang-11. This is required because we can't explicitly specify clang-11 for
+      # parameters.cc (it breaks the job name, because job names aren't allowed to have '-' in them)
+      - script: |
+          apt-get update
+          apt-get install -y clang-11
+          ln -s /usr/bin/clang-11 /usr/bin/clang
+          ln -s /usr/bin/clang++-11 /usr/bin/clang++
+        displayName: 'Install clang'
+
+    - ${{ if eq(parameters.arch, 'X86') }}:
+      - script: |
+          apt-get update
+          apt-get install -y gcc-multilib g++-multilib
+        displayName: 'Install x86 headers and libraries'
+
+    # Note: this assumes that the pipeline is always running on an AMD64 machine. When we have
+    # native ARM64 pipelines, we'll need to change this.
+    - ${{ if eq(parameters.arch, 'ARM64') }}:
+      - script: |
+          apt-get update
+          apt-get install -y binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu qemu-user
+        displayName: 'Install arm64 cross-compilation tools'
+
+    - script: |
+        pip install -r scripts/requirements.txt
+      displayName: 'Install Python dependencies'
+
+    - task: PythonScript@0
+      displayName: 'Run build script'
+      inputs:
+        scriptSource: 'filePath'
+        scriptPath: scripts/build.py
+        arguments: 'bin --arch ${{ parameters.arch }} --config ${{ parameters.config }} --cc ${{ parameters.cc }} --cxx ${{ parameters.cxx }} ${{ parameters.additionalArgs }}'
+        workingDirectory: $(Build.SourcesDirectory)
+
+    - ${{ if ne(parameters.arch, 'ARM64') }}:
+      - task: PythonScript@0
+        displayName: 'Run unit tests'
+        inputs:
+          scriptSource: 'filePath'
+          scriptPath: scripts/test.py
+          arguments: 'bin noperftests'
+          workingDirectory: $(Build.SourcesDirectory)
+
+      - ${{ if ne(parameters.config, 'Sanitize') }}:
+        - task: PythonScript@0
+          displayName: 'Run dynamic unit tests'
+          inputs:
+            scriptSource: 'filePath'
+            scriptPath: scripts/test.py
+            arguments: 'bin dynamic:bin/module/generic/libsymcrypt.so noperftests'
+            workingDirectory: $(Build.SourcesDirectory)
+
+    - ${{ if eq(parameters.arch, 'AMD64') }}:
+      - task: PythonScript@0
+        displayName: 'Run unit tests (test YMM save/restore)'
+        inputs:
+          scriptSource: 'filePath'
+          scriptPath: scripts/test.py
+          arguments: '--glibc-disable-ymm bin testSaveYmm'
+          workingDirectory: $(Build.SourcesDirectory)
+
+    - ${{ if eq(parameters.arch, 'ARM64') }}:
+      - task: PythonScript@0
+        displayName: 'Run unit tests'
+        inputs:
+          scriptSource: 'filePath'
+          scriptPath: scripts/test.py
+          arguments: '--emulator qemu-aarch64 --emulator-lib-dir /usr/aarch64-linux-gnu/ bin noperftests +symcrypt -dh -dsa -rsa'
+          workingDirectory: $(Build.SourcesDirectory)
+
+      - task: PythonScript@0
+        displayName: 'Run dynamic unit tests'
+        inputs:
+          scriptSource: 'filePath'
+          scriptPath: scripts/test.py
+          arguments: '--emulator qemu-aarch64 --emulator-lib-dir /usr/aarch64-linux-gnu/ bin dynamic:bin/module/generic/libsymcrypt.so noperftests +symcrypt -dh -dsa -rsa'
+          workingDirectory: $(Build.SourcesDirectory)
+
+    - task: PythonScript@0
+      displayName: 'Package build output'
+      inputs:
+        scriptSource: 'filePath'
+        scriptPath: scripts/package.py
+        arguments: 'bin ${{ parameters.arch }} ${{ parameters.config }} generic bin'
+        workingDirectory: $(Build.SourcesDirectory)