Added support for front panel port prefix regex

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -22,9 +22,9 @@ resources:
   repositories:
   - repository: buildimage
     type: github
-    name: Azure/sonic-buildimage
+    name: sonic-net/sonic-buildimage
     ref: master
-    endpoint: build
+    endpoint: sonic-net
 
 pool: sonicbld
 
@@ -38,7 +38,6 @@ parameters:
   - centec
   - centec-arm64
   - generic
-  - innovium
   - marvell-armhf
   - mellanox
 

.azure-pipelines/azure-pipelines-build.yml

@@ -16,6 +16,9 @@ parameters:
 - name: 'jobGroups'
   type: object
   default: ''
+- name: 'qemuOrCrossBuild'
+  type: boolean
+  default: false
 
 jobs:
 - template: azure-pipelines-image-template.yml
@@ -30,18 +33,23 @@ jobs:
       BUILD_OPTIONS: ${{ parameters.buildOptions }}
       DOCKER_DATA_ROOT_FOR_MULTIARCH: /data/march/docker
       dbg_image: no
+      asan_image: no
       swi_image: no
       raw_image: no
       docker_syncd_rpc_image: no
       syncd_rpc_image: no
       platform_rpc: no
+      ${{ if parameters.qemuOrCrossBuild }}:
+        SONIC_SLAVE_DOCKER_DRIVER: 'vfs'
+        CACHE_MODE: 'none'
     ${{ if ne(parameters.jobGroups, '') }}:
       jobGroups: ${{ parameters.jobGroups }}
     ${{ if eq(parameters.jobGroups, '') }}:
       jobGroups:
         - name: vs
           variables:
             dbg_image: yes
+            asan_image: yes
 
         - name: barefoot
           variables:
@@ -50,6 +58,7 @@ jobs:
             swi_image: yes
 
         - name: broadcom
+          timeoutInMinutes: 1440
           variables:
             dbg_image: yes
             swi_image: yes
@@ -64,7 +73,8 @@ jobs:
             platform_rpc: centec
 
         - name: centec-arm64
-          pool: sonicbld-arm64
+          ${{ if not(parameters.qemuOrCrossBuild) }}:
+            pool: sonicbld-arm64
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: arm64
@@ -78,7 +88,8 @@ jobs:
             dbg_image: yes
 
         - name: marvell-armhf
-          pool: sonicbld-armhf
+          ${{ if not(parameters.qemuOrCrossBuild) }}:
+            pool: sonicbld-armhf
           timeoutInMinutes: 2880
           variables:
             PLATFORM_ARCH: armhf
@@ -97,14 +108,20 @@ jobs:
             platform_rpc: nephos
 
     buildSteps:
+      - template: template-skipvstest.yml
       - bash: |
           set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
               make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz
               mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
             fi
+            if [ $(asan_image) == yes ]; then
+              make $BUILD_OPTIONS ENABLE_ASAN=y target/docker-sonic-vs.gz
+              mv target/docker-sonic-vs.gz target/docker-sonic-vs-asan.gz
+            fi
             make $BUILD_OPTIONS target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+            make $BUILD_OPTIONS target/docker-ptf-sai.gz
             if [ $(Build.Reason) != 'PullRequest' ];then
               gzip -kd target/sonic-vs.img.gz
               SONIC_RUN_CMDS="qemu-img convert target/sonic-vs.img -O vhdx -o subformat=dynamic target/sonic-vs.vhdx" make sonic-slave-run
@@ -122,7 +139,27 @@ jobs:
               make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).raw
             fi
             if [ $(docker_syncd_rpc_image) == yes ]; then
+              # workaround for issue in rules/sairedis.dep, git ls-files will list un-exist files for cache
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-rpc.gz
+              pushd ./src/sonic-sairedis/SAI
+              git stash
+              popd
+              if [ $(GROUP_NAME) == broadcom ]; then
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/docker-syncd-$(platform_rpc)-dnx-rpc.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd 
+                make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
+              fi
+              if [ $(GROUP_NAME) == barefoot ]; then
+                make $BUILD_OPTIONS SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-bfn.gz
+                pushd ./src/sonic-sairedis/SAI
+                git stash
+                popd
+              fi
             fi
             if [ $(syncd_rpc_image) == yes ]; then
               make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin
@@ -131,3 +168,4 @@ jobs:
             make $BUILD_OPTIONS target/sonic-$(GROUP_NAME).bin
           fi
         displayName: "Build sonic image"
+      - template: check-dirty-version.yml

.azure-pipelines/azure-pipelines-download-certificate.yml

@@ -0,0 +1,33 @@
+parameters:
+- name: connectionName
+  type: string
+  default: sonic-dev-connection
+- name: kevaultName
+  type: string
+  default: sonic-kv
+- name: certificateName
+  type: string
+  default: sonic-secure-boot
+
+steps:
+- task: AzureKeyVault@2
+  inputs:
+    connectedServiceName: ${{ parameters.connectionName }}
+    keyVaultName: ${{ parameters.kevaultName }}
+    secretsFilter: ${{ parameters.certificateName }}
+
+- script: |
+    set -e
+    TMP_FILE=$(mktemp)
+    echo "$CERTIFICATE" | base64 -d > $TMP_FILE
+    sudo mkdir -p /etc/certificates
+    mkdir -p $(Build.StagingDirectory)/target
+    # Save the public key
+    openssl pkcs12 -in $TMP_FILE -clcerts --nokeys -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN CERTIFICATE\)/\1/" > $(SIGNING_CERT)
+    # Save the private key
+    openssl pkcs12 -in $TMP_FILE -nocerts -nodes -passin pass: | sed -z -e "s/.*\(-----BEGIN PRIVATE KEY\)/\1/" | sudo tee $(SIGNING_KEY) 1>/dev/null
+    ls -lt $(SIGNING_CERT) $(SIGNING_KEY)
+    rm $TMP_FILE
+  env:
+    CERTIFICATE: $(${{ parameters.certificateName }})
+  displayName: "Save certificate"

.azure-pipelines/azure-pipelines-image-template.yml

@@ -36,7 +36,8 @@ jobs:
         displayName: "Set cache options"
       - checkout: self
         submodules: recursive
-        condition: eq(variables.SKIP_CHECKOUT, '') 
+        fetchDepth: 0
+        condition: and(succeeded(), eq(variables.SKIP_CHECKOUT, ''))
         displayName: 'Checkout code'
       - script: |
           BRANCH_NAME=$(Build.SourceBranchName)
@@ -48,16 +49,23 @@ jobs:
           ENABLE_DOCKER_BASE_PULL=y make PLATFORM=$(PLATFORM_AZP) PLATFORM_ARCH=$(PLATFORM_ARCH) $(BUILD_OPTIONS) configure
         displayName: 'Make configure'
     postSteps:
-      - script: cp target -r $(Build.ArtifactStagingDirectory)/
+      - script: |
+          mkdir -p $(Build.ArtifactStagingDirectory)/target
+          mv target/* $(Build.ArtifactStagingDirectory)/target/
         displayName: Copy Artifacts
         condition: always()
       - publish:  $(Build.ArtifactStagingDirectory)
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
         displayName: "Archive sonic image"
       - publish:  $(Build.ArtifactStagingDirectory)
-        condition: failed()
+        condition: or(failed(), canceled())
         artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)$(System.JobAttempt)'
         displayName: "Archive failed sonic image"
+      - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
+        - template: trigger-publish-artifacts-build.yml
+          parameters:
+            artifactName: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+            publishPrefix: '$(Build.DefinitionName)/$(Build.SourceBranchName)/$(GROUP_NAME)'
       - ${{ parameters.postSteps }}
       - template: cleanup.yml
     jobGroups: ${{ parameters.jobGroups }}

.azure-pipelines/azure-pipelines-repd-build-variables.yml

@@ -1,5 +1,7 @@
 variables:
-  ${{ if and(ge(variables['Build.SourceBranchName'], '202012'), le(variables['Build.SourceBranchName'], '299999')) }}:
+  ${{ if eq(variables['Build.Reason'],'PullRequest') }}:
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=$([[ "$(System.PullRequest.TargetBranch)" =~ ^20[2-9][0-9]{3}$ ]] && echo deb,py2,py3,web,git,docker)'
+  ${{ elseif ne(variables['Build.SourceBranchName'],'master') }}:
     VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
-  ${{ if or(lt(variables['Build.SourceBranchName'], '202012'), gt(variables['Build.SourceBranchName'], '299999')) }}:
+  ${{ else }}:
     VERSION_CONTROL_OPTIONS: ''

.azure-pipelines/build-commonlib.yml

@@ -11,9 +11,9 @@ resources:
   repositories:
   - repository: buildimage
     type: github
-    name: Azure/sonic-buildimage
+    name: sonic-net/sonic-buildimage
     ref: master
-    endpoint: build
+    endpoint: sonic-net
 
 jobs:
 - template: .azure-pipelines/template-commonlib.yml@buildimage

.azure-pipelines/build-template.yml

@@ -92,6 +92,7 @@ jobs:
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
+        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-ptf-sai.gz
       else
         if [ ${{ parameters.dbg_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \
@@ -105,6 +106,22 @@ jobs:
         fi
         if [ ${{ parameters.sync_rpc_image }} == true ]; then
           make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
+          # workaround for issue in rules/sairedis.dep,  git ls-files will list un-exist files for cache
+          pushd ./src/sonic-sairedis/SAI
+          git stash
+          popd 
+          if [ ${{ parameters.platform }} == broadcom ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y SAITHRIFT_V2=y target/docker-saiserverv2-brcm.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
+          if [ ${{ parameters.platform }} == barefoot ]; then
+            make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) SAITHRIFT_V2=y  ENABLE_SYNCD_RPC=y  target/docker-saiserverv2-bfn.gz
+            pushd ./src/sonic-sairedis/SAI
+            git stash
+            popd 
+          fi
         fi
 
         make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin

.azure-pipelines/check-dirty-version.yml

@@ -0,0 +1,16 @@
+steps:
+- script: |
+    . functions.sh
+    SONIC_VERSION=$(sonic_get_version)
+    echo "SONIC_VERSION=$SONIC_VERSION"
+    if [[ "$SONIC_VERSION" == *dirty* ]]; then
+      # Print the detail dirty info
+      git status --untracked-files=no -s --ignore-submodules
+
+      # Exit with error, if it is a PR build
+      if [ "$(Build.Reason)" == "PullRequest" ]; then
+        echo "Build failed for the dirty version: $SONIC_VERSION" 1>&2
+        exit 1
+      fi
+    fi
+  displayName: "Check the dirty version"

.azure-pipelines/docker-sonic-mgmt.yml

@@ -32,17 +32,18 @@ stages:
 - stage: Build
   jobs:
   - job: Build
-    pool: sonictest
+    pool: sonicbld
     timeoutInMinutes: 360
     steps:
     - template: cleanup.yml
     - checkout: self
       clean: true
-      submodules: recursive
     - bash: |
         set -xe
-        make configure PLATFORM=generic
-        make target/docker-sonic-mgmt.gz
+        git submodule update --init --recursive -- src/sonic-platform-daemons src/sonic-genl-packet src/sonic-sairedis src/ptf src/sonic-device-data
+
+        make SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y configure PLATFORM=generic
+        make SONIC_BUILD_JOBS=$(nproc) DEFAULT_CONTAINER_REGISTRY=publicmirror.azurecr.io ENABLE_DOCKER_BASE_PULL=y target/docker-sonic-mgmt.gz
         cp target -r $(Build.ArtifactStagingDirectory)/target
         docker load -i target/docker-sonic-mgmt.gz
         docker tag docker-sonic-mgmt $REGISTRY_SERVER/docker-sonic-mgmt:latest