azure-pipelines.yml

jobs:
- job: BiDiScan
  pool:
    vmImage: 'ubuntu-18.04'

  steps:
  - checkout: self
    fetchDepth: 32

  - script: |
      git fetch origin 8.0
      CHANGED_FILES=$(git diff --name-only --relative --diff-filter AMR origin/8.0 -- . | tr '\n' ' ')

      if [ -z "${CHANGED_FILES}" ]; then
          echo --- No changed files
      else
          python $(Build.SourcesDirectory)/scripts/find_unicode_control.py -p bidi -v ${CHANGED_FILES}
      fi

- job:
  timeoutInMinutes: 240
  pool:
    vmImage: $(imageName)

  variables:
    UBUNTU_CODE_NAME: bionic
    BOOST_VERSION: boost_1_73_0
    BOOST_DIR: $(Pipeline.Workspace)/boost
    USE_CCACHE: 1
    CCACHE_DIR: $(Pipeline.Workspace)/ccache
    CCACHE_COMPRESS: 1
    CCACHE_COMPRESSLEVEL: 9
    CCACHE_CPP2: 1
    CCACHE_MAXSIZE: 2G
    OS_NAME: $(Agent.OS)

  strategy:
    matrix:
      macOS 10.14 Release:
        imageName: 'macOS-10.14'
        Compiler: clang
        BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        macOS 10.14 Debug:
          imageName: 'macOS-10.14'
          Compiler: clang
          BuildType: Debug

      # clang-5 and newer compilers
      Ubuntu Bionic clang-12 Release:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 12
        BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-12 Release INVERTED=ON:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 12
          BuildType: RelWithDebInfo
          Inverted: ON

      Ubuntu Bionic clang-12 Debug INVERTED=ON:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 12
        BuildType: Debug
        Inverted: ON

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-12 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 12
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-11 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 11
          BuildType: RelWithDebInfo

      Ubuntu Bionic clang-11 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 11
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-10 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 10
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-10 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 10
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-9 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 9
          BuildType: RelWithDebInfo

      Ubuntu Bionic clang-9 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 9
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-8 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 8
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-8 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 8
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-7 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 7
          BuildType: RelWithDebInfo

      Ubuntu Bionic clang-7 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 7
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-6 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 6.0
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-6 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 6.0
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic clang-5 Release:
          imageName: 'ubuntu-18.04'
          Compiler: clang
          CompilerVer: 5.0
          BuildType: RelWithDebInfo

      Ubuntu Bionic clang-5 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: clang
        CompilerVer: 5.0
        BuildType: Debug

      # gcc-5 and newer compilers
      Ubuntu Bionic gcc-11 Release:
        imageName: 'ubuntu-18.04'
        Compiler: gcc
        CompilerVer: 11
        BuildType: RelWithDebInfo

      Ubuntu Bionic gcc-11 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: gcc
        CompilerVer: 11
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-10 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 10
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-10 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 10
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-9 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 9
          BuildType: RelWithDebInfo

      Ubuntu Bionic gcc-9 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: gcc
        CompilerVer: 9
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-8 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 8
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-8 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 8
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-7 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 7
          BuildType: RelWithDebInfo

      Ubuntu Bionic gcc-7 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: gcc
        CompilerVer: 7
        BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-6 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 6
          BuildType: RelWithDebInfo

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-6 Debug:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 6
          BuildType: Debug

      ${{ if ne(variables['Build.Reason'], 'IndividualCI') }}:
        Ubuntu Bionic gcc-5 Release:
          imageName: 'ubuntu-18.04'
          Compiler: gcc
          CompilerVer: 5
          BuildType: RelWithDebInfo

      Ubuntu Bionic gcc-5 Debug:
        imageName: 'ubuntu-18.04'
        Compiler: gcc
        CompilerVer: 5
        BuildType: Debug


  steps:
  - script: |
      uname -r
      df -Th
      ls -l ..
      pwd
      ls -l
      if [[ "$OS_NAME" == "Linux" ]]; then
        CC=$(Compiler)-$(CompilerVer)
        if [[ "$(Compiler)" == "clang" ]]; then
          CXX=clang++-$(CompilerVer)
          PACKAGES="$CC $PACKAGES"
        else
          CXX=g++-$(CompilerVer)
          PACKAGES="$CXX $PACKAGES"
        fi
      fi

      echo CC=$CC CXX=$CXX BuildType=$(BuildType) Ubuntu=$UBUNTU_CODE_NAME OS_NAME=$OS_NAME
      echo --- Configure required LLVM and Ubuntu Toolchain repositories
      if [[ "$OS_NAME" == "Linux" ]] && [[ "$CC" == "clang"* ]]; then
        PACKAGES="llvm-$(CompilerVer)-dev $PACKAGES"
        curl -sSL "http://apt.llvm.org/llvm-snapshot.gpg.key" | sudo -E apt-key add -
        echo "deb http://apt.llvm.org/$UBUNTU_CODE_NAME/ llvm-toolchain-$UBUNTU_CODE_NAME-$(CompilerVer) main" | sudo tee -a /etc/apt/sources.list > /dev/null
      fi

      echo --- Update list of packages and download dependencies
      if [[ "$OS_NAME" == "Linux" ]]; then
        sudo -E apt-add-repository -y "ppa:ubuntu-toolchain-r/test"
        sudo -E apt-get -yq update >> ~/apt-get-update.log 2>&1

        sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install $PACKAGES cmake cmake-curses-gui ccache bison libncurses5-dev libaio-dev libmecab-dev libnuma-dev liblzma-dev libssl-dev libreadline-dev libpam-dev libcurl4-openssl-dev libldap2-dev || exit 1;
        if [[ "$(Inverted)" != "ON" ]]; then
          sudo -E apt-get -yq --no-install-suggests --no-install-recommends --allow-unauthenticated install libevent-dev libeditline-dev liblz4-dev libre2-dev protobuf-compiler libprotobuf-dev libprotoc-dev libicu-dev || exit 1;
        fi
      else
         brew update
         brew install ccache protobuf lz4 re2 rapidjson
      fi

      UPDATE_TIME=$SECONDS
      echo --- Packages updated in $UPDATE_TIME seconds.

      $CC -v
      $CXX -v
      CCACHE_BIN=$(which ccache)
      echo ccache=$CCACHE_BIN
      ccache --version
      ccache -p
      ccache --zero-stats
      df -Th

      COMPILER_VER=$(CompilerVer)
      echo '##vso[task.setvariable variable=CompilerMajorVer]'${COMPILER_VER%.*}
      echo '##vso[task.setvariable variable=CC]'$CC
      echo '##vso[task.setvariable variable=CXX]'$CXX
      echo '##vso[task.setvariable variable=CCACHE_BIN]'$CCACHE_BIN
      echo '##vso[task.setvariable variable=UPDATE_TIME]'$UPDATE_TIME
      echo '##vso[task.prependpath]/usr/lib/ccache'

    displayName: '*** Install Build Dependencies'

  - task: Cache@2
    continueOnError: true
    inputs:
      key: ccache | $(Agent.OS)-$(Compiler)-$(CompilerMajorVer)-$(BuildType) | $(Build.SourceVersion)
      restoreKeys: ccache | $(Agent.OS)-$(Compiler)-$(CompilerMajorVer)-$(BuildType)
      path: $(CCACHE_DIR)
    displayName: '*** Download/upload ccached files'

  - task: Cache@2
    continueOnError: true
    inputs:
      key: $(BOOST_VERSION)
      restoreKeys: $(BOOST_VERSION)
      path: $(BOOST_DIR)
    displayName: '*** Download/upload $(BOOST_VERSION) libraries'

  - checkout: self
    fetchDepth: 32

  - script: |
      df -Th
      git submodule sync
      git submodule update --init --force --depth=256
    displayName: '*** Update git submodules'

  - script: |
      df -Th
      echo --- Set cmake parameters
      COMPILE_OPT+=(
        -DCMAKE_C_FLAGS_DEBUG=-g1
        -DCMAKE_CXX_FLAGS_DEBUG=-g1
        '-DCMAKE_C_FLAGS_RELWITHDEBINFO=-O2 -g1 -DNDEBUG'
        '-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=-O2 -g1 -DNDEBUG'
      )

      if [[ "$CC" =~ clang-(5.0|6.0|7|8)$ ]]; then
        COMPILE_OPT+=(
          '-DCMAKE_C_FLAGS=-isystem /usr/include/c++/9 -isystem /usr/include'
          '-DCMAKE_CXX_FLAGS=-isystem /usr/include/c++/9 -isystem /usr/include'
        )
      fi

      CMAKE_OPT="
        -DCMAKE_BUILD_TYPE=$(BuildType)
        -DBUILD_CONFIG=mysql_release
        -DENABLE_DOWNLOADS=1
        -DDOWNLOAD_BOOST=1
        -DWITH_BOOST=$(BOOST_DIR)
        -DCMAKE_C_COMPILER_LAUNCHER=$CCACHE_BIN
        -DCMAKE_CXX_COMPILER_LAUNCHER=$CCACHE_BIN
        -DWITH_KEYRING_VAULT=ON
        -DWITH_KEYRING_VAULT_TEST=ON
        -DWITH_PAM=ON
      "

      if [[ "$OS_NAME" == "Darwin" ]]; then
        CMAKE_OPT+="
          -DMYSQL_MAINTAINER_MODE=OFF
          -DWITH_ROCKSDB=OFF
          -DWITH_TOKUDB=OFF
          -DWITH_PROTOBUF=system
          -DWITH_SYSTEM_LIBS=ON
          -DWITH_ICU=/usr/local/opt/icu4c
        "
      else
        CMAKE_OPT+="
          -DMYSQL_MAINTAINER_MODE=ON
          -DWITH_ROCKSDB=ON
          -DWITH_TOKUDB=ON
          -DWITH_CURL=system
          -DWITH_MECAB=system
          -DWITH_RAPIDJSON=bundled
          -DWITH_SSL=system
          -DWITH_LIBEVENT=bundled
        "
        if [[ "$(Inverted)" != "ON" ]]; then
          CMAKE_OPT+="
            -DWITH_READLINE=system
            -DWITH_ICU=system
            -DWITH_LZ4=system
            -DWITH_PROTOBUF=system
            -DWITH_ZLIB=system
            -DWITH_NUMA=ON
          "
        else
          CMAKE_OPT+="
            -DWITH_EDITLINE=bundled
            -DWITH_ICU=bundled
            -DWITH_LZ4=bundled
            -DWITH_PROTOBUF=bundled
            -DWITH_ZLIB=bundled
            -DWITH_NUMA=OFF
            -DWITH_ARCHIVE_STORAGE_ENGINE=OFF
            -DWITH_BLACKHOLE_STORAGE_ENGINE=OFF
            -DWITH_EXAMPLE_STORAGE_ENGINE=ON
            -DWITH_FEDERATED_STORAGE_ENGINE=OFF
            -DWITHOUT_PERFSCHEMA_STORAGE_ENGINE=ON
            -DWITH_INNODB_MEMCACHED=ON
          "
        fi
      fi

      echo --- CMAKE_OPT=\"$CMAKE_OPT\"
      echo --- COMPILE_OPT=\"${COMPILE_OPT[@]}\"
      mkdir bin; cd bin
      CC=$CC CXX=$CXX cmake .. $CMAKE_OPT "${COMPILE_OPT[@]}" || exit 1
      rm -f $(BOOST_DIR)/$(BOOST_VERSION).tar.gz

      CMAKE_TIME=$SECONDS
      echo --- CMake took $CMAKE_TIME seconds. Packages updated in $UPDATE_TIME seconds.

      echo '##vso[task.setvariable variable=CMAKE_TIME]'$CMAKE_TIME

    displayName: '*** CC=$(Compiler)-$(CompilerVer) cmake .. -DCMAKE_BUILD_TYPE=$(BuildType)'

  - script: |
      df -Th
      cd bin
      make -j2 || exit 1
      ccache --show-stats

      BUILD_TIME=$SECONDS
      echo --- Total time $(($BUILD_TIME + $UPDATE_TIME + $CMAKE_TIME)) seconds. Build time $BUILD_TIME seconds. CMake took $CMAKE_TIME seconds. Packages updated in $UPDATE_TIME seconds.
      df -Th
      rm -rf *
      df -Th

    displayName: '*** Compile'