ci.yml

name: Main
on:
  pull_request:
    branches: [master]
  push:
    branches: [master]
  release:
    types: [published]
  schedule:
    - cron: '30 20 * * *' # Warning: Timezone dep - 20:00 is 1:00

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  build:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: ubu22-gcc9-clang13
            os: ubuntu-22.04
            compiler: gcc-9
            clang-runtime: '13'
            cling: On
#          - name: ubu22-gcc9-clang14
#            os: ubuntu-22.04
#            compiler: gcc-9
#            clang-runtime: '14'
          - name: ubu22-gcc9-clang16
            os: ubuntu-22.04
            compiler: gcc-9
            clang-runtime: '16'
            cling: Off
    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'
    - name: Save PR Info
      run: |
        mkdir -p ./pr
        echo ${{ github.event.number }} > ./pr/NR
        echo ${{ github.repository }} > ./pr/REPO

        cling_on="${{ matrix.cling }}"
        if [[ "${cling_on^^}" == "ON" ]]; then
          export CLING_HASH=$(git ls-remote https://github.com/root-project/cling.git HEAD| tr '\t' '-')
        else
          export CLING_HASH="Repl"
        fi
        # FIXME: We need something like cling-llvm to bump automatically...
        export LLVM_HASH=$(git ls-remote https://github.com/root-project/llvm-project.git cling-llvm${{ matrix.clang-runtime}} | tr '\t' '-')
        #patch in LLVM_HASH
        echo "CLING_HASH=$CLING_HASH" >> $GITHUB_ENV
        echo "LLVM_HASH=$LLVM_HASH" >> $GITHUB_ENV
    - uses: nelonoel/branch-name@v1.0.1
    - name: Setup compiler on Linux
      if: runner.os == 'Linux'
      run: |
        # https://www.gnu.org/software/bash/manual/html_node/Shell-Parameter-Expansion.html
        vers="${compiler#*-}"
        os_codename="`cat /etc/os-release | grep UBUNTU_CODENAME | cut -d = -f 2`"
        ###sudo apt update
        if [[ "${{ matrix.compiler }}" == *"gcc"* ]]; then
          sudo apt install -y gcc-${vers} g++-${vers} lld
          echo "CC=gcc-${vers}" >> $GITHUB_ENV
          echo "CXX=g++-${vers}" >> $GITHUB_ENV
        else
          if ! sudo apt install -y clang-${vers}; then
            curl https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
            echo "deb https://apt.llvm.org/${os_codename}/ llvm-toolchain-${os_codename}-${vers} main" | sudo tee -a /etc/apt/sources.list
            sudo apt update
            sudo apt install -y clang-${vers}
          fi
          echo "CC=clang-${vers}" >> $GITHUB_ENV
          echo "CXX=clang++-${vers}" >> $GITHUB_ENV
        fi
      env:
        compiler: ${{ matrix.compiler }}
    - name: Install deps on Linux
      if: runner.os == 'Linux'
      run: |
        # Install deps
        ###sudo apt update
        sudo apt install git g++ debhelper devscripts gnupg python3 valgrind
        conda install -y -q -c conda-forge \
          distro \
          pytest
    - name: Restore Cache LLVM/Clang runtime build directory
      uses: actions/cache/restore@v3
      id: cache
      with:
        path: |
          llvm-project
          ${{ matrix.cling=='On' && 'cling' || '' }}
        #key: ...-.x-patch-${{ hashFiles('patches/llvm/*') }}
        #key: ${{ env.CLING_HASH }}-${{ env.LLVM_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x
        key: ${{ env.CLING_HASH }}-${{ runner.os }}-${{ matrix.os }}-${{ matrix.compiler }}-clang-${{ matrix.clang-runtime }}.x-patch-${{ hashFiles(format('patches/llvm/clang{0}-*.patch', matrix.clang-runtime)) || 'none' }}
    - name: Build LLVM/Cling on Linux if the cache is invalid
      if: ${{ runner.os == 'Linux' && steps.cache.outputs.cache-hit != 'true' }}
      run: |
        cling_on="${{ matrix.cling }}"
        if [[ "${cling_on^^}" == "ON" ]]; then
          git clone --depth=1 https://github.com/root-project/cling.git
          git clone --depth=1 -b cling-llvm${{ matrix.clang-runtime }} https://github.com/root-project/llvm-project.git
        else # repl
           git clone --depth=1 -b release/${{ matrix.clang-runtime }}.x https://github.com/llvm/llvm-project.git
        fi
        cd llvm-project
        # Apply patches
        echo "Apply clang${{ matrix.clang-runtime }}-*.patch patches:"
        compgen -G "../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch" > /dev/null && find ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch -printf "%f\n" && git apply ../patches/llvm/clang${{ matrix.clang-runtime }}-*.patch
        # Build
        mkdir build
        cd build
        if [[ "${cling_on^^}" == "ON" ]]; then
          cmake -DLLVM_ENABLE_PROJECTS=clang                  \
                -DLLVM_EXTERNAL_PROJECTS=cling                \
                -DLLVM_EXTERNAL_CLING_SOURCE_DIR=../../cling  \
                -DLLVM_TARGETS_TO_BUILD="host;NVPTX"          \
                -DCMAKE_BUILD_TYPE=Release                    \
                -DLLVM_ENABLE_ASSERTIONS=ON                   \
                -DLLVM_USE_LINKER=lld                         \
                -DCLANG_ENABLE_STATIC_ANALYZER=OFF            \
                -DCLANG_ENABLE_ARCMT=OFF                      \
                -DCLANG_ENABLE_FORMAT=OFF                     \
                -DCLANG_ENABLE_BOOTSTRAP=OFF                  \
                ../llvm
          cmake --build . --target clang --parallel $(nproc --all)
          cmake --build . --target cling --parallel $(nproc --all)
          # Now build gtest.a and gtest_main for CppInterOp to run its tests.
          cmake --build . --target gtest_main --parallel $(nproc --all)
        else
          cmake -DLLVM_ENABLE_PROJECTS=clang                  \
                -DLLVM_TARGETS_TO_BUILD="host;NVPTX"          \
                -DCMAKE_BUILD_TYPE=Release                    \
                -DLLVM_ENABLE_ASSERTIONS=ON                   \
                -DLLVM_USE_LINKER=lld                         \
                -DCLANG_ENABLE_STATIC_ANALYZER=OFF            \
                -DCLANG_ENABLE_ARCMT=OFF                      \
                -DCLANG_ENABLE_FORMAT=OFF                     \
                -DCLANG_ENABLE_BOOTSTRAP=OFF                  \
                ../llvm
          cmake --build . --target clang clang-repl --parallel $(nproc --all)
        fi
        cd ../../
    - name: Save Cache LLVM/Clang runtime build directory
      uses: actions/cache/save@v3
      if: ${{ steps.cache.outputs.cache-hit != 'true' }}
      with:
        path: |
          llvm-project
          ${{ matrix.cling=='On' && 'cling' || '' }}
        key: ${{ steps.cache.outputs.cache-primary-key }}
    - name: Build and Install CppInterOp on Linux
      if: runner.os == 'Linux'
      run: |
        # Build CppInterOp next to cling and llvm-project.
        LLVM_DIR="$(realpath llvm-project)"
        LLVM_BUILD_DIR="$(realpath llvm-project/build)"
        cling_on="${{ matrix.cling }}"
        if [[ "${cling_on^^}" == "ON" ]]; then
          CLING_DIR="$(realpath cling)"
          CLING_BUILD_DIR="$(realpath cling/build)"
          CPLUS_INCLUDE_PATH="${CLING_DIR}/tools/cling/include:${CLING_BUILD_DIR}/include:${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include"
        else # repl
          CPLUS_INCLUDE_PATH="${LLVM_DIR}/llvm/include:${LLVM_DIR}/clang/include:${LLVM_BUILD_DIR}/include:${LLVM_BUILD_DIR}/tools/clang/include"
        fi
        git clone https://github.com/compiler-research/CppInterOp.git
        export CPPINTEROP_DIR=$PWD/cppyy-backend/python/cppyy_backend/
        cd CppInterOp
        mkdir build
        cd build
        export CPPINTEROP_BUILD_DIR=$PWD
        cling_on="${{ matrix.cling }}"
        if [[ "${cling_on^^}" == "ON" ]]; then
          cmake -DCMAKE_BUILD_TYPE=Release              \
                -DUSE_CLING=ON                          \
                -DUSE_REPL=OFF                          \
                -DCling_DIR=$LLVM_BUILD_DIR/tools/cling \
                -DLLVM_DIR=$LLVM_BUILD_DIR              \
                -DLLVM_USE_LINKER=lld                   \
                -DBUILD_SHARED_LIBS=ON                  \
                -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR  \
                ../
        else
          cmake -DCMAKE_BUILD_TYPE=Release  \
                -DUSE_CLING=OFF             \
                -DUSE_REPL=ON               \
                -DLLVM_DIR=$LLVM_BUILD_DIR  \
                -DLLVM_USE_LINKER=lld       \
                -DBUILD_SHARED_LIBS=ON      \
                -DCMAKE_INSTALL_PREFIX=$CPPINTEROP_DIR \
                ../
        fi
        cmake --build . --parallel $(nproc --all)
        cd ../..
        # We need CPPINTEROP_DIR, LLVM_BUILD_DIR and CPLUS_INCLUDE_PATH later
        echo "CPPINTEROP_DIR=$CPPINTEROP_DIR" >> $GITHUB_ENV
        echo "CPPINTEROP_BUILD_DIR=$CPPINTEROP_BUILD_DIR" >> $GITHUB_ENV
        echo "LLVM_BUILD_DIR=$LLVM_BUILD_DIR" >> $GITHUB_ENV
        echo "CPLUS_INCLUDE_PATH=$CPLUS_INCLUDE_PATH" >> $GITHUB_ENV
    - name: Build and Install cppyy-backend on Linux
      if: runner.os == 'Linux'
      run: |
        # Install cppyy-backend
        git clone https://github.com/compiler-research/cppyy-backend.git
        cd cppyy-backend
        mkdir -p python/cppyy_backend/lib build && cd build
        (cd $CPPINTEROP_BUILD_DIR && cmake --build . --target install --parallel $(nproc --all))
        cmake -DCppInterOp_DIR=$CPPINTEROP_DIR ..
        cmake --build . --parallel $(nproc --all)
        cp libcppyy-backend.so ../python/cppyy_backend/lib/
        #
        cd ../python
        export CB_PYTHON_DIR=$PWD
        cd ../..
        # We need CB_PYTHON_DIR later
        echo "CB_PYTHON_DIR=$CB_PYTHON_DIR" >> $GITHUB_ENV
    - name: Install CPyCppyy on Linux
      if: runner.os == 'Linux'
      run: |
        # Setup virtual environment
        python3 -m venv .venv
        source .venv/bin/activate
        # Install CPyCppyy
        git clone https://github.com/compiler-research/CPyCppyy.git
        cd CPyCppyy
        mkdir build && cd build
        cmake ..
        cmake --build . --parallel $(nproc --all)
        #
        export CPYCPPYY_DIR=$PWD
        cd ../..
        # We need CPYCPPYY_DIR later
        echo "CPYCPPYY_DIR=$CPYCPPYY_DIR" >> $GITHUB_ENV
    - name: Install cppyy on Linux
      if: runner.os == 'Linux'
      run: |
        # source virtual environment
        source .venv/bin/activate
        # Install cppyy
        python -m pip install --upgrade . --no-deps
    - name: Run cppyy on Linux
      if: runner.os == 'Linux'
      run: |
        # Run cppyy
        source .venv/bin/activate
        export PYTHONPATH=$PYTHONPATH:$CPYCPPYY_DIR:$CB_PYTHON_DIR
        python -c "import cppyy"
        # We need PYTHONPATH later
        echo "PYTHONPATH=$PYTHONPATH" >> $GITHUB_ENV
    - name: Run the tests on Linux
      if: runner.os == 'Linux'
      run: |
        # Run the tests
        source .venv/bin/activate
        cd test
        echo ::group::Prepare For Testing
        make all
        python -m pip install --upgrade pip
        python -m pip install pytest
        python -m pip install pytest-xdist
        python -m pip install numba
        echo ::endgroup::
        echo ::group::Run complete test suite
        python -m pytest -sv | tee complete_testrun.log 2>&1
        echo ::group::Crashing Test Logs
        # See if we don't have a crash that went away
        # Comment out all xfails but the ones that have a run=False condition.
        find . -name "*.py" -exec sed -i '/run=False/!s/^ *@mark.xfail\(.*\)/#&/' {} \;
        python -m pytest -n 1 -m "xfail" --runxfail -sv --max-worker-restart 512 | tee test_crashed.log 2>&1 || true
        git checkout .
        echo ::endgroup::
        echo ::group::XFAIL Test Logs
        # Rewrite all xfails that have a run clause to skipif. This way we will
        # avoid conditionally crashing xfails
        find . -name "*.py" -exec sed -i -E 's/(^ *)@mark.xfail\(run=(.*)/\1@mark.skipif(condition=not \2/g' {} \;
        # See if we don't have an xfail that went away
        python -m pytest --runxfail -sv  | tee test_xfailed.log 2>&1 || true
        git checkout .
        echo ::endgroup::
        echo ::group::Passing Test Logs

        # Run the rest of the non-crashing tests.
        declare -i RETCODE=0

        set -o pipefail
        if [[ "${{ matrix.clang-runtime }}" == "17" || "${{ matrix.clang-runtime }}" == "18" ]] && [[ "${{ matrix.os }}" != "macos-14" ]]; then
          echo "Valgrind reports true for clang-runtime 17 or 18, due to memory leaks with LLVM"
          valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v || true
        else
          echo "Running valgrind on passing tests"
          valgrind --error-exitcode=1 --suppressions=../etc/valgrind-cppyy-cling.supp python -m pytest -m "not xfail" -v
        fi
        export RETCODE=+$?
        echo ::endgroup::

        RETCODE=+$?

        echo "Complete Test Suite Summary: \n"
        tail -n1 complete_testrun.log
        echo "Crashing Summary: \n"
        tail -n1 test_crashed.log
        echo "XFAIL Summary:"
        tail -n1 test_xfailed.log
        echo "Return Code: ${RETCODE}"
        exit $RETCODE
    - name: Setup tmate session
      if: ${{ failure() }}
      uses: mxschmitt/action-tmate@v3
      # When debugging increase to a suitable value!
      timeout-minutes: ${{ github.event.pull_request && 1 || 20 }}