ci.yml

name: "MiNiFi-CPP CI"
on: [push, pull_request, workflow_dispatch]
env:
  DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
    -DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
    -DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON
  SCCACHE_GHA_ENABLE: true
jobs:
  macos_xcode:
    name: "macos-xcode"
    runs-on: macos-13
    timeout-minutes: 180
    env:
      CCACHE_BASEDIR: ${{ GITHUB.WORKSPACE }}
      CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - name: cache restore
        uses: actions/cache/restore@v3
        with:
          path: ${{ env.CCACHE_DIR }}
          key: macos-xcode-ccache-${{github.ref}}-${{github.sha}}
          restore-keys: |
            macos-xcode-ccache-${{github.ref}}-
            macos-xcode-ccache-refs/heads/main-
      - id: install_dependencies
        run: |
          # Skip brew update until https://github.com/actions/setup-python/issues/577 is fixed
          # brew update
          HOMEBREW_NO_AUTO_UPDATE=1 brew install ossp-uuid flex lua ccache sqliteodbc automake autoconf ninja
      - id: setup_env
        name: setup enviroment
        run: |
          echo "PATH=/usr/lib/ccache:/usr/local/opt/ccache/bin:/usr/local/opt/ccache/libexec:$PATH" >> $GITHUB_ENV
          echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
      - name: build
        run: |
          export PATH="/usr/local/opt/flex/bin:$PATH"
          export LDFLAGS="-L/usr/local/opt/flex/lib"
          export CPPFLAGS="-I/usr/local/opt/flex/include"
          # CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working
          ./bootstrap.sh -e -t && cd build  && cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4
      - name: cache save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ${{ env.CCACHE_DIR }}
          key: macos-xcode-ccache-${{github.ref}}-${{github.sha}}
      - name: test
        id: test
        run: |
          # Set core file size limit to 1GiB
          ulimit -c 1048576
          cd build && ctest -j4 --output-on-failure --timeout 300
      - name: linter
        run: cd build && ninja linter
      - name: check-cores
        if: ${{ failure() && steps.test.conclusion == 'failure' }}
        run: |
          if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: macos-coredumps
          path: /cores
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: macos-binaries
          path: build/bin
  windows_VS2019:
    name: "windows-2019"
    runs-on: windows-2019
    timeout-minutes: 180
    steps:
      - name: Support longpaths
        run: git config --system core.longpaths true
      - id: checkout
        uses: actions/checkout@v3
      - name: sccache cache restore
        uses: actions/cache/restore@v3
        with:
          path: ~/AppData/Local/Mozilla/sccache/cache
          key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-sccache-${{ github.ref }}
            ${{ runner.os }}-sccache-refs/heads/main
            ${{ runner.os }}-sccache
      - name: Run sccache-cache
        uses: mozilla-actions/sccache-action@v0.0.3
      - name: Install ninja-build tool
        uses: seanmiddleditch/gha-setup-ninja@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.11'
      - name: Set up Lua
        uses: xpol/setup-lua@v0.3
      - name: Set up NASM for OpenSSL
        uses: ilammy/setup-nasm@v1
      - id: install-sqliteodbc-driver
        run: |
          Invoke-WebRequest -Uri "http://www.ch-werner.de/sqliteodbc/sqliteodbc_w64.exe" -OutFile "sqliteodbc_w64.exe"
          if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne "bd93e8450bcbd4ae16894e06ecb1b8572d6a01448df5b8dd2964eb6d3d302db7") {Write "Hash mismatch"; Exit 1}
          ./sqliteodbc_w64.exe /S
        shell: powershell
      - name: build
        run: |
          for /f "usebackq delims=" %%i in (`vswhere.exe -latest -property installationPath`) do if exist "%%i\Common7\Tools\vsdevcmd.bat" call "%%i\Common7\Tools\vsdevcmd.bat" -arch=x64 -host_arch=x64
          win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA
          sccache --show-stats
        shell: cmd
      - name: sccache cache save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ~/AppData/Local/Mozilla/sccache/cache
          key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }}
      - name: test
        run: cd ..\b && ctest --timeout 300 --parallel %NUMBER_OF_PROCESSORS% -C Release --output-on-failure
        shell: cmd
      - name: linter
        run: cd ..\b && cmake --build . --target linter --config Release -j 8
        shell: cmd
  ubuntu_20_04:
    name: "ubuntu-20.04"
    runs-on: ubuntu-20.04
    timeout-minutes: 120
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - name: cache restore
        uses: actions/cache/restore@v3
        with:
          path: ~/.ccache
          key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}}
          restore-keys: |
            ubuntu-20.04-ccache-${{github.ref}}-
            ubuntu-20.04-ccache-refs/heads/main-
      - id: install_deps
        run: |
          sudo apt update
          sudo apt install -y ccache libfl-dev libboost-all-dev libpcap-dev
          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
          echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
      - name: build
        run: |
          export CC=gcc-11
          export CXX=g++-11
          ./bootstrap.sh -e -t
          cd build
          cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \
              -DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON  \
              -DENABLE_LUA_SCRIPTING=OFF -DENABLE_PYTHON_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_ELASTICSEARCH=OFF -DENABLE_KUBERNETES=OFF -DENABLE_OPC=OFF ..
          make -j$(nproc) VERBOSE=1
      - name: cache save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ~/.ccache
          key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}}
      - name: test
        id: test
        run: |
          # Set core file size limit to 1GiB
          ulimit -c 1048576
          cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
      - name: check-cores
        if: ${{ failure() && steps.test.conclusion == 'failure' }}
        run: |
          if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: ubuntu-coredumps
          path: /var/lib/apport/coredump/
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: ubuntu-binaries
          path: build/bin
  ubuntu_20_04_clang:
    name: "ubuntu-20.04-clang"
    runs-on: ubuntu-20.04
    timeout-minutes: 180
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - name: cache restore
        uses: actions/cache/restore@v3
        with:
          path: ~/.ccache
          key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
          restore-keys: |
            ubuntu-20.04-all-clang-ccache-${{github.ref}}-
            ubuntu-20.04-all-clang-ccache-refs/heads/main-
      - id: install_deps
        run: |
          wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key|sudo apt-key add -
          echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" | sudo tee -a /etc/apt/sources.list
          echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main" | sudo tee -a /etc/apt/sources.list
          echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | sudo tee -a /etc/apt/sources.list
          echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" | sudo tee -a /etc/apt/sources.list
          sudo apt update
          sudo apt install -y ccache libfl-dev libpcap-dev libboost-all-dev libusb-1.0-0-dev libpng-dev libgps-dev clang-14 clang-tidy-14 libc++-14-dev libc++abi-14-dev libsqliteodbc lua5.3 liblua5.3-dev flake8 parallel
          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
          echo -e "127.0.0.1\t$HOSTNAME" | sudo tee -a /etc/hosts > /dev/null
      - name: build
        run: |
          export CC=clang-14
          export CXX=clang++-14
          ./bootstrap.sh -e -t
          cd build
          export CXXFLAGS="${CXXFLAGS} -stdlib=libc++"
          export LDFLAGS="${LDFLAGS} -stdlib=libc++"
          cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
              -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
              -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
              -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
              -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
          cmake --build . --parallel $(nproc)
      - id: cache_save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ~/.ccache
          key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
      - name: test
        id: test
        run: |
          # Set core file size limit to 1GiB
          ulimit -c 1048576
          cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
      - name: linter
        run: cd build && make -j$(nproc) linter
      - name: shellcheck
        run: cd build && make shellcheck
      - id: flake8_check
        run: cd build && make flake8
      - id: files
        uses: Ana06/get-changed-files@v2.2.0
        continue-on-error: true
      - name: clang-tidy
        run: |
          if [[ -n "${{ steps.files.outputs.all }}" ]]; then
            FILES="${{ steps.files.outputs.all }}"
          else
            git fetch origin main && git checkout -b main origin/main || true
            FILES=$(git diff --name-only main ${{ github.sha }} --)
            git checkout ${{ github.sha }}
          fi
          # https://stackoverflow.com/questions/58466701/clang-tidy-cant-locate-stdlib-headers
          sed -i -e 's/\/usr\/lib\/ccache\/clang++-14/\/lib\/llvm-14\/bin\/clang++/g' build/compile_commands.json
          sed -i -e 's/\/usr\/lib\/ccache\/clang-14/\/lib\/llvm-14\/bin\/clang/g' build/compile_commands.json

          parallel -j$(( $(nproc) + 1 )) ./run_clang_tidy.sh ::: ${FILES}
      - name: check-cores
        if: ${{ failure() && steps.test.conclusion == 'failure' }}
        run: |
          if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: ubuntu-clang-coredumps
          path: /var/lib/apport/coredump/
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: ubuntu-clang-binaries
          path: build/bin
  centos:
    name: "centos"
    runs-on: ubuntu-20.04
    timeout-minutes: 180
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - name: cache restore
        uses: actions/cache/restore@v3
        with:
          path: ~/.ccache
          key: centos-ccache-${{github.ref}}-${{github.sha}}
          restore-keys: |
            centos-ccache-${{github.ref}}-
            centos-ccache-refs/heads/main-
      - id: install_deps
        run: |
          sudo apt update
          sudo apt install -y ccache
          echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
      - id: build
        run: |
          if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
          # centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here
          mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \
              -DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \
              -DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
              -DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \
              -DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test
      - name: cache save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ~/.ccache
          key: centos-ccache-${{github.ref}}-${{github.sha}}
      - id: test
        run: |
          # Set core file size limit to 1GiB
          ulimit -c 1048576
          echo '/cores/core.%e.%p' | sudo tee /proc/sys/kernel/core_pattern
          sudo mkdir /cores
          sudo chmod 777 /cores
          docker run --name centos-test --init --ulimit core=1048576 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images | grep apacheminificpp | grep centos | awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
      - name: check-cores
        if: ${{ failure() && steps.test.conclusion == 'failure' }}
        run: |
          if [ "$(ls -A /cores)" ]; then
            echo "CORES_EXIST=true" >> $GITHUB_ENV;
            docker cp centos-test:/opt/minifi/build/bin /tmp
            sudo chmod -R 777 /cores
          fi
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: centos-crashes
          path: /cores
      - uses: actions/upload-artifact@v3.1.2
        if: ${{ failure() && env.CORES_EXIST == 'true' }}
        with:
          name: centos-binaries
          path: /tmp/bin
  docker_build:
    name: "Docker build for integration tests"
    runs-on: ubuntu-20.04
    timeout-minutes: 180
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - name: cache restore
        uses: actions/cache/restore@v3
        with:
          path: ~/.ccache
          key: docker-ccache-${{github.ref}}-${{github.sha}}
          restore-keys: |
            docker-ccache-${{github.ref}}-
            docker-ccache-refs/heads/main
      - id: build
        run: |
          if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
          mkdir build
          cd build
          cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
          make docker
      - name: cache save
        uses: actions/cache/save@v3
        if: always()
        with:
          path: ~/.ccache
          key: docker-ccache-${{github.ref}}-${{github.sha}}
      - name: Save docker image
        run: cd build && docker save -o minifi_docker.tar apacheminificpp:$(grep CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt | cut -d "=" -f2)
      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: minifi_docker
          path: build/minifi_docker.tar
  docker_tests:
    name: "Docker integration tests"
    needs: docker_build
    runs-on: ubuntu-20.04
    timeout-minutes: 180
    steps:
      - id: checkout
        uses: actions/checkout@v3
      - id: run_cmake
        name: Run CMake
        run: |
          mkdir build
          cd build
          cmake ${DOCKER_CMAKE_FLAGS} ..
      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          name: minifi_docker
          path: build
      - name: Load Docker image
        run: |
          docker load --input ./build/minifi_docker.tar
      - id: install_deps
        name: Install dependencies for Docker Verify
        run: |
          sudo apt update
          sudo apt install -y python3-virtualenv
      - id: test
        name: Docker Verify
        working-directory: ./build
        run: make docker-verify
      - name: Test Reporter
        if: always()
        uses: phoenix-actions/test-reporting@v12
        with:
          name: Docker integration tests
          path: build/behavex_output/behave/*.xml
          reporter: java-junit
          output-to: 'step-summary'
          only-summary: 'true'
          list-tests: 'failed'
          list-suites: 'failed'
      - name: Upload artifact
        if: failure()
        uses: actions/upload-artifact@v3
        with:
          name: behavex_output
          path: build/behavex_output