MINIFICPP-2180 Package generation should avoid including test_processors #4346

Workflow file for this run

	name: "MiNiFi-CPP CI"
	on: [push, pull_request, workflow_dispatch]
	env:
	DOCKER_CMAKE_FLAGS: -DDOCKER_VERIFY_THREAD=3 -DUSE_SHARED_LIBS= -DSTRICT_GSL_CHECKS=AUDIT -DCI_BUILD=ON -DDISABLE_JEMALLOC=ON -DENABLE_AWS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_AZURE=ON -DENABLE_SQL=ON \
	-DENABLE_SPLUNK=ON -DENABLE_GCP=ON -DENABLE_OPC=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_TEST_PROCESSORS=ON -DENABLE_PROMETHEUS=ON \
	-DENABLE_ELASTICSEARCH=OFF -DDOCKER_BUILD_ONLY=ON
	SCCACHE_GHA_ENABLE: true
	jobs:
	macos_xcode:
	name: "macos-xcode"
	runs-on: macos-12
	timeout-minutes: 180
	env:
	CCACHE_BASEDIR: ${{ GITHUB.WORKSPACE }}
	CCACHE_DIR: ${{ GITHUB.WORKSPACE }}/.ccache
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- name: cache restore
	uses: actions/cache/restore@v3
	with:
	path: ${{ env.CCACHE_DIR }}
	key: macos-xcode-ccache-${{github.ref}}-${{github.sha}}
	restore-keys: \|
	macos-xcode-ccache-${{github.ref}}-
	macos-xcode-ccache-refs/heads/main-
	- id: install_dependencies
	run: \|
	# Skip brew update until https://github.com/actions/setup-python/issues/577 is fixed
	# brew update
	HOMEBREW_NO_AUTO_UPDATE=1 brew install ossp-uuid flex lua ccache sqliteodbc automake autoconf
	- id: setup_env
	name: setup enviroment
	run: \|
	echo "PATH=/usr/lib/ccache:/usr/local/opt/ccache/bin:/usr/local/opt/ccache/libexec:$PATH" >> $GITHUB_ENV
	echo -e "127.0.0.1\t$HOSTNAME" \| sudo tee -a /etc/hosts > /dev/null
	# https://github.com/actions/virtual-environments/blob/main/images/macos/macos-12-Readme.md#xcode
	sudo xcode-select -switch /Applications/Xcode_14.0.1.app
	- name: build
	run: \|
	export PATH="/usr/local/opt/flex/bin:$PATH"
	export LDFLAGS="-L/usr/local/opt/flex/lib"
	export CPPFLAGS="-I/usr/local/opt/flex/include"
	# CPPFLAGS are not recognized by cmake, so we have to force them to CFLAGS and CXXFLAGS to have flex 2.6 working
	./bootstrap.sh -e -t && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DCMAKE_C_FLAGS="${CPPFLAGS} ${CFLAGS}" -DCMAKE_CXX_FLAGS="${CPPFLAGS} ${CXXFLAGS}" -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_SQL=ON -DUSE_REAL_ODBC_TEST_DRIVER=ON -DENABLE_AZURE=ON -DENABLE_GCP=ON -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_RULE_MESSAGES=OFF -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON .. && cmake --build . --parallel 4
	- name: cache save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ${{ env.CCACHE_DIR }}
	key: macos-xcode-ccache-${{github.ref}}-${{github.sha}}
	- name: test
	id: test
	run: \|
	# Set core file size limit to 1GiB
	ulimit -c 1048576
	cd build && make test ARGS="--timeout 300 -j4 --output-on-failure"
	- name: linter
	run: cd build && make -j4 linter
	- name: check-cores
	if: ${{ failure() && steps.test.conclusion == 'failure' }}
	run: \|
	if [ "$(ls -A /cores)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: macos-coredumps
	path: /cores
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: macos-binaries
	path: build/bin
	windows_VS2019:
	name: "windows-2019"
	runs-on: windows-2019
	timeout-minutes: 180
	steps:
	- name: Support longpaths
	run: git config --system core.longpaths true
	- id: checkout
	uses: actions/checkout@v3
	- name: sccache cache restore
	uses: actions/cache/restore@v3
	with:
	path: ~/AppData/Local/Mozilla/sccache/cache
	key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }}
	restore-keys: \|
	${{ runner.os }}-sccache-${{ github.ref }}
	${{ runner.os }}-sccache-refs/heads/main
	${{ runner.os }}-sccache
	- name: Run sccache-cache
	uses: mozilla-actions/[email protected]
	- name: Install ninja-build tool
	uses: seanmiddleditch/gha-setup-ninja@v3
	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.11'
	- name: Set up Lua
	uses: xpol/[email protected]
	- name: Set up NASM for OpenSSL
	uses: ilammy/setup-nasm@v1
	- id: install-sqliteodbc-driver
	run: \|
	Invoke-WebRequest -Uri "http://www.ch-werner.de/sqliteodbc/sqliteodbc_w64.exe" -OutFile "sqliteodbc_w64.exe"
	if ((Get-FileHash 'sqliteodbc_w64.exe').Hash -ne "bd93e8450bcbd4ae16894e06ecb1b8572d6a01448df5b8dd2964eb6d3d302db7") {Write "Hash mismatch"; Exit 1}
	./sqliteodbc_w64.exe /S
	shell: powershell
	- name: build
	run: \|
	for /f "usebackq delims=" %%i in (`vswhere.exe -latest -property installationPath`) do if exist "%%i\Common7\Tools\vsdevcmd.bat" call "%%i\Common7\Tools\vsdevcmd.bat" -arch=x64 -host_arch=x64
	win_build_vs.bat ..\b /64 /CI /S /A /PDH /SPLUNK /GCP /ELASTIC /K /L /R /Z /N /RO /PR /PYTHON_SCRIPTING /LUA_SCRIPTING /MQTT /SCCACHE /NINJA
	sccache --show-stats
	shell: cmd
	- name: sccache cache save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ~/AppData/Local/Mozilla/sccache/cache
	key: ${{ runner.os }}-sccache-${{ github.ref }}-${{ github.sha }}
	- name: test
	run: cd ..\b && ctest --timeout 300 --parallel %NUMBER_OF_PROCESSORS% -C Release --output-on-failure
	shell: cmd
	- name: linter
	run: cd ..\b && cmake --build . --target linter --config Release -j 8
	shell: cmd
	ubuntu_20_04:
	name: "ubuntu-20.04"
	runs-on: ubuntu-20.04
	timeout-minutes: 120
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- name: cache restore
	uses: actions/cache/restore@v3
	with:
	path: ~/.ccache
	key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}}
	restore-keys: \|
	ubuntu-20.04-ccache-${{github.ref}}-
	ubuntu-20.04-ccache-refs/heads/main-
	- id: install_deps
	run: \|
	sudo apt update
	sudo apt install -y ccache libfl-dev libboost-all-dev libpcap-dev
	echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
	echo -e "127.0.0.1\t$HOSTNAME" \| sudo tee -a /etc/hosts > /dev/null
	- name: build
	run: \|
	export CC=gcc-11
	export CXX=g++-11
	./bootstrap.sh -e -t
	cd build
	cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_SQL=OFF -DENABLE_LIBRDKAFKA=OFF -DENABLE_AWS=OFF \
	-DENABLE_AZURE=OFF -DENABLE_SPLUNK=OFF -DENABLE_GCP=OFF -DENABLE_PROCFS=OFF -DENABLE_BUSTACHE=ON -DENABLE_PCAP=ON -DENABLE_JNI=ON -DENABLE_SFTP=ON \
	-DENABLE_LUA_SCRIPTING=OFF -DENABLE_PYTHON_SCRIPTING=OFF -DENABLE_MQTT=OFF -DENABLE_ELASTICSEARCH=OFF -DENABLE_KUBERNETES=OFF -DENABLE_OPC=OFF ..
	make -j$(nproc) VERBOSE=1
	- name: cache save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ~/.ccache
	key: ubuntu-20.04-ccache-${{github.ref}}-${{github.sha}}
	- name: test
	id: test
	run: \|
	# Set core file size limit to 1GiB
	ulimit -c 1048576
	cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
	- name: check-cores
	if: ${{ failure() && steps.test.conclusion == 'failure' }}
	run: \|
	if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: ubuntu-coredumps
	path: /var/lib/apport/coredump/
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: ubuntu-binaries
	path: build/bin
	ubuntu_20_04_clang:
	name: "ubuntu-20.04-clang"
	runs-on: ubuntu-20.04
	timeout-minutes: 180
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- name: cache restore
	uses: actions/cache/restore@v3
	with:
	path: ~/.ccache
	key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
	restore-keys: \|
	ubuntu-20.04-all-clang-ccache-${{github.ref}}-
	ubuntu-20.04-all-clang-ccache-refs/heads/main-
	- id: install_deps
	run: \|
	wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key\|sudo apt-key add -
	echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" \| sudo tee -a /etc/apt/sources.list
	echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal main" \| sudo tee -a /etc/apt/sources.list
	echo "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" \| sudo tee -a /etc/apt/sources.list
	echo "deb-src http://apt.llvm.org/focal/ llvm-toolchain-focal-14 main" \| sudo tee -a /etc/apt/sources.list
	sudo apt update
	sudo apt install -y ccache libfl-dev libpcap-dev libboost-all-dev libusb-1.0-0-dev libpng-dev libgps-dev clang-14 clang-tidy-14 libc++-14-dev libc++abi-14-dev libsqliteodbc lua5.3 liblua5.3-dev flake8 parallel
	echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
	echo -e "127.0.0.1\t$HOSTNAME" \| sudo tee -a /etc/hosts > /dev/null
	- name: build
	run: \|
	export CC=clang-14
	export CXX=clang++-14
	./bootstrap.sh -e -t
	cd build
	export CXXFLAGS="${CXXFLAGS} -stdlib=libc++"
	export LDFLAGS="${LDFLAGS} -stdlib=libc++"
	cmake -DUSE_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCI_BUILD=ON -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON -DENABLE_BUSTACHE=ON -DENABLE_COAP=ON \
	-DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON -DENABLE_OPENCV=ON \
	-DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_PCAP=ON -DENABLE_SENSORS=ON -DENABLE_SFTP=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
	-DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON -DENABLE_ELASTICSEARCH=ON \
	-DCMAKE_EXPORT_COMPILE_COMMANDS=ON ..
	cmake --build . --parallel $(nproc)
	- id: cache_save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ~/.ccache
	key: ubuntu-20.04-all-clang-ccache-${{github.ref}}-${{github.sha}}
	- name: test
	id: test
	run: \|
	# Set core file size limit to 1GiB
	ulimit -c 1048576
	cd build && make test ARGS="--timeout 300 -j8 --output-on-failure"
	- name: linter
	run: cd build && make -j$(nproc) linter
	- name: shellcheck
	run: cd build && make shellcheck
	- id: flake8_check
	run: cd build && make flake8
	- id: files
	uses: Ana06/[email protected]
	continue-on-error: true
	- name: clang-tidy
	run: \|
	if [[ -n "${{ steps.files.outputs.all }}" ]]; then
	FILES="${{ steps.files.outputs.all }}"
	else
	git fetch origin main && git checkout -b main origin/main \|\| true
	FILES=$(git diff --name-only main ${{ github.sha }} --)
	git checkout ${{ github.sha }}
	fi
	# https://stackoverflow.com/questions/58466701/clang-tidy-cant-locate-stdlib-headers
	sed -i -e 's/\/usr\/lib\/ccache\/clang++-14/\/lib\/llvm-14\/bin\/clang++/g' build/compile_commands.json
	sed -i -e 's/\/usr\/lib\/ccache\/clang-14/\/lib\/llvm-14\/bin\/clang/g' build/compile_commands.json

	parallel -j$(( $(nproc) + 1 )) ./run_clang_tidy.sh ::: ${FILES}
	- name: check-cores
	if: ${{ failure() && steps.test.conclusion == 'failure' }}
	run: \|
	if [ "$(ls -A /var/lib/apport/coredump/)" ]; then echo "CORES_EXIST=true" >> $GITHUB_ENV; fi
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: ubuntu-clang-coredumps
	path: /var/lib/apport/coredump/
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: ubuntu-clang-binaries
	path: build/bin
	centos:
	name: "centos"
	runs-on: ubuntu-20.04
	timeout-minutes: 180
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- name: cache restore
	uses: actions/cache/restore@v3
	with:
	path: ~/.ccache
	key: centos-ccache-${{github.ref}}-${{github.sha}}
	restore-keys: \|
	centos-ccache-${{github.ref}}-
	centos-ccache-refs/heads/main-
	- id: install_deps
	run: \|
	sudo apt update
	sudo apt install -y ccache
	echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
	- id: build
	run: \|
	if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
	# centos build can run out of the github runners' disk space if built with RelWithDebInfo so we keep the Release build here
	mkdir build && cd build && cmake -DUSE_SHARED_LIBS=ON -DCI_BUILD=ON -DCMAKE_BUILD_TYPE=Release -DSTRICT_GSL_CHECKS=AUDIT -DFAIL_ON_WARNINGS=ON -DENABLE_AWS=ON -DENABLE_AZURE=ON \
	-DENABLE_COAP=ON -DENABLE_ENCRYPT_CONFIG=ON -DENABLE_GPS=ON -DENABLE_LIBRDKAFKA=ON -DENABLE_MQTT=ON -DENABLE_NANOFI=ON -DENABLE_OPC=ON \
	-DENABLE_OPENCV=ON -DENABLE_OPENWSMAN=ON -DENABLE_OPS=ON -DENABLE_SENSORS=ON -DENABLE_SQL=ON -DENABLE_SYSTEMD=ON \
	-DENABLE_USB_CAMERA=ON -DENABLE_PYTHON_SCRIPTING=ON -DENABLE_LUA_SCRIPTING=ON -DENABLE_KUBERNETES=ON -DENABLE_GCP=ON -DENABLE_PROCFS=ON -DENABLE_PROMETHEUS=ON \
	-DENABLE_ELASTICSEARCH=ON -DDOCKER_SKIP_TESTS=OFF -DDOCKER_BUILD_ONLY=ON -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache .. && make centos-test
	- name: cache save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ~/.ccache
	key: centos-ccache-${{github.ref}}-${{github.sha}}
	- id: test
	run: \|
	# Set core file size limit to 1GiB
	ulimit -c 1048576
	echo '/cores/core.%e.%p' \| sudo tee /proc/sys/kernel/core_pattern
	sudo mkdir /cores
	sudo chmod 777 /cores
	docker run --name centos-test --init --ulimit core=1048576 --mount type=bind,source=/cores,target=/cores apacheminificpp:$(docker images \| grep apacheminificpp \| grep centos \| awk '{print $2}') bash -c 'cd /opt/minifi/build && make test ARGS="--timeout 300 -j8 --output-on-failure"'
	- name: check-cores
	if: ${{ failure() && steps.test.conclusion == 'failure' }}
	run: \|
	if [ "$(ls -A /cores)" ]; then
	echo "CORES_EXIST=true" >> $GITHUB_ENV;
	docker cp centos-test:/opt/minifi/build/bin /tmp
	sudo chmod -R 777 /cores
	fi
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: centos-crashes
	path: /cores
	- uses: actions/[email protected]
	if: ${{ failure() && env.CORES_EXIST == 'true' }}
	with:
	name: centos-binaries
	path: /tmp/bin
	docker_build:
	name: "Docker build for integration tests"
	runs-on: ubuntu-20.04
	timeout-minutes: 180
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- name: cache restore
	uses: actions/cache/restore@v3
	with:
	path: ~/.ccache
	key: docker-ccache-${{github.ref}}-${{github.sha}}
	restore-keys: \|
	docker-ccache-${{github.ref}}-
	docker-ccache-refs/heads/main
	- id: build
	run: \|
	if [ -d ~/.ccache ]; then mv ~/.ccache .; fi
	mkdir build
	cd build
	cmake ${DOCKER_CMAKE_FLAGS} -DDOCKER_CCACHE_DUMP_LOCATION=$HOME/.ccache ..
	make docker
	- name: cache save
	uses: actions/cache/save@v3
	if: always()
	with:
	path: ~/.ccache
	key: docker-ccache-${{github.ref}}-${{github.sha}}
	- name: Save docker image
	run: cd build && docker save -o minifi_docker.tar apacheminificpp:$(grep CMAKE_PROJECT_VERSION:STATIC CMakeCache.txt \| cut -d "=" -f2)
	- name: Upload artifact
	uses: actions/upload-artifact@v3
	with:
	name: minifi_docker
	path: build/minifi_docker.tar
	docker_tests:
	name: "Docker integration tests"
	needs: docker_build
	runs-on: ubuntu-20.04
	timeout-minutes: 180
	steps:
	- id: checkout
	uses: actions/checkout@v3
	- id: run_cmake
	name: Run CMake
	run: \|
	mkdir build
	cd build
	cmake ${DOCKER_CMAKE_FLAGS} ..
	- name: Download artifact
	uses: actions/download-artifact@v3
	with:
	name: minifi_docker
	path: build
	- name: Load Docker image
	run: \|
	docker load --input ./build/minifi_docker.tar
	- id: install_deps
	name: Install dependencies for Docker Verify
	run: \|
	sudo apt update
	sudo apt install -y python3-virtualenv
	- id: test
	name: Docker Verify
	working-directory: ./build
	run: make docker-verify
	- name: Test Reporter
	if: always()
	uses: phoenix-actions/test-reporting@v12
	with:
	name: Docker integration tests
	path: build/behavex_output/behave/*.xml
	reporter: java-junit
	output-to: 'step-summary'
	only-summary: 'true'
	list-tests: 'failed'
	list-suites: 'failed'
	- name: Upload artifact
	if: failure()
	uses: actions/upload-artifact@v3
	with:
	name: behavex_output
	path: build/behavex_output

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

MINIFICPP-2180 Package generation should avoid including test_processors #4346

Workflow file

MINIFICPP-2180 Package generation should avoid including test_processors #4346

Jobs

Run details

Workflow file for this run