Skip to content

Update CI & fix failures #756

Update CI & fix failures

Update CI & fix failures #756

name: '[Kafka API] CI Tests'
on:
pull_request:
push:
branches:
- main
env:
KAFKA_SRC_LINK: https://archive.apache.org/dist/kafka/3.3.1/kafka_2.13-3.3.1.tgz
CPU_CORE_NUM: 2
LIBRDKAFKA_TAG: v2.4.0
BUILD_SUB_DIR: builds/sub-build
jobs:
kafka-api-tests:
runs-on: ${{ matrix.os }}
env:
OS_VERSION: ${{ matrix.os }}
BUILD_CXX: ${{ matrix.build-cxx }}
BUILD_TYPE: ${{ matrix.build-type }}
CXX_STANDARD: ${{ matrix.cxx-standard }}
ENABLE_UT_STUBS: ${{ matrix.enable-ut-stubs }}
CHECK_OPTION: ${{ matrix.check-option }}
GENERATE_DOC: ${{ matrix.generate-doc }}
WITH_INSTALLATION: ${{ matrix.with-installation }}
TEST_LABELS: ${{ matrix.test-labels }}
strategy:
matrix:
include:
- os: macos-12
build-cxx: clang++
test-labels: unit|integration
enable-ut-stubs: true
- os: macos-12
build-cxx: clang++
test-labels: robustness
- os: macos-12
build-cxx: clang++
check-option: clang-tidy
- os: ubuntu-22.04
build-cxx: g++
build-type: Debug
test-labels: unit|integration
enable-ut-stubs: true
- os: ubuntu-24.04
build-cxx: g++
build-type: Release
test-labels: robustness
- os: ubuntu-24.04
build-cxx: g++
build-type: Release
cxx-standard: 14
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: asan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: tsan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: ubsan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: clang++
test-labels: unit|integration
generate-doc: true
with-installation: true
- os: ubuntu-24.04
build-cxx: clang++
check-option: clang-tidy
enable-ut-stubs: true
- os: ubuntu-22.04
build-cxx: g++
test-labels: unit|integration
- os: ubuntu-22.04
build-cxx: clang++
test-labels: robustness
steps:
- uses: actions/checkout@v3
- name: Preparation
if: ${{!contains(matrix.os, 'windows')}}
run: |
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sed -e 's/azure.archive.ubuntu.com/us.archive.ubuntu.com/g' -e t -e d /etc/apt/sources.list | sudo tee /etc/apt/sources.list.d/nonazure.list
sudo apt-get update
elif [[ ${OS_VERSION} == 'macos'* ]]; then
echo "CPU_CORE_NUM=3" >> $GITHUB_ENV
fi
- name: Install Dependencies
run: |
mkdir -p ${BUILD_SUB_DIR}
cd ${BUILD_SUB_DIR}
# 1. Install clang/clang-tidy
if [[ ${CHECK_OPTION} == 'clang-tidy' ]]; then
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y clang-tidy
elif [[ ${OS_VERSION} == 'macos'* ]]; then
brew install llvm
export PATH=/usr/local/opt/llvm/bin/:$PATH
fi
fi
# 2. Install googletest (v1.11.0)
wget -nv https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz
tar -xzf release-1.11.0.tar.gz
cd googletest-release-1.11.0
env CXX=${BUILD_CXX} cmake ./
make -j${CPU_CORE_NUM} && sudo make install
cd ../
# 3. Install boost lib
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y libboost-all-dev
elif [[ ${OS_VERSION} == 'macos'* ]]; then
brew install boost
fi
# 4. Install librdkafka
git clone https://github.com/edenhill/librdkafka.git
cd librdkafka
git reset --hard ${LIBRDKAFKA_TAG}
./configure --cxx=${BUILD_CXX}
make -j${CPU_CORE_NUM} && sudo make install
cd ../
# 5. Install rapidjson (for `addons/KafkaMetrics.h`)
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y rapidjson-dev
else
wget -nv https://github.com/Tencent/rapidjson/archive/refs/tags/v1.1.0.tar.gz
tar -xzf v1.1.0.tar.gz
fi
# 6. Install tools to generate document
if [ ${GENERATE_DOC} ]; then
sudo apt install -y python3-markdown
sudo apt install -y doxygen
fi
# 6. Install Jre, Kafka for integration/regression tests
if [[ ${TEST_LABELS} == *'integration'* ]] || [[ ${TEST_LABELS} == *'robustness'* ]]; then
# Install Jre
if [[ ${OS_VERSION} == 'ubuntu-22.04' ]]; then
sudo apt install -y openjdk-18-jre
fi
# Install kafka
time wget -nv ${KAFKA_SRC_LINK}
tar -xzf `basename ${KAFKA_SRC_LINK}`
export PATH=`pwd`/`basename ${KAFKA_SRC_LINK} .tgz`/bin:$PATH
fi
# 7. Install gtest-parallel for integration test
if [[ ${TEST_LABELS} == *'integration'* ]]; then
wget https://github.com/google/gtest-parallel/archive/refs/heads/master.zip
unzip master.zip
export PATH=`pwd`/gtest-parallel-master:$PATH
fi
# Save the PATH to environment file
echo "PATH=${PATH}" >> $GITHUB_ENV
# Print dev env
cmake --version | head -n 1
g++ --version | head -n 1
clang++ --version | head -n 1
- name: Config
run: |
cd ${BUILD_SUB_DIR}
# Restore the PATH from environment file
export PATH=${{ env.PATH }}
if [ ${CXX_STANDARD} ]; then
export CMAKE_CXX_STANDARD="-DCMAKE_CXX_STANDARD=${CXX_STANDARD}"
fi
if [ ${BUILD_TYPE} ]; then
export CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
else
export CMAKE_BUILD_TYPE=""
fi
if [[ ${ENABLE_UT_STUBS} ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_ENABLE_UT_STUBS=ON"
fi
if [[ ${CHECK_OPTION} == 'clang-tidy' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_CLANG_TIDY=ON"
fi
if [[ ${CHECK_OPTION} == 'asan' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_ASAN=ON"
fi
if [[ ${CHECK_OPTION} == 'tsan' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_TSAN=ON"
fi
if [[ ${CHECK_OPTION} == *"ubsan" ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_UBSAN=ON"
fi
if [ ${GENERATE_DOC} ]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_GEN_DOC=ON"
fi
if [ -d "rapidjson-1.1.0" ]; then
export RAPIDJSON_INCLUDE_DIRS=`pwd`/rapidjson-1.1.0/include
fi
env CXX=${BUILD_CXX} cmake ../.. ${CMAKE_CXX_STANDARD} ${CMAKE_BUILD_TYPE} ${BUILD_OPTION}
- name: Build
run: |
cd ${BUILD_SUB_DIR}
make -j${CPU_CORE_NUM} VERBOSE=1
- name: Install
run: |
if [ ${WITH_INSTALLATION} ]; then
cd ${BUILD_SUB_DIR}
sudo make install
fi
- name: Test
if: matrix.test-labels
timeout-minutes: 15
run: |
cd ${BUILD_SUB_DIR}
# Restore the PATH from environment file
export PATH=${{ env.PATH }}
# Start Kafka cluster
rm -f test.env
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --temp-dir ./tmp &
# Wait server to be ready
for i in {1..60}; do cat test.env 2>/dev/null && break || sleep 1; done
# Set the environment variables: KAFKA_BROKER_LIST, KAFKA_BROKER_PIDS
source test.env
# Run tests
ctest -VV -L "${TEST_LABELS}"
# Stop Kafka cluster for integration test & regression test
if [[ ${TEST_LABELS} == *'integration'* ]] || [[ ${TEST_LABELS} == *'robustness'* ]]; then
# Stop kafka cluster
kafka-server-stop.sh
zookeeper-server-stop.sh
fi
kafka-api-tests-windows:
runs-on: windows-latest
env:
TEST_LABELS: ${{ matrix.test-labels }}
strategy:
matrix:
include:
- test-labels: ''
steps:
- uses: actions/checkout@v3
- name: Install Dependencies
run: |
mkdir -p $Env:BUILD_SUB_DIR
cd $Env:BUILD_SUB_DIR
# Install librdkafka
vcpkg install librdkafka
# Install googletest
vcpkg install gtest
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\manual-link\gtest_main*" "C:\VCPKG\INSTALLED\x64-windows\lib\"
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\manual-link\gtest_main*" "C:\VCPKG\INSTALLED\x64-windows\lib\"
# Install boost headers/libraries
vcpkg install boost-optional
vcpkg install boost-algorithm
vcpkg install boost-program-options
ls "C:\VCPKG\INSTALLED\x64-windows\lib"
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\boost_program_options-vc144-mt-x64-1_85.lib" "C:\VCPKG\INSTALLED\x64-windows\lib\boost_program_options.lib"
# Install rapidjson
vcpkg install rapidjson
vcpkg integrate install
- name: Config
run: |
cd $Env:BUILD_SUB_DIR
$Env:GTEST_ROOT='C:\VCPKG\INSTALLED\x64-windows\'
$Env:BOOST_ROOT='C:\VCPKG\INSTALLED\x64-windows\'
$Env:LIBRDKAFKA_INCLUDE_DIR='C:\VCPKG\INSTALLED\x64-windows\include\'
$Env:LIBRDKAFKA_LIBRARY_DIR='C:\VCPKG\INSTALLED\x64-windows\lib\'
$Env:RAPIDJSON_INCLUDE_DIRS='C:\VCPKG\INSTALLED\x64-windows\include\'
cmake -B ./ -A x64 -S ../.. "-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake"
- name: Build
run: |
cd $Env:BUILD_SUB_DIR
cmake --build ./
- name: Test
if: matrix.test-labels
timeout-minutes: 15
run: |
cd $Env:BUILD_SUB_DIR
tree "tests"
# Install kafka
Invoke-WebRequest -Uri $Env:KAFKA_SRC_LINK -OutFile kafka_2.13-2.8.1.tgz
tar xvzf kafka_2.13-2.8.1.tgz
ctest -VV -L $Env:TEST_LABELS