From 8a7f7744dcc936fe6e4aa6da25bc85faeacba2c8 Mon Sep 17 00:00:00 2001 From: Patrick Kappl Date: Sun, 24 Nov 2024 12:24:05 +0000 Subject: [PATCH] Rework golden tests, part 1 --- Tests/GoldenTests/CMakeLists.txt | 18 +++---- Tests/GoldenTests/HelloDummy.test.cpp | 4 -- Tests/GoldenTests/HelloWorld.test.cpp | 4 -- Tests/GoldenTests/Scripts/TestRunner.sh | 27 +++++++--- Tests/GoldenTests/Scripts/ThreadTestRunner.sh | 15 ------ cmake/golden-tests.cmake | 52 +++++-------------- 6 files changed, 41 insertions(+), 79 deletions(-) delete mode 100644 Tests/GoldenTests/Scripts/ThreadTestRunner.sh diff --git a/Tests/GoldenTests/CMakeLists.txt b/Tests/GoldenTests/CMakeLists.txt index 1c4c2b61..ea20fc50 100644 --- a/Tests/GoldenTests/CMakeLists.txt +++ b/Tests/GoldenTests/CMakeLists.txt @@ -1,12 +1,12 @@ -include(${CMAKE_SOURCE_DIR}/cmake/golden-tests.cmake) +include("${CMAKE_SOURCE_DIR}/cmake/golden-tests.cmake") -# Initialize "global" variables used by macros -set(output_files "") +add_golden_test(TESTFILE HelloWorld.test.cpp LIBS rodos::rodos) +add_golden_test(TESTFILE HelloDummy.test.cpp LIBS rodos::rodos etl::etl Sts1CobcSw_Dummy) -set(Sts1CobcSw "${CMAKE_SOURCE_DIR}/Sts1CobcSw") +# --- All golden tests --- -add_golden_test(TESTFILE "HelloWorld.test.cpp" LIB rodos::rodos) - -add_golden_test(TESTFILE "HelloDummy.test.cpp" LIB rodos::rodos etl::etl Sts1CobcSw_Dummy) - -add_custom_target(AllGoldenTests DEPENDS ${output_files}) +get_property( + all_golden_test_targets DIRECTORY ${CMAKE_CURRENT_LIST_DIR} PROPERTY BUILDSYSTEM_TARGETS +) +add_custom_target(AllGoldenTests) # Must be defined after getting all hardware test targets +add_dependencies(AllGoldenTests ${all_golden_test_targets}) diff --git a/Tests/GoldenTests/HelloDummy.test.cpp b/Tests/GoldenTests/HelloDummy.test.cpp index 15828ded..a97d91c8 100644 --- a/Tests/GoldenTests/HelloDummy.test.cpp +++ b/Tests/GoldenTests/HelloDummy.test.cpp @@ -7,16 +7,12 @@ #include -std::uint32_t printfMask = 0; - - namespace sts1cobcsw { class HelloDummy : public RODOS::StaticThread<> { void run() override { - printfMask = 1; auto const dummy = Dummy(); RODOS::PRINTF("Hello, %s!\n", dummy.name.data()); diff --git a/Tests/GoldenTests/HelloWorld.test.cpp b/Tests/GoldenTests/HelloWorld.test.cpp index 2054c224..b7d5776e 100644 --- a/Tests/GoldenTests/HelloWorld.test.cpp +++ b/Tests/GoldenTests/HelloWorld.test.cpp @@ -3,16 +3,12 @@ #include -std::uint32_t printfMask = 0; - - namespace sts1cobcsw { class HelloWorld : public RODOS::StaticThread<> { void run() override { - printfMask = 1; RODOS::PRINTF("Hello, World!\n"); RODOS::hwResetAndReboot(); } diff --git a/Tests/GoldenTests/Scripts/TestRunner.sh b/Tests/GoldenTests/Scripts/TestRunner.sh index ea0c4a32..5f17bb48 100644 --- a/Tests/GoldenTests/Scripts/TestRunner.sh +++ b/Tests/GoldenTests/Scripts/TestRunner.sh @@ -1,12 +1,23 @@ #!/usr/bin/env bash -echo "::: Running $1" -set -e -# Kill test executable (with SIGKILL) after 8 seconds. Necessary -# because sometimes tests might deadlock. -if timeout -s 9 8 $1 > "$1.output" -then - true +if [ $# -ne 2 ]; then + echo "Error: wrong number of arguments" + echo "Usage: $0 " + exit 1 +fi +echo "Running $1" + +# Remove Rodos header/intro from output +$1 > "$1.output" +pattern="--------------- Application running ------------" +# Search for the pattern and store everything after it in "$1.output.trimmed" +awk "/$pattern/ {found=1; next} found" "$1.output" > "$1.output.trimmed" + +# Compare the trimmed output with the expected one +if diff -q "$1.output.trimmed" "$2" >/dev/null; then + echo "-> passed: test generated expected output" + exit 0 else - echo ":: KILLED BY TIMEOUT" >> "$1.output" + echo "-> failed: test did not generate expected output" + exit 1 fi diff --git a/Tests/GoldenTests/Scripts/ThreadTestRunner.sh b/Tests/GoldenTests/Scripts/ThreadTestRunner.sh deleted file mode 100644 index ec0980da..00000000 --- a/Tests/GoldenTests/Scripts/ThreadTestRunner.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -e -# Kill test executable (with SIGKILL) after 8 seconds. Necessary -# because sometimes tests might deadlock. -if timeout -s 9 8 $1 > "$1.output" -then - true -else - echo ":: KILLED BY TIMEOUT" >> "$1.output" -fi - -# Remove header / intro from output and expected output -linecount=`wc -l < "$1"` -pattern="--------------- Application running ------------" -grep -h -B 0 -A $linecount -e "$pattern" $1.output > $1.output.trimmed || true diff --git a/cmake/golden-tests.cmake b/cmake/golden-tests.cmake index 0f7ab4ee..87e6f723 100644 --- a/cmake/golden-tests.cmake +++ b/cmake/golden-tests.cmake @@ -1,55 +1,29 @@ +# TODO: Check if this can be turned into a function macro(add_golden_test) - # Parse arguments set(options "") set(one_value_args TESTFILE) - set(multi_value_args SOURCE LIB) + set(multi_value_args SOURCES LIBS) cmake_parse_arguments(GT "${options}" "${one_value_args}" "${multi_value_args}" ${ARGN}) - # Retrieve file name get_filename_component(test_filename ${GT_TESTFILE} NAME_WE) set(target_name ${PROJECT_NAME}_${test_filename}) - # Sanity check if(NOT ("${GT_TESTFILE}" MATCHES "\.test\.cpp$")) - # In case we are testing one of the real threads - if(NOT ("${GT_TESTFILE}" MATCHES ".*Sts1CobcSw/.*Thread.cpp")) - message(FATAL_ERROR "Name of test file `${GT_TESTFILE}` do not end with .test.cpp") - endif() + message(FATAL_ERROR "Name of test file `${GT_TESTFILE}` do not end with .test.cpp") endif() - # Create executable - add_executable(${target_name} EXCLUDE_FROM_ALL ${GT_TESTFILE} ${GT_SOURCE}) + add_executable(${target_name} EXCLUDE_FROM_ALL ${GT_TESTFILE} ${GT_SOURCES}) target_include_directories(${target_name} ${warning_guard} PUBLIC "${CMAKE_SOURCE_DIR}") set_target_properties(${target_name} PROPERTIES OUTPUT_NAME ${test_filename}) - target_link_libraries(${target_name} PUBLIC ${GT_LIB}) + target_link_libraries(${target_name} PUBLIC ${GT_LIBS}) - if("${GT_TESTFILE}" MATCHES "\.test\.cpp$") - add_custom_command( - OUTPUT "${test_filename}.output" - COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/Scripts/TestRunner.sh" - $ DEPENDS ${target_name} Scripts/TestRunner.sh - ) - add_test(NAME ${test_filename}_Test - COMMAND diff "${test_filename}.output" - "${CMAKE_CURRENT_SOURCE_DIR}/ExpectedOutputs/${test_filename}.txt" - ) - else() - add_custom_command( - OUTPUT "${test_filename}.output" - COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/Scripts/ThreadTestRunner.sh" - $ - DEPENDS ${target_name} Scripts/ThreadTestRunner.sh - ) - add_test(NAME ${test_filename}_Test - COMMAND diff "${test_filename}.output.trimmed" - "${CMAKE_CURRENT_SOURCE_DIR}/ExpectedOutputs/${test_filename}.txt" - ) - endif() - - list(APPEND output_files "${test_filename}.output") - - # Create clean target - add_custom_target( - ${target_name}_Clean COMMAND ${CMAKE_COMMAND} -E remove -f "${test_filename}.output*" + add_test( + NAME ${test_filename}_Test + COMMAND + bash "${CMAKE_SOURCE_DIR}/Tests/GoldenTests/Scripts/TestRunner.sh" + $ + "${CMAKE_SOURCE_DIR}/Tests/GoldenTests/ExpectedOutputs/${test_filename}.txt" ) endmacro() + +# TODO: Convert TestRunner.sh to a CMake script