diff --git a/.vscode/launch.json b/.vscode/launch.json index fecf99ad5..bd2ab6791 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -105,6 +105,7 @@ ], "sourceMaps": true, "outFiles": [ + "${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/out/*", "${workspaceFolder}/out/src/**", "${workspaceFolder}/out/test/*", @@ -118,6 +119,32 @@ "TEST_FILTER": ".*" }, }, + { + "name": "Run single-root-ctest Tests", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "${workspaceFolder}/test/end-to-end-tests/single-root-ctest/project-folder", + "--extensionDevelopmentPath=${workspaceFolder}", + "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/index" + ], + "sourceMaps": true, + "outFiles": [ + "${workspaceFolder}/dist/**/*.js", + "${workspaceFolder}/out/*", + "${workspaceFolder}/out/src/**", + "${workspaceFolder}/out/test/*", + "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/*", + "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/test/*" + ], + "preLaunchTask": "Pretest", + "env": { + "CMT_TESTING": "1", + "CMT_QUIET_CONSOLE": "1", + "TEST_FILTER": ".*" + }, + }, { "name": "Run multi-root-UI Tests", "type": "extensionHost", diff --git a/package.json b/package.json index 20bc2c09e..9b89c3c97 100644 --- a/package.json +++ b/package.json @@ -3698,6 +3698,7 @@ "integrationTests": "yarn run pretest && node ./out/test/integration-tests/runTest.js", "endToEndTestsSuccessfulBuild": "yarn run pretest && node ./out/test/end-to-end-tests/successful-build/runTest.js", "endToEndTestsSingleRoot": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-UI/runTest.js", + "endToEndTestsSingleRootCTest": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-ctest/runTest.js", "endToEndTestsMultiRoot": "yarn run pretest && node ./out/test/end-to-end-tests/multi-root-UI/runTest.js", "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/unit-tests/backend/**/*.test.ts", "build-product-icon-font": "yarn --cwd ./tools/product-icon-font-generator/ install && yarn --cwd ./tools/product-icon-font-generator/ build && node ./tools/product-icon-font-generator/dist/index.js --source-directory ./res/product-icons/ --output-directory ./res/ --woff2" diff --git a/src/cmakeProject.ts b/src/cmakeProject.ts index e36932d5e..479ce1400 100644 --- a/src/cmakeProject.ts +++ b/src/cmakeProject.ts @@ -2318,7 +2318,7 @@ export class CMakeProject { async ctest(fromWorkflow: boolean = false): Promise { const drv = await this.preTest(fromWorkflow); const retc = await this.cTestController.runCTest(drv); - return (retc) ? 0 : -1; + return (retc === 0) ? 0 : -1; } async cpack(fromWorkflow: boolean = false): Promise { diff --git a/src/ctest.ts b/src/ctest.ts index c2fd575b4..62e2815ca 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -710,11 +710,6 @@ export class CTestDriver implements vscode.Disposable { return -1; } - if (util.isTestMode()) { - // ProjectController can't be initialized in test mode, so we don't have a usable test explorer - return 0; - } - const initializedTestExplorer = this.ensureTestExplorerInitialized(); const sourceDir = util.platformNormalizePath(driver.sourceDir); const testExplorerRoot = initializedTestExplorer.items.get(sourceDir); diff --git a/test/end-to-end-tests/single-root-ctest/index.ts b/test/end-to-end-tests/single-root-ctest/index.ts new file mode 100644 index 000000000..9391b101c --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/index.ts @@ -0,0 +1,54 @@ +// eslint-disable-next-line import/no-unassigned-import +import 'module-alias/register'; + +import * as path from 'path'; +import * as Mocha from 'mocha'; +import * as glob from 'glob'; +import { Logger } from '@cmt/logging'; + +export function run(): Promise { + // Create the mocha test + const mocha = new Mocha({ + ui: 'tdd', + color: true + }); + + const testsRoot = __dirname; + + return new Promise((c, e) => { + glob('**/*.test.js', { cwd: testsRoot }, (err, files) => { + if (err) { + return e(err); + } + + // Add files to the test suite + const regex = process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : /.*/; + files.forEach(f => { + if (regex.test(f)) { + mocha.addFile(path.resolve(testsRoot, f)); + } + }); + + try { + // Run the mocha test + mocha.timeout(100000); + + // Log the name of each test before it starts. + const beforeEach: Mocha.Func = function (this: Mocha.Context, done: Mocha.Done) { + Logger.logTestName(this.currentTest?.parent?.title, this.currentTest?.title); + done(); + }; + mocha.rootHooks({beforeEach}); + mocha.run(failures => { + if (failures > 0) { + e(new Error(`${failures} tests failed.`)); + } else { + c(); + } + }); + } catch (err) { + e(err); + } + }); + }); +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json new file mode 100644 index 000000000..2857722b5 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json @@ -0,0 +1,6 @@ +{ + "cmake.buildDirectory": "${workspaceFolder}/build", + "cmake.useCMakePresets": "always", + "cmake.configureOnOpen": false, + "cmake.ctest.allowParallelJobs": false +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt new file mode 100644 index 000000000..b60c1ead6 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -0,0 +1,55 @@ +cmake_minimum_required(VERSION 3.6.0) +project(TestCTestProcess VERSION 0.1.0) + +add_library(TestUtils SHARED) +target_sources(TestUtils PRIVATE test_utils.cpp) +target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED + ON) + +set(TESTS_DIR + "" + CACHE STRING "Directory where the files generated by tests will be written") +# Will generate a file get_test_dir.h with the content of get_test_dir.h.in It's +# goal is to provide a function that returns the directory where the tests will +# be executed +configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY) +add_library(GetTestDir INTERFACE) +target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) + +# Adds an executable that will generate an output file by concatenating the +# content of each file in the test directory +add_executable(GenerateOutputFile generate_output_file.cpp) +target_link_libraries(GenerateOutputFile PRIVATE GetTestDir) +set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON) + +enable_testing() + +# The generation of json readable output file must occur after each test run +# Declares a fixture that will be executed after each test run +add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) +set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +include(RegisterTest) + +set(TESTS_OUTPUT_FILES + "" + CACHE STRING "Output files generated by tests") +set(TESTS_NAMES + "" + CACHE STRING "Names of the tests") +set(TESTS_SUCCESS + "" + CACHE BOOL "Success of the tests") + +register_tests( + TEST_DIRECTORY + ${TESTS_DIR} + TEST_NAME_LIST + ${TESTS_NAMES} + TEST_OUTPUT_FILE_LIST + ${TESTS_OUTPUT_FILES} + TEST_SUCCESS_LIST + ${TESTS_SUCCESS}) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json new file mode 100644 index 000000000..bd091181e --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -0,0 +1,44 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "2Successes", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", + "TESTS_SUCCESS": "true;true" + } + }, + { + "name": "2Successes1Failure", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", + "TESTS_SUCCESS": "true;false;true" + } + }, + { + "name": "3Failures", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", + "TESTS_SUCCESS": "false;false;false" + } + } + ] +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake new file mode 100644 index 000000000..981334956 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -0,0 +1,154 @@ +#-------------------------------------------------------------------- +# Generate the source file of the test in argument +# +# Parameters: +# output_file_path: path to the file, the test will write +# test_success: true if the test end successfully. False otherwise. +# Returns: +# test_source: name of the source file that will be generated +#-------------------------------------------------------------------- +function(generate_test_source_file output_file_path test_success) + get_filename_component(output_file_name ${output_file_path} NAME_WE) + # Declare variables used in the template file (test.cpp.in) + set(test_filename ${output_file_path}) + set(success ${test_success}) + # Generate test source file + set(test_source "${output_file_name}.cpp") + configure_file(test.cpp.in ${test_source} @ONLY) + set(test_source "${test_source}" PARENT_SCOPE) +endfunction() + +#-------------------------------------------------------------------- +# Build the name of the test executable from the name of the test source file +# +# The name of the executable will be the one of the test source file without '_' +# and in CamelCase. +# +# Parameters: +# test_source: name of the test source file +# Returns: +# test_exe: name of the test executable +#-------------------------------------------------------------------- +function(build_test_exe_name test_source) + get_filename_component(test_name ${test_source} NAME_WE) + # Replace _ with ; so that the name becomes a list of sub-words + string(REPLACE "_" ";" splitted_test_name ${test_name}) + set(test_exe) + # For each of the sub-word, extract the first letter + # from the rest of the word (radical) + foreach(word ${splitted_test_name}) + string(SUBSTRING ${word} 0 1 first_letter) + string(SUBSTRING ${word} 1 -1 radical) + # Turns first sub-word letter into upper case + string(TOUPPER ${first_letter} up_first_letter) + # Concat uppercase first letter and radical + set(test_exe "${test_exe}${up_first_letter}${radical}") + endforeach() + # Returns test_exe + set(test_exe ${test_exe} PARENT_SCOPE) +endfunction() + +#-------------------------------------------------------------------- +# Create and register a test +# +# Usage: +# register_test(TEST_DIR TEST_NAME TEST_OUTPUT_FILE TEST_SUCCESS ) +# Parameters: +# TEST_DIR: path to the directory where the test will write its output file +# TEST_NAME: name of the test +# TEST_OUTPUT_FILE: name of the file the test should generate +# TEST_SUCCESS: whether or not the test should end successfully +#-------------------------------------------------------------------- +function(register_test) + set(options) + set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE;TEST_SUCCESS") + ### PARSING ARGUMENTS + cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(DEFINED register_test_KEYWORDS_MISSING_VALUES) + message( + FATAL_ERROR + "In the call to register_test function, the keywords ${register_test_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" + ) + endif() + if(DEFINED register_test_UNPARSED_ARGUMENTS) + message( + FATAL_ERROR + "Following arguments are unknown to register_test function: ${register_test_UNPARSED_ARGUMENTS}" + ) + endif() + if(NOT DEFINED register_test_TEST_DIR) + message(FATAL_ERROR "The function register_test is awaiting for TEST_DIR keyword") + endif() + if(NOT DEFINED register_test_TEST_NAME) + message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword") + endif() + if(NOT DEFINED register_test_TEST_OUTPUT_FILE) + message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE keyword") + endif() + if(NOT DEFINED register_test_TEST_SUCCESS) + message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword") + endif() + + set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE}") + message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${test_output_file_path} returning as success: ${register_test_TEST_SUCCESS}") + ### GENERATE TEST + generate_test_source_file(${test_output_file_path} ${register_test_TEST_SUCCESS}) # => returns test_source + build_test_exe_name(${test_source}) # => returns test_exe + message(STATUS "--> Creating test executable ${test_exe} with source ${test_source}") + add_executable(${test_exe} ${test_source}) + target_link_libraries(${test_exe} PRIVATE TestUtils GetTestDir) + add_test(NAME "${register_test_TEST_NAME}" COMMAND "${test_exe}") + set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT) +endfunction() + +#-------------------------------------------------------------------- +# Create and register tests in arguments +# +# Usage: +# register_tests(TEST_DIRECTORY TEST_NAME_LIST TEST_OUTPUT_FILE_LIST TEST_SUCCESS_LIST ) +# Parameters: +# TEST_DIRECTORY: path to the directory where the tests will write their output files +# TEST_NAME_LIST: list of test names +# TEST_OUTPUT_FILE_LIST: list of file names the tests should generate +# TEST_SUCCESS_LIST: list of boolean values indicating whether or not the tests should end successfully +#-------------------------------------------------------------------- +function(register_tests) + set(options) + set(oneValueArgs "TEST_DIRECTORY") + set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_LIST;TEST_SUCCESS_LIST") + ### PARSING ARGUMENTS + cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(DEFINED register_tests_KEYWORDS_MISSING_VALUES) + message( + FATAL_ERROR + "In the call to register_tests function, the keywords ${register_tests_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" + ) + endif() + if(DEFINED register_tests_UNPARSED_ARGUMENTS) + message( + FATAL_ERROR + "Following arguments are unknown to register_tests function: ${register_tests_UNPARSED_ARGUMENTS}" + ) + endif() + if(NOT DEFINED register_tests_TEST_DIRECTORY) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_DIRECTORY keyword") + endif() + if(NOT DEFINED register_tests_TEST_NAME_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword") + endif() + if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_LIST keyword") + endif() + if(NOT DEFINED register_tests_TEST_SUCCESS_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_SUCCESS_LIST keyword") + endif() + + list(LENGTH register_tests_TEST_NAME_LIST NB_TESTS) + math(EXPR MAX_INDEX "${NB_TESTS}-1") + foreach(test_index RANGE ${MAX_INDEX}) + list(GET register_tests_TEST_OUTPUT_FILE_LIST ${test_index} test_output) + list(GET register_tests_TEST_NAME_LIST ${test_index} test_name) + list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success) + register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE ${test_output} TEST_SUCCESS ${test_success}) + endforeach() +endfunction() \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp new file mode 100644 index 000000000..c84f6069a --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp @@ -0,0 +1,122 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "get_test_dir.h" + +/********************************************************************************/ +/** + * @brief Dump the content of a file to a string. + * + * @param filename : The name of the file to dump. + * @return std::string : The content of the file or empty in case of error. + */ +/********************************************************************************/ +std::string dump_file(const std::string &filename) +{ + std::filesystem::path filepath(filename); + if (!std::filesystem::exists(filepath)) + { + std::cerr << "File does not exist: " << filename << '\n'; + return {}; + } + std::ifstream ifs(filepath, std::ifstream::in); + if (!ifs) + { + std::cerr << "Failed to open file: " << filename << '\n'; + return {}; + } + std::ostringstream oss; + for (std::string line; std::getline(ifs, line);) + { + oss << line; + if (ifs.good()) + { + oss << '\n'; + } + } + if (ifs.bad()) + { + std::cerr << "Failed to read file: " << filename << '\n'; + return {}; + } + return oss.str(); +} + +/********************************************************************************/ +/** + * @brief Generate an output file containing the content of the input files + * separated by commas and enclosed in curly braces. + * + * The output file is named output_test.txt and is created in the current + * directory. + * + * The output file is in json format provided that the input files are in text + * format. + * + * @param file_names : The list of input files to dump. + * @return int : 0 if the output file was successfully generated, 1 otherwise. + */ +/********************************************************************************/ +int generate_output_file(const std::vector &file_names) +{ + std::ofstream ofs_test("output_test.txt"); + if (!ofs_test) + { + std::cerr << "Failed to open output_test.txt\n"; + return 1; + } + + ofs_test << "{\n"; + + for (auto iter{std::cbegin(file_names)}; iter != std::cend(file_names); ++iter) + { + const auto &ccontent = dump_file(*iter); + if (!ccontent.empty()) + { + ofs_test << ccontent; + } + + if (std::next(iter) != std::cend(file_names)) + { + const bool has_empty_successor = dump_file(*(std::next(iter))).empty(); + if (!has_empty_successor) + { + ofs_test << ","; + } + } + ofs_test << "\n"; + } + + ofs_test << "}\n"; + return 0; +} + +/*----------------------------------------------------------------------------*/ +/** + * @brief Main function. + * + * @return int : 0 if the output file was successfully generated, 1 otherwise. + */ +/*----------------------------------------------------------------------------*/ +int main(int, char **) +{ + auto test_dir = get_test_dir(); + std::vector test_files{}; + if (!std::filesystem::exists(test_dir)) + { + // May happen in sequential test execution if the GenerateOutputFile test is executed first + return 0; + } + for (auto const& dir_entry : std::filesystem::directory_iterator{test_dir}) + { + std::cout << "Test file " << dir_entry.path() << " detected!" << std::endl; + test_files.emplace_back(dir_entry.path()); + } + + return generate_output_file(test_files); +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in b/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in new file mode 100644 index 000000000..35c274f28 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in @@ -0,0 +1,6 @@ +#include + +std::filesystem::path get_test_dir() +{ + return {"@TESTS_DIR@"}; +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in new file mode 100644 index 000000000..89fa928f9 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in @@ -0,0 +1,12 @@ +#include "test_utils.h" +#include "get_test_dir.h" +#include + +int main() { + auto test_dir = get_test_dir(); + if (!std::filesystem::exists(test_dir)) + { + std::filesystem::create_directory(test_dir); + } + return generic_test("@test_filename@", @success@); +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp new file mode 100644 index 000000000..932b442bf --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp @@ -0,0 +1,31 @@ +#include "test_utils.h" + +#include +#include +#include + +/********************************************************************************/ +/** + * @brief Generic test function that writes a file with the test result. + * If the test is successful, the file will contain the test name and "OK". + * Otherwise, the file will contain the test name and "KO". + * + * @param test_filepath : the path to the file to write + * @param success : the test result + * @return int : 0 if the test is successful, 1 otherwise + */ +/********************************************************************************/ +int generic_test(const std::string& test_filepath, const bool success) { + std::filesystem::path test_path(test_filepath); + const auto& test_name{test_path.stem()}; + + std::ofstream outfile(test_filepath); + if (outfile.is_open()) { + outfile << test_name << " : \"" << (success ? "OK" : "KO") << "\""; + outfile.close(); + std::cout << "File written successfully." << std::endl; + } else { + std::cerr << "Error opening file." << std::endl; + } + return success ? 0 : 1; +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h new file mode 100644 index 000000000..32db4db87 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h @@ -0,0 +1,14 @@ +#ifndef TEST_UTILS_H +#define TEST_UTILS_H + +/********************************************************************************/ +/********************************************************************************/ +#include + +/********************************************************************************/ +/********************************************************************************/ +int generic_test(const std::string& test_filepath, const bool success); + +/********************************************************************************/ +/********************************************************************************/ +#endif // TEST_UTILS_H \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/runTest.ts b/test/end-to-end-tests/single-root-ctest/runTest.ts new file mode 100644 index 000000000..0118708e0 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/runTest.ts @@ -0,0 +1,34 @@ +import * as path from 'path'; + +import { runTests } from '@vscode/test-electron'; + +async function main() { + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionDevelopmentPath = path.resolve(__dirname, '../../../../'); + + // The path to the extension test runner script + // Passed to --extensionTestsPath + const extensionTestsPath = path.resolve(__dirname, './index'); + + const testWorkspace = path.resolve(extensionDevelopmentPath, 'test/end-to-end-tests/single-root-ctest/project-folder'); + + const launchArgs = ["--disable-extensions", "--disable-workspace-trust", testWorkspace]; + + const extensionTestsEnv: { [key: string]: string | undefined } = { + "CMT_TESTING": "1", + "CMT_QUIET_CONSOLE": "1", + "TEST_FILTER": process.env.TEST_FILTER ?? ".*" + }; + + // Download VS Code, unzip it and run the integration test + await runTests({ launchArgs, extensionDevelopmentPath, extensionTestsPath, extensionTestsEnv }); + } catch (err) { + console.error(err); + console.error('Failed to run tests'); + process.exit(1); + } +} + +void main(); diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts new file mode 100644 index 000000000..85e59d453 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -0,0 +1,317 @@ +/** + * This test suite will test the ctest command with different test results. + * + * Each test suite is defined in the project-folder's CMakePresets.json file through CMake cache variables: + * - TESTS_DIR: The directory where the test results will be stored + * - TESTS_OUTPUT_FILES: A list of file names that will contain the test results + * - TESTS_NAMES: A list of names of the tests to run + * - TESTS_SUCCESS: A list of the expected results of the tests + * + * All of those lists should have the same size. + * + * Each invocation of the ctest command will run the tests defined in the TESTS_NAMES list, storing their TESTS_OUTPUT_FILES + * in the TESTS_DIR directory and will make the test command ends according to the TESTS_SUCCESS list. + * After each invocation of the ctest command, the test results will be concatenated in the output_test.txt file in JSon format so that + * the test suite can check the results. + * + * Each test suite will have the following tests: + * - Run ctest without parallel jobs + * - Run ctest with parallel jobs + * - Run ctest without parallel jobs. Use test suite delimiter + * - Run ctest with parallel jobs. Use test suite delimiter + */ +import { fs } from '@cmt/pr'; +import { + DefaultEnvironment, + expect +} from '@test/util'; +import * as path from 'path'; +import * as vscode from 'vscode'; +import paths from '@cmt/paths'; + +/** + * Given a CMakePresets.json content, this function will return the configure preset with the given name + * + * @param presets_content: The content of the CMakePresets.json file as a JSON object + * @param preset_name: The name of the configure preset to find + * @returns The configure preset with the given name or undefined if not found + */ +async function getSpecificPreset(presets_content: any, preset_name: string) { + expect(presets_content['configurePresets']).to.not.eq('', "Unable to find configurePresets section!"); + const all_conf_presets = presets_content['configurePresets']; + for (let index = 0; index < all_conf_presets.length; index++) { + const conf_preset = all_conf_presets[index]; + expect(conf_preset['name']).to.not.eq('', "Unable to find name of the current configure preset!"); + if (conf_preset['name'] === preset_name) { + return conf_preset; + } + } + return undefined; +} + +/** + * This function removes the test result files and the test directory + */ +async function cleanUpTestResultFiles(test_env: DefaultEnvironment, configure_preset: string) { + const used_preset = await getSpecificPreset(await getCMakePresetsAsJson(test_env), configure_preset); + expect(used_preset['cacheVariables']['TESTS_DIR']).to.not.eq('', "Unable to find the TESTS_DIR cache variable in the configure preset!"); + const test_dir_path = used_preset['cacheVariables']['TESTS_DIR']; + expect("/" + test_dir_path.split('/')[1]).to.eq(paths.tmpDir, `WARNING: The TESTS_DIR variable (${test_dir_path}) does not seem to point to the temporary directory (${paths.tmpDir})!`); + await fs.rmdir(test_dir_path); + const output_test_path: string = path.join(test_env.projectFolder.location, test_env.buildLocation, test_env.executableResult); + if (await fs.exists(output_test_path)) { + await fs.unlink(output_test_path); + } +} + +/** + * + * @returns The content of the CMakePresets.json file as a JSON object + */ +async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { + const preset_location: string = path.join(test_env.projectFolder.location, "CMakePresets.json"); + expect(fs.existsSync(preset_location)).to.eq(true, `CMakePresets.json file ${preset_location} was not found`); + const content = await fs.readFile(preset_location); + expect(content.toLocaleString()).to.not.eq(''); + return JSON.parse(content.toString()); +} + +/** + * This function will setup the test environment by setting the configure, build, test, package and workflow presets + * before building the project + * + * @param configure_preset: The name of the configure preset to use + * @returns The test environment + */ +async function commonSetup(configure_preset: string) { + const build_loc = 'build'; + const exe_res = 'output_test.txt'; + + const test_env: DefaultEnvironment = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); + test_env.projectFolder.buildDirectory.clear(); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + await vscode.commands.executeCommand('cmake.setConfigurePreset', configure_preset); + await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); + await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); + await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); + await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); + + await vscode.commands.executeCommand('cmake.build'); + + return test_env; +} + +suite('Ctest: 2 successfull tests', () => { + let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "2Successes"; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + testEnv = await commonSetup(usedConfigPreset); + }); + + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + teardown(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + }); + + test('Run ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); +}); + +suite('Ctest: 2 successfull tests 1 failing test', () => { + let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "2Successes1Failure"; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + testEnv = await commonSetup(usedConfigPreset); + }); + + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + teardown(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + }); + + test('Run ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); + }).timeout(100000); +}); + +suite('Ctest: 3 failing tests', () => { + let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "3Failures"; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + testEnv = await commonSetup(usedConfigPreset); + }); + + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + teardown(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + }); + + test('Run ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); +});