From 9c1b1f22ea20a57ad6de3607cced7a1e55f0b11b Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 8 Nov 2024 16:51:29 +0100 Subject: [PATCH 01/59] Adds a new end to end test directory devoted to ctest tests --- .../single-root-ctest/index.ts | 54 ++++++++++++++++ .../project-folder/.vscode/cmake-kits.json | 56 +++++++++++++++++ .../.vscode/cmake-variants.json | 29 +++++++++ .../project-folder/.vscode/settings.json | 5 ++ .../project-folder/CMakeLists.txt | 43 +++++++++++++ .../project-folder/CMakePresets.json | 18 ++++++ .../project-folder/CMakeUserPresets.json | 53 ++++++++++++++++ .../project-folder/TestCacheInit.cmake | 1 + .../include/IncludedPresets1.json | 19 ++++++ .../include/IncludedPresets2.json | 16 +++++ .../single-root-ctest/project-folder/main.cpp | 38 +++++++++++ .../project-folder/test-toolchain.cmake | 2 + .../single-root-ctest/runTest.ts | 34 ++++++++++ .../test/ctest-run-tests.test.ts | 63 +++++++++++++++++++ 14 files changed, 431 insertions(+) create mode 100644 test/end-to-end-tests/single-root-ctest/index.ts create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/main.cpp create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake create mode 100644 test/end-to-end-tests/single-root-ctest/runTest.ts create mode 100644 test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts diff --git a/test/end-to-end-tests/single-root-ctest/index.ts b/test/end-to-end-tests/single-root-ctest/index.ts new file mode 100644 index 000000000..9391b101c --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/index.ts @@ -0,0 +1,54 @@ +// eslint-disable-next-line import/no-unassigned-import +import 'module-alias/register'; + +import * as path from 'path'; +import * as Mocha from 'mocha'; +import * as glob from 'glob'; +import { Logger } from '@cmt/logging'; + +export function run(): Promise { + // Create the mocha test + const mocha = new Mocha({ + ui: 'tdd', + color: true + }); + + const testsRoot = __dirname; + + return new Promise((c, e) => { + glob('**/*.test.js', { cwd: testsRoot }, (err, files) => { + if (err) { + return e(err); + } + + // Add files to the test suite + const regex = process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : /.*/; + files.forEach(f => { + if (regex.test(f)) { + mocha.addFile(path.resolve(testsRoot, f)); + } + }); + + try { + // Run the mocha test + mocha.timeout(100000); + + // Log the name of each test before it starts. + const beforeEach: Mocha.Func = function (this: Mocha.Context, done: Mocha.Done) { + Logger.logTestName(this.currentTest?.parent?.title, this.currentTest?.title); + done(); + }; + mocha.rootHooks({beforeEach}); + mocha.run(failures => { + if (failures > 0) { + e(new Error(`${failures} tests failed.`)); + } else { + c(); + } + }); + } catch (err) { + e(err); + } + }); + }); +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json new file mode 100644 index 000000000..954a8c841 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json @@ -0,0 +1,56 @@ +[ + { + "name": "Test Toolchain", + "toolchainFile": "${workspaceFolder}\\test-toolchain.cmake" + }, + { + "name": "Generator switch test GCC Make", + "compilers": { + "CXX": "g++", + "C": "gcc" + }, + "preferredGenerator": { + "name": "Unix Makefiles" + } + }, + { + "name": "Generator switch test GCC Ninja", + "compilers": { + "CXX": "g++", + "C": "gcc" + }, + "preferredGenerator": { + "name": "Ninja" + } + }, + { + "name": "Generator switch test GCC no generator", + "compilers": { + "CXX": "g++", + "C": "gcc" + } + }, + { + "name": "Generator switch test VS 2019", + "visualStudio": "VisualStudio.16.0", + "visualStudioArchitecture": "x86", + "preferredGenerator": { + "name": "Visual Studio 16 2019", + "platform": "win32", + "toolset": "host=x86" + } + }, + { + "name": "Generator switch test VS 2019 Ninja", + "visualStudio": "VisualStudio.16.0", + "visualStudioArchitecture": "x86", + "preferredGenerator": { + "name": "Ninja" + } + }, + { + "name": "Generator switch test VS 2019 no generator", + "visualStudio": "VisualStudio.16.0", + "visualStudioArchitecture": "x86" + } +] diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json new file mode 100644 index 000000000..2041ef42a --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json @@ -0,0 +1,29 @@ +{ + "buildType": { + "default": "debug-label", + "choices": { + "debug-label": { + "short": "debug-label short", + "buildType": "Debug" + }, + "not-debug": { + "short": "not-debug short", + "buildType": "Release" + } + } + }, + "otherVariant": { + "default": "option1", + "choices": { + "option1": { + "short": "option1 short", + "env": { + "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" + } + }, + "option2": { + "short": "option2 short" + } + } + } +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json new file mode 100644 index 000000000..ff3d48f17 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "cmake.buildDirectory": "${workspaceFolder}/build", + "cmake.useCMakePresets": "always", + "cmake.configureOnOpen": false +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt new file mode 100644 index 000000000..739516ab4 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -0,0 +1,43 @@ +cmake_minimum_required(VERSION 3.0.0) +project(TestBuildProcess VERSION 0.1.0) + +set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable") + +add_executable(TestBuildProcess main.cpp) +set_property(TARGET TestBuildProcess PROPERTY CXX_STANDARD 98) + +add_custom_command( + TARGET TestBuildProcess + POST_BUILD + COMMAND $ > output.txt + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Running in ${CMAKE_CURRENT_BINARY_DIR}" +) + +set(configureEnvironment "$ENV{_CONFIGURE_ENV}" CACHE STRING "configureEnvironment") +set(buildEnvironment "$ENV{_BUILD_ENV}" CACHE STRING "buildEnvironment") +set(environment "$ENV{_ENV}" CACHE STRING "environment") + +set(variantEnv "$ENV{TEST_VARIANT_ENV}" CACHE STRING "variantEnv") + +add_definitions( + -D_CMAKE_VERSION="${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" + -D_GENERATOR="${CMAKE_GENERATOR}" + -D_CRT_SECURE_NO_WARNINGS +) + +target_compile_definitions(TestBuildProcess PRIVATE + "CMT_COOKIE=\"${CMT_COOKIE}\"" + "C_COMPILER_ID=\"${CMAKE_C_COMPILER_ID}\"" +) + +add_custom_target(runTestTarget DEPENDS TestBuildProcess + COMMAND $ > output_target.txt + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + COMMENT "Run test target" +) + +enable_testing() + +add_test(NAME "Suite1.TestA" COMMAND "cmake -E echo Suite1.TestA") +add_test(NAME "Suite2.TestB" COMMAND "cmake -E echo Suite2.TestB") diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json new file mode 100644 index 000000000..e78e95aae --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -0,0 +1,18 @@ +{ + "version": 2, + "configurePresets": [ + { + "name": "Linux1", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_TOOLCHAIN_FILE": { + "type": "FILEPATH", + "value": "${workspaceFolder}/test-toolchain.cmake" + } + } + } + ] +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json new file mode 100644 index 000000000..e85796119 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json @@ -0,0 +1,53 @@ +{ + "version": 4, + "include": [ + "include/IncludedPresets1.json" + ], + "configurePresets": [ + { + "name": "LinuxUser1", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Unix Makefiles", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_COMPILER": "gcc", + "CMAKE_CXX_COMPILER": "g++" + }, + "environment": { + "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" + } + }, + { + "name": "WindowsUser1", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Visual Studio 16 2019", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_COMPILER": "cl.exe", + "CMAKE_CXX_COMPILER": "cl.exe" + }, + "environment": { + "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" + } + }, + { + "name": "NoGenerator", + "description": "Skips setting a generator", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_COMPILER": "gcc", + "CMAKE_CXX_COMPILER": "g++" + }, + "environment": { + "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" + } + }, + { + "name": "TestInheritFromPreset", + "inherits": "Linux1" + } + ] +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake new file mode 100644 index 000000000..df752c30f --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake @@ -0,0 +1 @@ +set(CMT_COOKIE cache-init-cookie CACHE STRING "Cache-init forced cookie value") diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json new file mode 100644 index 000000000..e0dfc3f3d --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json @@ -0,0 +1,19 @@ +{ + "version": 4, + "include": [ + "IncludedPresets2.json" + ], + "configurePresets": [ + { + "name": "LinuxIncluded1", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Unix Makefiles", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_COMPILER": "gcc", + "CMAKE_CXX_COMPILER": "g++" + } + } + ] +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json new file mode 100644 index 000000000..8faab3d3b --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json @@ -0,0 +1,16 @@ +{ + "version": 4, + "configurePresets": [ + { + "name": "LinuxIncluded2", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Unix Makefiles", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "CMAKE_C_COMPILER": "gcc", + "CMAKE_CXX_COMPILER": "g++" + } + } + ] +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp new file mode 100644 index 000000000..41c922ad9 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include + +#ifndef _CMAKE_VERSION + #define _CMAKE_VERSION "0.0" +#endif + +#ifndef _GENERATOR + #define _GENERATOR "" +#endif + +std::string getCompilerName() { + return C_COMPILER_ID; +} + +std::string get_env_var(const std::string& key) { + const char* env = std::getenv(key.c_str()); + return env != NULL ? env : ""; +} + +int main(int, char**) { + std::cout << "{\n"; + std::cout << " \"compiler\": \"" << getCompilerName() << "\",\n"; + std::cout << " \"cookie\": \"" CMT_COOKIE "\",\n"; + std::cout << " \"cmake-version\": \"" << _CMAKE_VERSION << "\",\n"; + std::cout << " \"cmake-generator\": \"" << _GENERATOR << "\",\n"; + std::cout << " \"configure-env\": \"" << get_env_var("_CONFIGURE_ENV") << "\",\n"; + std::cout << " \"build-env\": \"" << get_env_var("_BUILD_ENV") << "\",\n"; + std::cout << " \"env\": \"" << get_env_var("_ENV") << "\"\n"; + std::cout << "}\n"; + + std::ofstream ofs ("test.txt", std::ofstream::out); + ofs << "{\n"; + ofs << " \"cookie\": \"" CMT_COOKIE "\",\n"; + ofs << "}\n"; +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake new file mode 100644 index 000000000..acc40f85d --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake @@ -0,0 +1,2 @@ +# This file is only here as a test toolchain! +message("Hello from test-toolchain.cmake") diff --git a/test/end-to-end-tests/single-root-ctest/runTest.ts b/test/end-to-end-tests/single-root-ctest/runTest.ts new file mode 100644 index 000000000..5f26a42b0 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/runTest.ts @@ -0,0 +1,34 @@ +import * as path from 'path'; + +import { runTests } from '@vscode/test-electron'; + +async function main() { + try { + // The folder containing the Extension Manifest package.json + // Passed to `--extensionDevelopmentPath` + const extensionDevelopmentPath = path.resolve(__dirname, '../../../../'); + + // The path to the extension test runner script + // Passed to --extensionTestsPath + const extensionTestsPath = path.resolve(__dirname, './index'); + + const testWorkspace = path.resolve(extensionDevelopmentPath, 'test/end-to-end-tests/single-root-ctest/project-folder'); + + const launchArgs = ["--disable-extensions", "--disable-workspace-trust", testWorkspace]; + + const extensionTestsEnv: { [key: string]: string | undefined } = { + "CMT_TESTING": "0", + "CMT_QUIET_CONSOLE": "1", + "TEST_FILTER": process.env.TEST_FILTER ?? ".*" + }; + + // Download VS Code, unzip it and run the integration test + await runTests({ launchArgs, extensionDevelopmentPath, extensionTestsPath, extensionTestsEnv }); + } catch (err) { + console.error(err); + console.error('Failed to run tests'); + process.exit(1); + } +} + +void main(); diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts new file mode 100644 index 000000000..bd82a4395 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -0,0 +1,63 @@ +import { fs } from '@cmt/pr'; +import { + clearExistingKitConfigurationFile, + DefaultEnvironment, + expect +} from '@test/util'; +import * as path from 'path'; +import * as vscode from 'vscode'; + +suite('Ctest run tests', () => { + let testEnv: DefaultEnvironment; + let compdb_cp_path: string; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + + const build_loc = 'build'; + const exe_res = 'output.txt'; + + // CMakePresets.json and CMakeUserPresets.json exist so will use presets by default + testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); + compdb_cp_path = path.join(testEnv.projectFolder.location, 'compdb_cp.json'); + + await clearExistingKitConfigurationFile(); + }); + + setup(async function (this: Mocha.Context) { + this.timeout(100000); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + // await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); + await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); + await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); + await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); + await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); + + await vscode.commands.executeCommand('cmake.build'); + + // testEnv.projectFolder.buildDirectory.clear(); + }); + + teardown(async function (this: Mocha.Context) { + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + if (await fs.exists(compdb_cp_path)) { + await fs.unlink(compdb_cp_path); + } + }); + + test('Test of ctest', async () => { + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['cookie']).to.eq('passed-cookie'); + }).timeout(100000); + +}); From e101ea01ec33029b11a5be87858be4845f4e8401 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 8 Nov 2024 16:52:31 +0100 Subject: [PATCH 02/59] Fix non stopping breakpoints --- .vscode/launch.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index fecf99ad5..3b85099ec 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -105,6 +105,7 @@ ], "sourceMaps": true, "outFiles": [ + "${workspaceFolder}/dist/**/*.js", "${workspaceFolder}/out/*", "${workspaceFolder}/out/src/**", "${workspaceFolder}/out/test/*", From 072859fa1c101769210597e8fb73929b847404e8 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 8 Nov 2024 16:52:57 +0100 Subject: [PATCH 03/59] Adds scripts entry for new end to end ctest --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 20bc2c09e..9b89c3c97 100644 --- a/package.json +++ b/package.json @@ -3698,6 +3698,7 @@ "integrationTests": "yarn run pretest && node ./out/test/integration-tests/runTest.js", "endToEndTestsSuccessfulBuild": "yarn run pretest && node ./out/test/end-to-end-tests/successful-build/runTest.js", "endToEndTestsSingleRoot": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-UI/runTest.js", + "endToEndTestsSingleRootCTest": "yarn run pretest && node ./out/test/end-to-end-tests/single-root-ctest/runTest.js", "endToEndTestsMultiRoot": "yarn run pretest && node ./out/test/end-to-end-tests/multi-root-UI/runTest.js", "backendTests": "node ./node_modules/mocha/bin/_mocha -u tdd --timeout 999999 --colors -r ts-node/register -r tsconfig-paths/register ./test/unit-tests/backend/**/*.test.ts", "build-product-icon-font": "yarn --cwd ./tools/product-icon-font-generator/ install && yarn --cwd ./tools/product-icon-font-generator/ build && node ./tools/product-icon-font-generator/dist/index.js --source-directory ./res/product-icons/ --output-directory ./res/ --woff2" From f562055915b3e444816553e54336d96a4723bcaa Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 8 Nov 2024 16:55:00 +0100 Subject: [PATCH 04/59] Adds configurations for new end to end ctest test --- .vscode/launch.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3b85099ec..8947ccf5e 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -119,6 +119,32 @@ "TEST_FILTER": ".*" }, }, + { + "name": "Run single-root-ctest Tests", + "type": "extensionHost", + "request": "launch", + "runtimeExecutable": "${execPath}", + "args": [ + "${workspaceFolder}/test/end-to-end-tests/single-root-ctest/project-folder", + "--extensionDevelopmentPath=${workspaceFolder}", + "--extensionTestsPath=${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/index" + ], + "sourceMaps": true, + "outFiles": [ + "${workspaceFolder}/dist/**/*.js", + "${workspaceFolder}/out/*", + "${workspaceFolder}/out/src/**", + "${workspaceFolder}/out/test/*", + "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/*", + "${workspaceFolder}/out/test/end-to-end-tests/single-root-ctest/test/*" + ], + "preLaunchTask": "Pretest", + "env": { + "CMT_TESTING": "0", + "CMT_QUIET_CONSOLE": "1", + "TEST_FILTER": ".*" + }, + }, { "name": "Run multi-root-UI Tests", "type": "extensionHost", From 24c9e3f98695e59ac3edcf1508bb0131896d1bd4 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 8 Nov 2024 16:55:54 +0100 Subject: [PATCH 05/59] Do not forbid test refreshing even in test mode --- src/ctest.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ctest.ts b/src/ctest.ts index c2fd575b4..cad1a8b50 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -710,10 +710,10 @@ export class CTestDriver implements vscode.Disposable { return -1; } - if (util.isTestMode()) { - // ProjectController can't be initialized in test mode, so we don't have a usable test explorer - return 0; - } + // if (util.isTestMode()) { + // // ProjectController can't be initialized in test mode, so we don't have a usable test explorer + // return 0; + // } const initializedTestExplorer = this.ensureTestExplorerInitialized(); const sourceDir = util.platformNormalizePath(driver.sourceDir); From 2e83da265436233d9b92f09b3e1bdf8264bbcf52 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 12 Nov 2024 12:34:05 +0100 Subject: [PATCH 06/59] Fix end to end test devoted to ctest --- .../project-folder/CMakeLists.txt | 54 +++++--------- .../project-folder/CMakePresets.json | 6 +- .../project-folder/CMakeUserPresets.json | 53 -------------- .../project-folder/TestCacheInit.cmake | 1 - .../include/IncludedPresets1.json | 19 ----- .../include/IncludedPresets2.json | 16 ----- .../single-root-ctest/project-folder/main.cpp | 70 +++++++++++-------- .../project-folder/test-toolchain.cmake | 2 - .../project-folder/test_a.cpp | 14 ++++ .../project-folder/test_b.cpp | 14 ++++ .../project-folder/tests_cleanup.cpp | 32 +++++++++ 11 files changed, 118 insertions(+), 163 deletions(-) delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 739516ab4..19e454988 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -1,43 +1,23 @@ cmake_minimum_required(VERSION 3.0.0) -project(TestBuildProcess VERSION 0.1.0) +project(TestCTestProcess VERSION 0.1.0) -set(CMT_COOKIE passed-cookie CACHE STRING "Cookie to be written by the main executable") +add_executable(ConcatTestOutput main.cpp) +set_target_properties(ConcatTestOutput PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) +add_executable(TestA test_a.cpp) +add_executable(TestB test_b.cpp) +add_executable(TestCleanup tests_cleanup.cpp) +set_target_properties(TestCleanup PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) -add_executable(TestBuildProcess main.cpp) -set_property(TARGET TestBuildProcess PROPERTY CXX_STANDARD 98) - -add_custom_command( - TARGET TestBuildProcess - POST_BUILD - COMMAND $ > output.txt - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Running in ${CMAKE_CURRENT_BINARY_DIR}" -) - -set(configureEnvironment "$ENV{_CONFIGURE_ENV}" CACHE STRING "configureEnvironment") -set(buildEnvironment "$ENV{_BUILD_ENV}" CACHE STRING "buildEnvironment") -set(environment "$ENV{_ENV}" CACHE STRING "environment") - -set(variantEnv "$ENV{TEST_VARIANT_ENV}" CACHE STRING "variantEnv") - -add_definitions( - -D_CMAKE_VERSION="${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION}" - -D_GENERATOR="${CMAKE_GENERATOR}" - -D_CRT_SECURE_NO_WARNINGS -) - -target_compile_definitions(TestBuildProcess PRIVATE - "CMT_COOKIE=\"${CMT_COOKIE}\"" - "C_COMPILER_ID=\"${CMAKE_C_COMPILER_ID}\"" -) +enable_testing() -add_custom_target(runTestTarget DEPENDS TestBuildProcess - COMMAND $ > output_target.txt - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - COMMENT "Run test target" -) +add_test(NAME "Suite1.TestA" COMMAND TestA) +add_test(NAME "Suite2.TestB" COMMAND TestB) +add_test(NAME "Concatenate_Test_Output" COMMAND ConcatTestOutput) +add_test(NAME "Clean_Up_Test" COMMAND TestCleanup) -enable_testing() +set_tests_properties("Clean_Up_Test" PROPERTIES FIXTURES_CLEANUP GENOUT) +set_tests_properties("Concatenate_Test_Output" PROPERTIES FIXTURES_CLEANUP GENOUT) +set_tests_properties("Clean_Up_Test" PROPERTIES DEPENDS Concatenate_Test_Output) -add_test(NAME "Suite1.TestA" COMMAND "cmake -E echo Suite1.TestA") -add_test(NAME "Suite2.TestB" COMMAND "cmake -E echo Suite2.TestB") +set_tests_properties("Suite1.TestA" PROPERTIES FIXTURES_REQUIRED GENOUT) +set_tests_properties("Suite2.TestB" PROPERTIES FIXTURES_REQUIRED GENOUT) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index e78e95aae..b6a305585 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -7,11 +7,7 @@ "generator": "Ninja", "binaryDir": "${workspaceFolder}/build", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_TOOLCHAIN_FILE": { - "type": "FILEPATH", - "value": "${workspaceFolder}/test-toolchain.cmake" - } + "CMAKE_BUILD_TYPE": "Debug" } } ] diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json deleted file mode 100644 index e85796119..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeUserPresets.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "version": 4, - "include": [ - "include/IncludedPresets1.json" - ], - "configurePresets": [ - { - "name": "LinuxUser1", - "description": "Sets generator, build and install directory, vcpkg", - "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++" - }, - "environment": { - "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" - } - }, - { - "name": "WindowsUser1", - "description": "Sets generator, build and install directory, vcpkg", - "generator": "Visual Studio 16 2019", - "binaryDir": "${workspaceFolder}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_C_COMPILER": "cl.exe", - "CMAKE_CXX_COMPILER": "cl.exe" - }, - "environment": { - "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" - } - }, - { - "name": "NoGenerator", - "description": "Skips setting a generator", - "binaryDir": "${workspaceFolder}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++" - }, - "environment": { - "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" - } - }, - { - "name": "TestInheritFromPreset", - "inherits": "Linux1" - } - ] -} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake deleted file mode 100644 index df752c30f..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/TestCacheInit.cmake +++ /dev/null @@ -1 +0,0 @@ -set(CMT_COOKIE cache-init-cookie CACHE STRING "Cache-init forced cookie value") diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json deleted file mode 100644 index e0dfc3f3d..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets1.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "version": 4, - "include": [ - "IncludedPresets2.json" - ], - "configurePresets": [ - { - "name": "LinuxIncluded1", - "description": "Sets generator, build and install directory, vcpkg", - "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++" - } - } - ] -} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json b/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json deleted file mode 100644 index 8faab3d3b..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/include/IncludedPresets2.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "version": 4, - "configurePresets": [ - { - "name": "LinuxIncluded2", - "description": "Sets generator, build and install directory, vcpkg", - "generator": "Unix Makefiles", - "binaryDir": "${workspaceFolder}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug", - "CMAKE_C_COMPILER": "gcc", - "CMAKE_CXX_COMPILER": "g++" - } - } - ] -} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index 41c922ad9..44bb70f77 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -1,38 +1,48 @@ -#include #include +#include #include +#include #include -#ifndef _CMAKE_VERSION - #define _CMAKE_VERSION "0.0" -#endif - -#ifndef _GENERATOR - #define _GENERATOR "" -#endif - -std::string getCompilerName() { - return C_COMPILER_ID; -} - -std::string get_env_var(const std::string& key) { - const char* env = std::getenv(key.c_str()); - return env != NULL ? env : ""; +/********************************************************************************/ +/** + * @brief Dump the content of a file to a string. + * + * @param filename : The name of the file to dump. + * @return std::string : The content of the file. + */ +/********************************************************************************/ +std::string dump_file(const std::string& filename) { + std::filesystem::path filepath(filename); + if (!std::filesystem::exists(filepath)) { + std::cerr << "File does not exist: " << filename << '\n'; + return {}; + } + std::ifstream ifs (filepath, std::ifstream::in); + if (!ifs) { + std::cerr << "Failed to open file: " << filename << '\n'; + return {}; + } + std::ostringstream oss; + for (std::string line; std::getline(ifs, line);) { + oss << line << '\n'; + } + if (ifs.bad()) { + std::cerr << "Failed to read file: " << filename << '\n'; + return {}; + } + return oss.str(); } int main(int, char**) { - std::cout << "{\n"; - std::cout << " \"compiler\": \"" << getCompilerName() << "\",\n"; - std::cout << " \"cookie\": \"" CMT_COOKIE "\",\n"; - std::cout << " \"cmake-version\": \"" << _CMAKE_VERSION << "\",\n"; - std::cout << " \"cmake-generator\": \"" << _GENERATOR << "\",\n"; - std::cout << " \"configure-env\": \"" << get_env_var("_CONFIGURE_ENV") << "\",\n"; - std::cout << " \"build-env\": \"" << get_env_var("_BUILD_ENV") << "\",\n"; - std::cout << " \"env\": \"" << get_env_var("_ENV") << "\"\n"; - std::cout << "}\n"; - - std::ofstream ofs ("test.txt", std::ofstream::out); - ofs << "{\n"; - ofs << " \"cookie\": \"" CMT_COOKIE "\",\n"; - ofs << "}\n"; + std::ofstream ofs_test("output_test.txt"); + if (!ofs_test) { + std::cerr << "Failed to open output_test.txt\n"; + return 1; + } + ofs_test << "{\n"; + ofs_test << dump_file("test_a.txt"); + ofs_test << dump_file("test_b.txt"); + ofs_test << "}\n"; + return 0; } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake deleted file mode 100644 index acc40f85d..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test-toolchain.cmake +++ /dev/null @@ -1,2 +0,0 @@ -# This file is only here as a test toolchain! -message("Hello from test-toolchain.cmake") diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp new file mode 100644 index 000000000..61761730f --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp @@ -0,0 +1,14 @@ +#include +#include + +int main() { + std::ofstream outfile("test_a.txt"); + if (outfile.is_open()) { + outfile << "{test_a: OK}"; + outfile.close(); + std::cout << "File written successfully." << std::endl; + } else { + std::cerr << "Error opening file." << std::endl; + } + return 0; +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp new file mode 100644 index 000000000..f20b040f2 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp @@ -0,0 +1,14 @@ +#include +#include + +int main() { + std::ofstream outfile("test_b.txt"); + if (outfile.is_open()) { + outfile << "{test_b: OK}"; + outfile.close(); + std::cout << "File written successfully." << std::endl; + } else { + std::cerr << "Error opening file." << std::endl; + } + return 0; +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp new file mode 100644 index 000000000..29df6806b --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp @@ -0,0 +1,32 @@ +#include +#include + +void delete_file(const std::string &file_name) +{ + try + { + if (std::filesystem::remove(file_name)) + { + std::cout << file_name << " was deleted successfully.\n"; + } + else + { + std::cout << file_name << " does not exist.\n"; + } + } + catch (const std::filesystem::filesystem_error &e) + { + std::cerr << "Filesystem error: " << e.what() << '\n'; + } +} + +int main() +{ + const auto file_a = "test_a.txt"; + const auto file_b = "test_b.txt"; + + delete_file(file_a); + delete_file(file_b); + + return 0; +} \ No newline at end of file From fa8a41980d547827388d9cb3a4ae09c71dda7b45 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 12 Nov 2024 16:26:40 +0100 Subject: [PATCH 07/59] Test is functionnal when running through debug --- src/cmakeProject.ts | 2 +- .../project-folder/CMakeLists.txt | 6 +++--- .../single-root-ctest/project-folder/main.cpp | 15 +++++++++++++-- .../single-root-ctest/project-folder/test_a.cpp | 2 +- .../single-root-ctest/project-folder/test_b.cpp | 2 +- .../test/ctest-run-tests.test.ts | 8 +++----- 6 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/cmakeProject.ts b/src/cmakeProject.ts index e36932d5e..479ce1400 100644 --- a/src/cmakeProject.ts +++ b/src/cmakeProject.ts @@ -2318,7 +2318,7 @@ export class CMakeProject { async ctest(fromWorkflow: boolean = false): Promise { const drv = await this.preTest(fromWorkflow); const retc = await this.cTestController.runCTest(drv); - return (retc) ? 0 : -1; + return (retc === 0) ? 0 : -1; } async cpack(fromWorkflow: boolean = false): Promise { diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 19e454988..5c89c96e4 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -13,11 +13,11 @@ enable_testing() add_test(NAME "Suite1.TestA" COMMAND TestA) add_test(NAME "Suite2.TestB" COMMAND TestB) add_test(NAME "Concatenate_Test_Output" COMMAND ConcatTestOutput) -add_test(NAME "Clean_Up_Test" COMMAND TestCleanup) +add_test(NAME "Test_Cleanup" COMMAND TestCleanup) -set_tests_properties("Clean_Up_Test" PROPERTIES FIXTURES_CLEANUP GENOUT) +set_tests_properties("Test_Cleanup" PROPERTIES FIXTURES_CLEANUP GENOUT) set_tests_properties("Concatenate_Test_Output" PROPERTIES FIXTURES_CLEANUP GENOUT) -set_tests_properties("Clean_Up_Test" PROPERTIES DEPENDS Concatenate_Test_Output) +set_tests_properties("Test_Cleanup" PROPERTIES DEPENDS Concatenate_Test_Output) set_tests_properties("Suite1.TestA" PROPERTIES FIXTURES_REQUIRED GENOUT) set_tests_properties("Suite2.TestB" PROPERTIES FIXTURES_REQUIRED GENOUT) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index 44bb70f77..8ed84658d 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -25,7 +25,10 @@ std::string dump_file(const std::string& filename) { } std::ostringstream oss; for (std::string line; std::getline(ifs, line);) { - oss << line << '\n'; + oss << line; + if (ifs.good()) { + oss <<'\n'; + } } if (ifs.bad()) { std::cerr << "Failed to read file: " << filename << '\n'; @@ -40,8 +43,16 @@ int main(int, char**) { std::cerr << "Failed to open output_test.txt\n"; return 1; } + + const auto& content_a = dump_file("test_a.txt"); + const auto& content_b = dump_file("test_b.txt"); + ofs_test << "{\n"; - ofs_test << dump_file("test_a.txt"); + if (!content_a.empty()) + { + ofs_test << dump_file("test_a.txt"); + ofs_test << ",\n"; + } ofs_test << dump_file("test_b.txt"); ofs_test << "}\n"; return 0; diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp index 61761730f..ad5e2679c 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp @@ -4,7 +4,7 @@ int main() { std::ofstream outfile("test_a.txt"); if (outfile.is_open()) { - outfile << "{test_a: OK}"; + outfile << "\"test_a\": \"OK\""; outfile.close(); std::cout << "File written successfully." << std::endl; } else { diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp index f20b040f2..0610c8807 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp @@ -4,7 +4,7 @@ int main() { std::ofstream outfile("test_b.txt"); if (outfile.is_open()) { - outfile << "{test_b: OK}"; + outfile << "\"test_b\": \"OK\""; outfile.close(); std::cout << "File written successfully." << std::endl; } else { diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index bd82a4395..e6f769f5f 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -15,7 +15,7 @@ suite('Ctest run tests', () => { this.timeout(100000); const build_loc = 'build'; - const exe_res = 'output.txt'; + const exe_res = 'output_test.txt'; // CMakePresets.json and CMakeUserPresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); @@ -28,7 +28,6 @@ suite('Ctest run tests', () => { this.timeout(100000); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); - // await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); @@ -37,8 +36,6 @@ suite('Ctest run tests', () => { await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); await vscode.commands.executeCommand('cmake.build'); - - // testEnv.projectFolder.buildDirectory.clear(); }); teardown(async function (this: Mocha.Context) { @@ -57,7 +54,8 @@ suite('Ctest run tests', () => { expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); - expect(result['cookie']).to.eq('passed-cookie'); + expect(result['test_a']).to.eq('OK'); + expect(result['test_b']).to.eq('OK'); }).timeout(100000); }); From b4d2839897a1f0da08ab5c04fbaeaf2e5159f6ec Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 13:48:16 +0100 Subject: [PATCH 08/59] Add "allowParallelJobs" in the config --- .../single-root-ctest/test/ctest-run-tests.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index e6f769f5f..8b7ce06b7 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -28,6 +28,7 @@ suite('Ctest run tests', () => { this.timeout(100000); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', 'true'); await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); From 237079bff6ba90235ba691f7aeef3f8e1648cd73 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 15:30:33 +0100 Subject: [PATCH 09/59] Removes useless files --- .../project-folder/.vscode/cmake-kits.json | 56 ------------------- .../.vscode/cmake-variants.json | 29 ---------- 2 files changed, 85 deletions(-) delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json deleted file mode 100644 index 954a8c841..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-kits.json +++ /dev/null @@ -1,56 +0,0 @@ -[ - { - "name": "Test Toolchain", - "toolchainFile": "${workspaceFolder}\\test-toolchain.cmake" - }, - { - "name": "Generator switch test GCC Make", - "compilers": { - "CXX": "g++", - "C": "gcc" - }, - "preferredGenerator": { - "name": "Unix Makefiles" - } - }, - { - "name": "Generator switch test GCC Ninja", - "compilers": { - "CXX": "g++", - "C": "gcc" - }, - "preferredGenerator": { - "name": "Ninja" - } - }, - { - "name": "Generator switch test GCC no generator", - "compilers": { - "CXX": "g++", - "C": "gcc" - } - }, - { - "name": "Generator switch test VS 2019", - "visualStudio": "VisualStudio.16.0", - "visualStudioArchitecture": "x86", - "preferredGenerator": { - "name": "Visual Studio 16 2019", - "platform": "win32", - "toolset": "host=x86" - } - }, - { - "name": "Generator switch test VS 2019 Ninja", - "visualStudio": "VisualStudio.16.0", - "visualStudioArchitecture": "x86", - "preferredGenerator": { - "name": "Ninja" - } - }, - { - "name": "Generator switch test VS 2019 no generator", - "visualStudio": "VisualStudio.16.0", - "visualStudioArchitecture": "x86" - } -] diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json deleted file mode 100644 index 2041ef42a..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/cmake-variants.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "buildType": { - "default": "debug-label", - "choices": { - "debug-label": { - "short": "debug-label short", - "buildType": "Debug" - }, - "not-debug": { - "short": "not-debug short", - "buildType": "Release" - } - } - }, - "otherVariant": { - "default": "option1", - "choices": { - "option1": { - "short": "option1 short", - "env": { - "TEST_VARIANT_ENV": "0cbfb6ae-f2ec-4017-8ded-89df8759c502" - } - }, - "option2": { - "short": "option2 short" - } - } - } -} From 6c4642f1394c31e31310fe181ebf59bc4387da9a Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 15:32:29 +0100 Subject: [PATCH 10/59] Replaces TestCleanup target with simpler call in the teardown method --- .../single-root-ctest/project-folder/CMakeLists.txt | 6 ------ .../single-root-ctest/project-folder/main.cpp | 8 ++++---- .../single-root-ctest/project-folder/test_a.cpp | 2 +- .../single-root-ctest/project-folder/test_b.cpp | 2 +- .../single-root-ctest/test/ctest-run-tests.test.ts | 5 ++++- 5 files changed, 10 insertions(+), 13 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 5c89c96e4..39f12fcd3 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -5,19 +5,13 @@ add_executable(ConcatTestOutput main.cpp) set_target_properties(ConcatTestOutput PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) add_executable(TestA test_a.cpp) add_executable(TestB test_b.cpp) -add_executable(TestCleanup tests_cleanup.cpp) -set_target_properties(TestCleanup PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) enable_testing() add_test(NAME "Suite1.TestA" COMMAND TestA) add_test(NAME "Suite2.TestB" COMMAND TestB) add_test(NAME "Concatenate_Test_Output" COMMAND ConcatTestOutput) -add_test(NAME "Test_Cleanup" COMMAND TestCleanup) -set_tests_properties("Test_Cleanup" PROPERTIES FIXTURES_CLEANUP GENOUT) set_tests_properties("Concatenate_Test_Output" PROPERTIES FIXTURES_CLEANUP GENOUT) -set_tests_properties("Test_Cleanup" PROPERTIES DEPENDS Concatenate_Test_Output) - set_tests_properties("Suite1.TestA" PROPERTIES FIXTURES_REQUIRED GENOUT) set_tests_properties("Suite2.TestB" PROPERTIES FIXTURES_REQUIRED GENOUT) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index 8ed84658d..09be9fdaf 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -44,16 +44,16 @@ int main(int, char**) { return 1; } - const auto& content_a = dump_file("test_a.txt"); - const auto& content_b = dump_file("test_b.txt"); + const auto& content_a = dump_file("/tmp/test_a.txt"); + const auto& content_b = dump_file("/tmp/test_b.txt"); ofs_test << "{\n"; if (!content_a.empty()) { - ofs_test << dump_file("test_a.txt"); + ofs_test << content_a; ofs_test << ",\n"; } - ofs_test << dump_file("test_b.txt"); + ofs_test << content_b; ofs_test << "}\n"; return 0; } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp index ad5e2679c..d926a181e 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp @@ -2,7 +2,7 @@ #include int main() { - std::ofstream outfile("test_a.txt"); + std::ofstream outfile("/tmp/test_a.txt"); if (outfile.is_open()) { outfile << "\"test_a\": \"OK\""; outfile.close(); diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp index 0610c8807..f991ca771 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp @@ -2,7 +2,7 @@ #include int main() { - std::ofstream outfile("test_b.txt"); + std::ofstream outfile("/tmp/test_b.txt"); if (outfile.is_open()) { outfile << "\"test_b\": \"OK\""; outfile.close(); diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 8b7ce06b7..dbfd822a2 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -6,6 +6,7 @@ import { } from '@test/util'; import * as path from 'path'; import * as vscode from 'vscode'; +import paths from '@cmt/paths'; suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; @@ -28,7 +29,7 @@ suite('Ctest run tests', () => { this.timeout(100000); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', 'true'); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); @@ -40,6 +41,8 @@ suite('Ctest run tests', () => { }); teardown(async function (this: Mocha.Context) { + await fs.writeFile(path.join(paths.tmpDir, 'test_a.txt'), '[]'); + await fs.writeFile(path.join(paths.tmpDir, 'test_b.txt'), '[]'); }); suiteTeardown(async () => { From c8d5ce6d9ac76c768042c20a3f199e101073cb7b Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 15:33:32 +0100 Subject: [PATCH 11/59] Add cmake.ctest.allowParallelJobs setting --- .../single-root-ctest/project-folder/.vscode/settings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json index ff3d48f17..2857722b5 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/.vscode/settings.json @@ -1,5 +1,6 @@ { "cmake.buildDirectory": "${workspaceFolder}/build", "cmake.useCMakePresets": "always", - "cmake.configureOnOpen": false + "cmake.configureOnOpen": false, + "cmake.ctest.allowParallelJobs": false } \ No newline at end of file From 4cfe9fcd98e9305184344f601ac21b9a76a16074 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 15:53:59 +0100 Subject: [PATCH 12/59] Upgrade cmake minimum required version to avoid warnings --- .../single-root-ctest/project-folder/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 39f12fcd3..3739e2206 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.0.0) +cmake_minimum_required(VERSION 3.6.0) project(TestCTestProcess VERSION 0.1.0) add_executable(ConcatTestOutput main.cpp) From 4a6202e8dbf33fb7b9534bd74ce4a8f0af2ba18e Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 15:55:09 +0100 Subject: [PATCH 13/59] Sets up two tests. One with parallel jobs enabled and the other without --- .../single-root-ctest/test/ctest-run-tests.test.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index dbfd822a2..dcd44bd7a 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -29,7 +29,6 @@ suite('Ctest run tests', () => { this.timeout(100000); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); @@ -54,7 +53,8 @@ suite('Ctest run tests', () => { } }); - test('Test of ctest', async () => { + test('Test of ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -62,4 +62,12 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK'); }).timeout(100000); + test('Test of ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK'); + expect(result['test_b']).to.eq('OK'); + }).timeout(100000); }); From add157aad473824741121bd64e9c08c37e318d5f Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 16:00:43 +0100 Subject: [PATCH 14/59] Remove useless file --- .../project-folder/tests_cleanup.cpp | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp deleted file mode 100644 index 29df6806b..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/tests_cleanup.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include - -void delete_file(const std::string &file_name) -{ - try - { - if (std::filesystem::remove(file_name)) - { - std::cout << file_name << " was deleted successfully.\n"; - } - else - { - std::cout << file_name << " does not exist.\n"; - } - } - catch (const std::filesystem::filesystem_error &e) - { - std::cerr << "Filesystem error: " << e.what() << '\n'; - } -} - -int main() -{ - const auto file_a = "test_a.txt"; - const auto file_b = "test_b.txt"; - - delete_file(file_a); - delete_file(file_b); - - return 0; -} \ No newline at end of file From 14d7963c027d9e9bf74c85765dd7167000cd8d93 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 16:19:46 +0100 Subject: [PATCH 15/59] Adds two tests dealing with testSuiteDelimiter --- .../test/ctest-run-tests.test.ts | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index dcd44bd7a..984496c4c 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -62,6 +62,16 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK'); }).timeout(100000); + test('Test of ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "."); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK'); + expect(result['test_b']).to.eq('OK'); + }).timeout(100000); + test('Test of ctest with parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); @@ -70,4 +80,14 @@ suite('Ctest run tests', () => { expect(result['test_a']).to.eq('OK'); expect(result['test_b']).to.eq('OK'); }).timeout(100000); + + test('Test of ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "."); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK'); + expect(result['test_b']).to.eq('OK'); + }).timeout(100000); }); From 8c057e36c7acd14c78ab822f9e13b14bc6d10b62 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 19:27:34 +0100 Subject: [PATCH 16/59] Remove generated test file between each test --- .../single-root-ctest/test/ctest-run-tests.test.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 984496c4c..362cf31ec 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -40,8 +40,9 @@ suite('Ctest run tests', () => { }); teardown(async function (this: Mocha.Context) { - await fs.writeFile(path.join(paths.tmpDir, 'test_a.txt'), '[]'); - await fs.writeFile(path.join(paths.tmpDir, 'test_b.txt'), '[]'); + await fs.unlink(path.join(paths.tmpDir, 'test_a.txt')); + await fs.unlink(path.join(paths.tmpDir, 'test_b.txt')); + await fs.unlink(path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult)); }); suiteTeardown(async () => { From 9d90005e2b454bca47c9a90e0b5c65cf8657ff83 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 14 Nov 2024 20:04:53 +0100 Subject: [PATCH 17/59] Do not delete file if it is not present --- .../test/ctest-run-tests.test.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 362cf31ec..5e7e1c5c1 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -40,9 +40,18 @@ suite('Ctest run tests', () => { }); teardown(async function (this: Mocha.Context) { - await fs.unlink(path.join(paths.tmpDir, 'test_a.txt')); - await fs.unlink(path.join(paths.tmpDir, 'test_b.txt')); - await fs.unlink(path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult)); + const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); + if (await fs.exists(file_a_path)) { + await fs.unlink(file_a_path); + } + const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); + if (await fs.exists(file_b_path)) { + await fs.unlink(file_b_path); + } + const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); + if (await fs.exists(output_test_path)) { + await fs.unlink(output_test_path); + } }); suiteTeardown(async () => { @@ -56,6 +65,7 @@ suite('Ctest run tests', () => { test('Test of ctest without parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', ""); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -75,6 +85,7 @@ suite('Ctest run tests', () => { test('Test of ctest with parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', ""); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); From 5107e59b7bafbef625519c343802b268e2eeeee3 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 14:24:36 +0100 Subject: [PATCH 18/59] Fix test suite delimiter --- .../single-root-ctest/test/ctest-run-tests.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 5e7e1c5c1..1a2f7cc01 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -75,7 +75,7 @@ suite('Ctest run tests', () => { test('Test of ctest without parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "."); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -95,7 +95,7 @@ suite('Ctest run tests', () => { test('Test of ctest with parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "."); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); From cc818aeb11c825d22737c8cb4800a03241e2ab7a Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 14:25:34 +0100 Subject: [PATCH 19/59] Do not add a comma if no text is appended --- .../single-root-ctest/project-folder/main.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index 09be9fdaf..ccb5208ac 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -51,7 +51,11 @@ int main(int, char**) { if (!content_a.empty()) { ofs_test << content_a; - ofs_test << ",\n"; + if (!content_b.empty()) + { + ofs_test << ","; + } + ofs_test << "\n"; } ofs_test << content_b; ofs_test << "}\n"; From f3dbbe2bf76d5cdbdae52facf0f4d821014ece44 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 14:25:58 +0100 Subject: [PATCH 20/59] Delete test file results before the test --- .../single-root-ctest/test/ctest-run-tests.test.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 1a2f7cc01..95cb1a617 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -26,6 +26,19 @@ suite('Ctest run tests', () => { }); setup(async function (this: Mocha.Context) { + const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); + if (await fs.exists(file_a_path)) { + await fs.unlink(file_a_path); + } + const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); + if (await fs.exists(file_b_path)) { + await fs.unlink(file_b_path); + } + const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); + if (await fs.exists(output_test_path)) { + await fs.unlink(output_test_path); + } + this.timeout(100000); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); From 3d9f137531bd8d10571fdacbb0feab42c3311417 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 14:36:22 +0100 Subject: [PATCH 21/59] Create cleanUpTestResultFiles to factorize code --- .../test/ctest-run-tests.test.ts | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 95cb1a617..c355ff5f0 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -12,6 +12,21 @@ suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; let compdb_cp_path: string; + async function cleanUpTestResultFiles() { + const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); + if (await fs.exists(file_a_path)) { + await fs.unlink(file_a_path); + } + const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); + if (await fs.exists(file_b_path)) { + await fs.unlink(file_b_path); + } + const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); + if (await fs.exists(output_test_path)) { + await fs.unlink(output_test_path); + } + } + suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); @@ -26,18 +41,7 @@ suite('Ctest run tests', () => { }); setup(async function (this: Mocha.Context) { - const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); - if (await fs.exists(file_a_path)) { - await fs.unlink(file_a_path); - } - const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); - if (await fs.exists(file_b_path)) { - await fs.unlink(file_b_path); - } - const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); - if (await fs.exists(output_test_path)) { - await fs.unlink(output_test_path); - } + await cleanUpTestResultFiles(); this.timeout(100000); @@ -53,18 +57,7 @@ suite('Ctest run tests', () => { }); teardown(async function (this: Mocha.Context) { - const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); - if (await fs.exists(file_a_path)) { - await fs.unlink(file_a_path); - } - const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); - if (await fs.exists(file_b_path)) { - await fs.unlink(file_b_path); - } - const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); - if (await fs.exists(output_test_path)) { - await fs.unlink(output_test_path); - } + await cleanUpTestResultFiles(); }); suiteTeardown(async () => { From 838d8a2c4bfa89d2163294280831f94c9e494339 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 14:36:44 +0100 Subject: [PATCH 22/59] Adds explicit message to failing assertions --- .../test/ctest-run-tests.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index c355ff5f0..7c62d1634 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -75,8 +75,8 @@ suite('Ctest run tests', () => { expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); - expect(result['test_a']).to.eq('OK'); - expect(result['test_b']).to.eq('OK'); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); test('Test of ctest without parallel jobs. Use test suite delimiter', async () => { @@ -85,8 +85,8 @@ suite('Ctest run tests', () => { expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); - expect(result['test_a']).to.eq('OK'); - expect(result['test_b']).to.eq('OK'); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); test('Test of ctest with parallel jobs', async () => { @@ -95,8 +95,8 @@ suite('Ctest run tests', () => { expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); - expect(result['test_a']).to.eq('OK'); - expect(result['test_b']).to.eq('OK'); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); test('Test of ctest with parallel jobs. Use test suite delimiter', async () => { @@ -105,7 +105,7 @@ suite('Ctest run tests', () => { expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); - expect(result['test_a']).to.eq('OK'); - expect(result['test_b']).to.eq('OK'); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); }); From cbe131ccd3c093739c4c0bcad897628d4a6495b5 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 17:16:01 +0100 Subject: [PATCH 23/59] Fix comment --- .../single-root-ctest/test/ctest-run-tests.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 7c62d1634..21f46cdd0 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -33,7 +33,7 @@ suite('Ctest run tests', () => { const build_loc = 'build'; const exe_res = 'output_test.txt'; - // CMakePresets.json and CMakeUserPresets.json exist so will use presets by default + // CMakePresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); compdb_cp_path = path.join(testEnv.projectFolder.location, 'compdb_cp.json'); From 2e7a64aa2a75918532e8a7d19fd9e1c6ea200274 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Fri, 15 Nov 2024 17:16:20 +0100 Subject: [PATCH 24/59] Generalize output writing --- .../single-root-ctest/project-folder/main.cpp | 66 +++++++++++++------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index ccb5208ac..5b7106774 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -1,63 +1,89 @@ -#include +#include #include +#include #include #include #include +#include /********************************************************************************/ /** * @brief Dump the content of a file to a string. - * + * * @param filename : The name of the file to dump. - * @return std::string : The content of the file. + * @return std::string : The content of the file or empty in case of error. */ /********************************************************************************/ -std::string dump_file(const std::string& filename) { +std::string dump_file(const std::string &filename) +{ std::filesystem::path filepath(filename); - if (!std::filesystem::exists(filepath)) { + if (!std::filesystem::exists(filepath)) + { std::cerr << "File does not exist: " << filename << '\n'; return {}; } - std::ifstream ifs (filepath, std::ifstream::in); - if (!ifs) { + std::ifstream ifs(filepath, std::ifstream::in); + if (!ifs) + { std::cerr << "Failed to open file: " << filename << '\n'; return {}; } std::ostringstream oss; - for (std::string line; std::getline(ifs, line);) { + for (std::string line; std::getline(ifs, line);) + { oss << line; - if (ifs.good()) { - oss <<'\n'; + if (ifs.good()) + { + oss << '\n'; } } - if (ifs.bad()) { + if (ifs.bad()) + { std::cerr << "Failed to read file: " << filename << '\n'; return {}; } return oss.str(); } -int main(int, char**) { +int generate_output_file(const std::vector &file_names) +{ std::ofstream ofs_test("output_test.txt"); - if (!ofs_test) { + if (!ofs_test) + { std::cerr << "Failed to open output_test.txt\n"; return 1; } - const auto& content_a = dump_file("/tmp/test_a.txt"); - const auto& content_b = dump_file("/tmp/test_b.txt"); - ofs_test << "{\n"; - if (!content_a.empty()) + + for (auto iter{std::cbegin(file_names)}; iter != std::cend(file_names); ++iter) { - ofs_test << content_a; - if (!content_b.empty()) + const auto& ccontent = dump_file(*iter); + if (!ccontent.empty()) + { + ofs_test << ccontent; + } + + const bool has_empty_successor = dump_file(*(std::next(iter))).empty(); + if (!has_empty_successor) { ofs_test << ","; } ofs_test << "\n"; } - ofs_test << content_b; + ofs_test << "}\n"; return 0; +} + +/*----------------------------------------------------------------------------*/ +/** + * @brief + * + * @return int + */ +/*----------------------------------------------------------------------------*/ +int main(int, char **) +{ + return generate_output_file({"/tmp/test_a.txt", "/tmp/test_b.txt"}); } \ No newline at end of file From 6c6290f14090cbd2b939566f38800b6fa68fc56b Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 16 Nov 2024 12:20:16 +0100 Subject: [PATCH 25/59] Protect empy successor when dealing with last element of the container --- .../single-root-ctest/project-folder/main.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index 5b7106774..c68bb2f1f 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -64,10 +64,13 @@ int generate_output_file(const std::vector &file_names) ofs_test << ccontent; } - const bool has_empty_successor = dump_file(*(std::next(iter))).empty(); - if (!has_empty_successor) + if (std::next(iter) != std::cend(file_names)) { - ofs_test << ","; + const bool has_empty_successor = dump_file(*(std::next(iter))).empty(); + if (!has_empty_successor) + { + ofs_test << ","; + } } ofs_test << "\n"; } From 7f05cff397c2f3659d1f01cadeba5fe7df761026 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 16 Nov 2024 12:23:49 +0100 Subject: [PATCH 26/59] Adds doc and format --- .../single-root-ctest/project-folder/main.cpp | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp index c68bb2f1f..05d0370d4 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp @@ -45,6 +45,21 @@ std::string dump_file(const std::string &filename) return oss.str(); } +/********************************************************************************/ +/** + * @brief Generate an output file containing the content of the input files + * separated by commas and enclosed in curly braces. + * + * The output file is named output_test.txt and is created in the current + * directory. + * + * The output file is in json format provided that the input files are in text + * format. + * + * @param file_names : The list of input files to dump. + * @return int : 0 if the output file was successfully generated, 1 otherwise. + */ +/********************************************************************************/ int generate_output_file(const std::vector &file_names) { std::ofstream ofs_test("output_test.txt"); @@ -58,7 +73,7 @@ int generate_output_file(const std::vector &file_names) for (auto iter{std::cbegin(file_names)}; iter != std::cend(file_names); ++iter) { - const auto& ccontent = dump_file(*iter); + const auto &ccontent = dump_file(*iter); if (!ccontent.empty()) { ofs_test << ccontent; @@ -81,12 +96,12 @@ int generate_output_file(const std::vector &file_names) /*----------------------------------------------------------------------------*/ /** - * @brief + * @brief Main function. * - * @return int + * @return int : 0 if the output file was successfully generated, 1 otherwise. */ /*----------------------------------------------------------------------------*/ int main(int, char **) { - return generate_output_file({"/tmp/test_a.txt", "/tmp/test_b.txt"}); + return generate_output_file({"/tmp/test_a.txt", "/tmp/test_b.txt"}); } \ No newline at end of file From 3858fff4719cc6efb0c4e7785cbabdd76c14520c Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 12:20:20 +0100 Subject: [PATCH 27/59] Move suite initialization in the right method --- .../test/ctest-run-tests.test.ts | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 21f46cdd0..5319422dd 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -1,6 +1,5 @@ import { fs } from '@cmt/pr'; import { - clearExistingKitConfigurationFile, DefaultEnvironment, expect } from '@test/util'; @@ -10,7 +9,6 @@ import paths from '@cmt/paths'; suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; - let compdb_cp_path: string; async function cleanUpTestResultFiles() { const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); @@ -35,15 +33,7 @@ suite('Ctest run tests', () => { // CMakePresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); - compdb_cp_path = path.join(testEnv.projectFolder.location, 'compdb_cp.json'); - - await clearExistingKitConfigurationFile(); - }); - - setup(async function (this: Mocha.Context) { - await cleanUpTestResultFiles(); - - this.timeout(100000); + testEnv.projectFolder.buildDirectory.clear(); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); @@ -56,6 +46,10 @@ suite('Ctest run tests', () => { await vscode.commands.executeCommand('cmake.build'); }); + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(); + }); + teardown(async function (this: Mocha.Context) { await cleanUpTestResultFiles(); }); @@ -64,9 +58,6 @@ suite('Ctest run tests', () => { if (testEnv) { testEnv.teardown(); } - if (await fs.exists(compdb_cp_path)) { - await fs.unlink(compdb_cp_path); - } }); test('Test of ctest without parallel jobs', async () => { From 31119f20c0273e0744e652b9dfed8bfa27c36bf4 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 13:25:53 +0100 Subject: [PATCH 28/59] Await for settings change promise --- src/extension.ts | 8 ++++---- .../single-root-ctest/test/ctest-run-tests.test.ts | 9 +++++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index b22908de8..0f49a4b9b 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2275,10 +2275,10 @@ async function setup(context: vscode.ExtensionContext, progress?: ProgressHandle log.trace(localize('register.command', 'Register CMakeTools extension command {0}', `cmake.${key}`)); context.subscriptions.push(register(key)); } - if (util.isTestMode()) { - log.trace(localize('register.command', 'Register CMakeTools extension command cmake.getSettingsChangePromise')); - context.subscriptions.push(vscode.commands.registerCommand('cmake.getSettingsChangePromise', () => getSettingsChangePromise())); - } + // if (util.isTestMode()) { + log.trace(localize('register.command', 'Register CMakeTools extension command cmake.getSettingsChangePromise')); + context.subscriptions.push(vscode.commands.registerCommand('cmake.getSettingsChangePromise', () => getSettingsChangePromise())); + // } context.subscriptions.push(...[ // Special commands that don't require logging or separate error handling diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 5319422dd..34a29d315 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -36,6 +36,7 @@ suite('Ctest run tests', () => { testEnv.projectFolder.buildDirectory.clear(); await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); @@ -62,7 +63,8 @@ suite('Ctest run tests', () => { test('Test of ctest without parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', ""); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -73,6 +75,7 @@ suite('Ctest run tests', () => { test('Test of ctest without parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -82,7 +85,8 @@ suite('Ctest run tests', () => { test('Test of ctest with parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); - await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', ""); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); @@ -93,6 +97,7 @@ suite('Ctest run tests', () => { test('Test of ctest with parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(0); const result = await testEnv.result.getResultAsJson(); From 6dec1a4001a109ef3eb97356bd9e177c73d45c30 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 14:57:03 +0100 Subject: [PATCH 29/59] Set CMT_TESTING to one --- .vscode/launch.json | 2 +- test/end-to-end-tests/single-root-ctest/runTest.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 8947ccf5e..bd2ab6791 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -140,7 +140,7 @@ ], "preLaunchTask": "Pretest", "env": { - "CMT_TESTING": "0", + "CMT_TESTING": "1", "CMT_QUIET_CONSOLE": "1", "TEST_FILTER": ".*" }, diff --git a/test/end-to-end-tests/single-root-ctest/runTest.ts b/test/end-to-end-tests/single-root-ctest/runTest.ts index 5f26a42b0..0118708e0 100644 --- a/test/end-to-end-tests/single-root-ctest/runTest.ts +++ b/test/end-to-end-tests/single-root-ctest/runTest.ts @@ -17,7 +17,7 @@ async function main() { const launchArgs = ["--disable-extensions", "--disable-workspace-trust", testWorkspace]; const extensionTestsEnv: { [key: string]: string | undefined } = { - "CMT_TESTING": "0", + "CMT_TESTING": "1", "CMT_QUIET_CONSOLE": "1", "TEST_FILTER": process.env.TEST_FILTER ?? ".*" }; From acfd55e233a2786f7510b3121657ef98874c6da9 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 14:58:43 +0100 Subject: [PATCH 30/59] Restore previous state --- src/extension.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/extension.ts b/src/extension.ts index 0f49a4b9b..b22908de8 100644 --- a/src/extension.ts +++ b/src/extension.ts @@ -2275,10 +2275,10 @@ async function setup(context: vscode.ExtensionContext, progress?: ProgressHandle log.trace(localize('register.command', 'Register CMakeTools extension command {0}', `cmake.${key}`)); context.subscriptions.push(register(key)); } - // if (util.isTestMode()) { - log.trace(localize('register.command', 'Register CMakeTools extension command cmake.getSettingsChangePromise')); - context.subscriptions.push(vscode.commands.registerCommand('cmake.getSettingsChangePromise', () => getSettingsChangePromise())); - // } + if (util.isTestMode()) { + log.trace(localize('register.command', 'Register CMakeTools extension command cmake.getSettingsChangePromise')); + context.subscriptions.push(vscode.commands.registerCommand('cmake.getSettingsChangePromise', () => getSettingsChangePromise())); + } context.subscriptions.push(...[ // Special commands that don't require logging or separate error handling From 0306a8279cd2271d0401eb8d9f34777656ed1d91 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 15:51:22 +0100 Subject: [PATCH 31/59] Renames files and targets --- .../single-root-ctest/project-folder/CMakeLists.txt | 8 ++++---- .../project-folder/{main.cpp => generate_output_file.cpp} | 0 2 files changed, 4 insertions(+), 4 deletions(-) rename test/end-to-end-tests/single-root-ctest/project-folder/{main.cpp => generate_output_file.cpp} (100%) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 3739e2206..bff85473a 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -1,8 +1,8 @@ cmake_minimum_required(VERSION 3.6.0) project(TestCTestProcess VERSION 0.1.0) -add_executable(ConcatTestOutput main.cpp) -set_target_properties(ConcatTestOutput PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) +add_executable(GenerateOutputFile generate_output_file.cpp) +set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) add_executable(TestA test_a.cpp) add_executable(TestB test_b.cpp) @@ -10,8 +10,8 @@ enable_testing() add_test(NAME "Suite1.TestA" COMMAND TestA) add_test(NAME "Suite2.TestB" COMMAND TestB) -add_test(NAME "Concatenate_Test_Output" COMMAND ConcatTestOutput) +add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) -set_tests_properties("Concatenate_Test_Output" PROPERTIES FIXTURES_CLEANUP GENOUT) +set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) set_tests_properties("Suite1.TestA" PROPERTIES FIXTURES_REQUIRED GENOUT) set_tests_properties("Suite2.TestB" PROPERTIES FIXTURES_REQUIRED GENOUT) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/main.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp similarity index 100% rename from test/end-to-end-tests/single-root-ctest/project-folder/main.cpp rename to test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp From 3c592706aed53d86ead27885645f9cbd7137de70 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 16:58:43 +0100 Subject: [PATCH 32/59] Create a TestUtils lib to factorize code --- .../project-folder/CMakeLists.txt | 8 +++++ .../project-folder/test_a.cpp | 13 ++------ .../project-folder/test_b.cpp | 13 ++------ .../project-folder/test_utils.cpp | 31 +++++++++++++++++++ .../project-folder/test_utils.h | 14 +++++++++ 5 files changed, 57 insertions(+), 22 deletions(-) create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index bff85473a..0739e4d74 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -3,8 +3,16 @@ project(TestCTestProcess VERSION 0.1.0) add_executable(GenerateOutputFile generate_output_file.cpp) set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + +add_library(TestUtils) +target_sources(TestUtils PRIVATE test_utils.cpp) +target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) +set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + add_executable(TestA test_a.cpp) +target_link_libraries(TestA PRIVATE TestUtils) add_executable(TestB test_b.cpp) +target_link_libraries(TestB PRIVATE TestUtils) enable_testing() diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp index d926a181e..cf8365c19 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp @@ -1,14 +1,5 @@ -#include -#include +#include "test_utils.h" int main() { - std::ofstream outfile("/tmp/test_a.txt"); - if (outfile.is_open()) { - outfile << "\"test_a\": \"OK\""; - outfile.close(); - std::cout << "File written successfully." << std::endl; - } else { - std::cerr << "Error opening file." << std::endl; - } - return 0; + return generic_test("/tmp/test_a.txt", true); } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp index f991ca771..4f52b2fed 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp @@ -1,14 +1,5 @@ -#include -#include +#include "test_utils.h" int main() { - std::ofstream outfile("/tmp/test_b.txt"); - if (outfile.is_open()) { - outfile << "\"test_b\": \"OK\""; - outfile.close(); - std::cout << "File written successfully." << std::endl; - } else { - std::cerr << "Error opening file." << std::endl; - } - return 0; + return generic_test("/tmp/test_b.txt", true); } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp new file mode 100644 index 000000000..932b442bf --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.cpp @@ -0,0 +1,31 @@ +#include "test_utils.h" + +#include +#include +#include + +/********************************************************************************/ +/** + * @brief Generic test function that writes a file with the test result. + * If the test is successful, the file will contain the test name and "OK". + * Otherwise, the file will contain the test name and "KO". + * + * @param test_filepath : the path to the file to write + * @param success : the test result + * @return int : 0 if the test is successful, 1 otherwise + */ +/********************************************************************************/ +int generic_test(const std::string& test_filepath, const bool success) { + std::filesystem::path test_path(test_filepath); + const auto& test_name{test_path.stem()}; + + std::ofstream outfile(test_filepath); + if (outfile.is_open()) { + outfile << test_name << " : \"" << (success ? "OK" : "KO") << "\""; + outfile.close(); + std::cout << "File written successfully." << std::endl; + } else { + std::cerr << "Error opening file." << std::endl; + } + return success ? 0 : 1; +} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h new file mode 100644 index 000000000..32db4db87 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test_utils.h @@ -0,0 +1,14 @@ +#ifndef TEST_UTILS_H +#define TEST_UTILS_H + +/********************************************************************************/ +/********************************************************************************/ +#include + +/********************************************************************************/ +/********************************************************************************/ +int generic_test(const std::string& test_filepath, const bool success); + +/********************************************************************************/ +/********************************************************************************/ +#endif // TEST_UTILS_H \ No newline at end of file From 699c540c08b0b5cfac79841b58cebe17394da544 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 17 Nov 2024 18:01:19 +0100 Subject: [PATCH 33/59] Start templating test file --- .../single-root-ctest/project-folder/CMakeLists.txt | 11 +++++++++++ .../single-root-ctest/project-folder/test.cpp.in | 5 +++++ 2 files changed, 16 insertions(+) create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 0739e4d74..cd904dc37 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -9,6 +9,17 @@ target_sources(TestUtils PRIVATE test_utils.cpp) target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + +# set(TESTS_FILES "/tmp/test_a.txt;/tmp/test_b.txt") + +# foreach(TEST_FILE ${TESTS_FILES}) +# get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) +# configure_file(test.cpp.in "${TEST_FILE}.cpp" @ONLY) +# add_executable(TestA test_a.cpp) +# target_link_libraries(TestA PRIVATE TestUtils) +# endforeach() + + add_executable(TestA test_a.cpp) target_link_libraries(TestA PRIVATE TestUtils) add_executable(TestB test_b.cpp) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in new file mode 100644 index 000000000..66203e450 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in @@ -0,0 +1,5 @@ +#include "test_utils.h" + +int main() { + return generic_test("@test_filename@", @success@); +} From b8b7e4516c6630705644fecd2bd92781bb759edc Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Mon, 18 Nov 2024 16:28:44 +0100 Subject: [PATCH 34/59] Move cmake stuff in its own module --- .../project-folder/CMakeLists.txt | 30 ++-- .../project-folder/cmake/RegisterTest.cmake | 134 ++++++++++++++++++ 2 files changed, 143 insertions(+), 21 deletions(-) create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index cd904dc37..6fe129468 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -9,28 +9,16 @@ target_sources(TestUtils PRIVATE test_utils.cpp) target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) - -# set(TESTS_FILES "/tmp/test_a.txt;/tmp/test_b.txt") - -# foreach(TEST_FILE ${TESTS_FILES}) -# get_filename_component(TEST_NAME ${TEST_FILE} NAME_WE) -# configure_file(test.cpp.in "${TEST_FILE}.cpp" @ONLY) -# add_executable(TestA test_a.cpp) -# target_link_libraries(TestA PRIVATE TestUtils) -# endforeach() - - -add_executable(TestA test_a.cpp) -target_link_libraries(TestA PRIVATE TestUtils) -add_executable(TestB test_b.cpp) -target_link_libraries(TestB PRIVATE TestUtils) - enable_testing() -add_test(NAME "Suite1.TestA" COMMAND TestA) -add_test(NAME "Suite2.TestB" COMMAND TestB) add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) - set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) -set_tests_properties("Suite1.TestA" PROPERTIES FIXTURES_REQUIRED GENOUT) -set_tests_properties("Suite2.TestB" PROPERTIES FIXTURES_REQUIRED GENOUT) + +list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") +include(RegisterTest) + +set(TESTS_OUTPUT_FILES "/tmp/test_a.txt;/tmp/test_b.txt") +set(TESTS_NAMES "Suite1.TestA;Suite2.TestB") +set(TESTS_SUCCESS "true;true") + +register_tests(TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_PATH_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake new file mode 100644 index 000000000..5a26fd52c --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -0,0 +1,134 @@ +#-------------------------------------------------------------------- +# Generate the source file of the test in argument +# +# Parameters: +# output_file_path: path to the file, the test will write +# test_success: true if the test end successfully. False otherwise. +# Returns: +# test_source: name of the source file that will be generated +#-------------------------------------------------------------------- +function(generate_test_source_file output_file_path test_success) + get_filename_component(output_file_name ${output_file_path} NAME_WE) + # Declare variables used in the template file (test.cpp.in) + set(test_filename ${output_file_path}) + set(success ${test_success}) + # Generate test source file + set(test_source "${output_file_name}.cpp" PARENT_SCOPE) + configure_file(test.cpp.in ${test_source} @ONLY) +endfunction() + +#-------------------------------------------------------------------- +# Build the name of the test executable from the name of the test source file +# +# The name of the executable will be the one of the test source file without '_' +# and in CamelCase. +# +# Parameters: +# test_source: name of the test source file +# Returns: +# test_exe: name of the test executable +#-------------------------------------------------------------------- +function(build_test_exe_name test_source) + get_filename_component(test_name ${test_source} NAME_WE) + # Replace _ with ; so that the name becomes a list of sub-words + string(REPLACE "_" ";" splitted_test_name ${test_name}) + set(test_exe) + # For each of the sub-word, extract the first letter + # from the rest of the word (radical) + foreach(word ${splitted_test_name}) + string(SUBSTRING ${word} 0 1 first_letter) + string(SUBSTRING ${word} 1 -1 radical) + # Turns first sub-word letter into upper case + string(TOUPPER ${first_letter} up_first_letter) + # Concat uppercase first letter and radical + set(test_exe "${test_exe}${up_first_letter}${radical}") + endforeach() + # Returns test_exe + set(test_exe ${test_exe} PARENT_SCOPE) +endfunction() + +#-------------------------------------------------------------------- +# Create and register a test +# +# Usage: +# register_test(TEST_NAME TEST_OUTPUT_FILE_PATH TEST_SUCCESS ) +# Parameters: +# TEST_NAME: name of the test +# TEST_OUTPUT_FILE_PATH: path to the file the test should generate +# TEST_SUCCESS: whether or not the test should end successfully +#-------------------------------------------------------------------- +function(register_test) + set(options) + set(oneValueArgs "TEST_NAME;TEST_OUTPUT_FILE_PATH;TEST_SUCCESS") + ### PARSING ARGUMENTS + cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(DEFINED register_test_KEYWORDS_MISSING_VALUES) + message( + FATAL_ERROR + "In the call to register_test function, the keywords ${register_test_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" + ) + endif() + if(DEFINED register_test_UNPARSED_ARGUMENTS) + message( + FATAL_ERROR + "Following arguments are unknown to register_test function: ${register_test_UNPARSED_ARGUMENTS}" + ) + endif() + if(NOT DEFINED register_test_TEST_NAME) + message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword") + endif() + if(NOT DEFINED register_test_TEST_OUTPUT_FILE_PATH) + message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE_PATH keyword") + endif() + if(NOT DEFINED register_test_TEST_SUCCESS) + message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword") + endif() + + message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${register_test_TEST_OUTPUT_FILE_PATH} returning as success: ${register_test_TEST_SUCCESS}") + ### GENERATE TEST + generate_test_source_file(${register_test_TEST_OUTPUT_FILE_PATH} ${register_test_TEST_SUCCESS}) # => returns test_source + build_test_exe_name(${test_source}) # => returns test_exe + message(STATUS "--> Creating test executable ${test_exe} with source ${test_source}") + add_executable(${test_exe} ${test_source}) + target_link_libraries(${test_exe} PRIVATE TestUtils) + add_test(NAME "${register_test_TEST_NAME}" COMMAND "${test_exe}") + set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT) +endfunction() + + +function(register_tests) + set(options) + set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_PATH_LIST;TEST_SUCCESS_LIST") + ### PARSING ARGUMENTS + cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + if(DEFINED register_tests_KEYWORDS_MISSING_VALUES) + message( + FATAL_ERROR + "In the call to register_tests function, the keywords ${register_tests_KEYWORDS_MISSING_VALUES} are awaiting for at least one value" + ) + endif() + if(DEFINED register_tests_UNPARSED_ARGUMENTS) + message( + FATAL_ERROR + "Following arguments are unknown to register_tests function: ${register_tests_UNPARSED_ARGUMENTS}" + ) + endif() + if(NOT DEFINED register_tests_TEST_NAME_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword") + endif() + if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_PATH_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_PATH_LIST keyword") + endif() + if(NOT DEFINED register_tests_TEST_SUCCESS_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_SUCCESS_LIST keyword") + endif() + + list(LENGTH register_tests_TEST_NAME_LIST NB_TESTS) + math(EXPR MAX_INDEX "${NB_TESTS}-1") + foreach(test_index RANGE ${MAX_INDEX}) + list(GET register_tests_TEST_OUTPUT_FILE_PATH_LIST ${test_index} test_output) + list(GET register_tests_TEST_NAME_LIST ${test_index} test_name) + list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success) + register_test(TEST_NAME ${test_name} TEST_OUTPUT_FILE_PATH ${test_output} TEST_SUCCESS ${test_success}) + endforeach() +endfunction() \ No newline at end of file From 82c51b4f640ef718b2e170c8baed24b81d1ebe2d Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Mon, 18 Nov 2024 21:10:23 +0100 Subject: [PATCH 35/59] Adds doc --- .../project-folder/cmake/RegisterTest.cmake | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake index 5a26fd52c..133cfa9cd 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -95,7 +95,16 @@ function(register_test) set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT) endfunction() - +#-------------------------------------------------------------------- +# Create and register tests in arguments +# +# Usage: +# register_tests(TEST_NAME_LIST TEST_OUTPUT_FILE_PATH_LIST TEST_SUCCESS_LIST ) +# Parameters: +# TEST_NAME_LIST: name of the test +# TEST_OUTPUT_FILE_PATH_LIST: path to the file the test should generate +# TEST_SUCCESS_LIST: whether or not the test should end successfully +#-------------------------------------------------------------------- function(register_tests) set(options) set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_PATH_LIST;TEST_SUCCESS_LIST") From 006e9d693147e6c7f3326e60bdb72b48a309558a Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Mon, 18 Nov 2024 21:10:53 +0100 Subject: [PATCH 36/59] Set tests definition in CMake presets --- .../single-root-ctest/project-folder/CMakeLists.txt | 6 +++--- .../single-root-ctest/project-folder/CMakePresets.json | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 6fe129468..909dd4a92 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -17,8 +17,8 @@ set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(RegisterTest) -set(TESTS_OUTPUT_FILES "/tmp/test_a.txt;/tmp/test_b.txt") -set(TESTS_NAMES "Suite1.TestA;Suite2.TestB") -set(TESTS_SUCCESS "true;true") +set(TESTS_OUTPUT_FILES "" CACHE STRING "Output files generated by tests") +set(TESTS_NAMES "" CACHE STRING "Names of the tests") +set(TESTS_SUCCESS "" CACHE BOOL "Success of the tests") register_tests(TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_PATH_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index b6a305585..43e0e10f3 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -7,7 +7,10 @@ "generator": "Ninja", "binaryDir": "${workspaceFolder}/build", "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_OUTPUT_FILES": "/tmp/test_a.txt;/tmp/test_b.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", + "TESTS_SUCCESS": "true;true" } } ] From cdc12adaf20f4ea8de3509cafdc9876d9a6eba50 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Mon, 18 Nov 2024 21:16:13 +0100 Subject: [PATCH 37/59] Renames preset --- .../single-root-ctest/project-folder/CMakePresets.json | 2 +- .../single-root-ctest/test/ctest-run-tests.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index 43e0e10f3..606151744 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -2,7 +2,7 @@ "version": 2, "configurePresets": [ { - "name": "Linux1", + "name": "AllTestsSuccessfull", "description": "Sets generator, build and install directory, vcpkg", "generator": "Ninja", "binaryDir": "${workspaceFolder}/build", diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 34a29d315..012e3d420 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -38,7 +38,7 @@ suite('Ctest run tests', () => { await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); - await vscode.commands.executeCommand('cmake.setConfigurePreset', 'Linux1'); + await vscode.commands.executeCommand('cmake.setConfigurePreset', 'AllTestsSuccessfull'); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); From 8084838a5032170ee0334261fd90d0dab9b20a29 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 19 Nov 2024 08:40:17 +0100 Subject: [PATCH 38/59] Remove generated test from git --- .../single-root-ctest/project-folder/test_a.cpp | 5 ----- .../single-root-ctest/project-folder/test_b.cpp | 5 ----- 2 files changed, 10 deletions(-) delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp delete mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp deleted file mode 100644 index cf8365c19..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_a.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "test_utils.h" - -int main() { - return generic_test("/tmp/test_a.txt", true); -} \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp deleted file mode 100644 index 4f52b2fed..000000000 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test_b.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "test_utils.h" - -int main() { - return generic_test("/tmp/test_b.txt", true); -} \ No newline at end of file From bf7d48115d6b83d9a925843bb8d68a1dd3b2d528 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 19 Nov 2024 08:51:15 +0100 Subject: [PATCH 39/59] Fix generate_test_source_file function and return value --- .../single-root-ctest/project-folder/cmake/RegisterTest.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake index 133cfa9cd..cd922c433 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -13,8 +13,9 @@ function(generate_test_source_file output_file_path test_success) set(test_filename ${output_file_path}) set(success ${test_success}) # Generate test source file - set(test_source "${output_file_name}.cpp" PARENT_SCOPE) + set(test_source "${output_file_name}.cpp") configure_file(test.cpp.in ${test_source} @ONLY) + set(test_source "${test_source}" PARENT_SCOPE) endfunction() #-------------------------------------------------------------------- From b8f085a9006232988eac0f70c5099187923369c7 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 19 Nov 2024 10:22:07 +0100 Subject: [PATCH 40/59] Introduces use of common directory to store test result --- .../project-folder/CMakePresets.json | 2 +- .../project-folder/generate_output_file.cpp | 15 ++++++++++++++- .../single-root-ctest/project-folder/test.cpp.in | 6 ++++++ .../test/ctest-run-tests.test.ts | 10 ++-------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index 606151744..c4e59a9d3 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -8,7 +8,7 @@ "binaryDir": "${workspaceFolder}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", - "TESTS_OUTPUT_FILES": "/tmp/test_a.txt;/tmp/test_b.txt", + "TESTS_OUTPUT_FILES": "/tmp/vscode-cmake-tools-tests/test_a.txt;/tmp/vscode-cmake-tools-tests/test_b.txt", "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", "TESTS_SUCCESS": "true;true" } diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp index 05d0370d4..029ad3288 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp @@ -103,5 +103,18 @@ int generate_output_file(const std::vector &file_names) /*----------------------------------------------------------------------------*/ int main(int, char **) { - return generate_output_file({"/tmp/test_a.txt", "/tmp/test_b.txt"}); + auto test_dir = std::filesystem::path{"/tmp/vscode-cmake-tools-tests"}; + std::vector test_files{}; + if (!std::filesystem::exists(test_dir)) + { + // May happen in sequential test execution if the GenerateOutputFile test is executed first + return 0; + } + for (auto const& dir_entry : std::filesystem::directory_iterator{test_dir}) + { + std::cout << "Test file " << dir_entry.path() << " detected!" << std::endl; + test_files.emplace_back(dir_entry.path()); + } + + return generate_output_file(test_files); } \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in index 66203e450..c032f99c6 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in @@ -1,5 +1,11 @@ #include "test_utils.h" +#include int main() { + auto test_dir = std::filesystem::path{"/tmp/vscode-cmake-tools-tests"}; + if (!std::filesystem::exists(test_dir)) + { + std::filesystem::create_directory(test_dir); + } return generic_test("@test_filename@", @success@); } diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 012e3d420..5c9057c17 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -11,14 +11,8 @@ suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; async function cleanUpTestResultFiles() { - const file_a_path: string = path.join(paths.tmpDir, 'test_a.txt'); - if (await fs.exists(file_a_path)) { - await fs.unlink(file_a_path); - } - const file_b_path: string = path.join(paths.tmpDir, 'test_b.txt'); - if (await fs.exists(file_b_path)) { - await fs.unlink(file_b_path); - } + const test_dir: string = path.join(paths.tmpDir, "vscode-cmake-tools-tests"); + await fs.rmdir(test_dir); const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); if (await fs.exists(output_test_path)) { await fs.unlink(output_test_path); From 8945d0dcb0c747170489e53e5fe91396d4b8ffca Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 19 Nov 2024 11:38:32 +0100 Subject: [PATCH 41/59] Defines common tests directory in cmake cache variables --- .../project-folder/CMakeLists.txt | 16 +++++++++++----- .../project-folder/CMakePresets.json | 3 ++- .../project-folder/cmake/RegisterTest.cmake | 18 +++++++++++++----- .../project-folder/generate_output_file.cpp | 4 +++- .../project-folder/get_test_dir.h.in | 6 ++++++ .../project-folder/test.cpp.in | 3 ++- 6 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 909dd4a92..1fdab60d0 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -1,14 +1,20 @@ cmake_minimum_required(VERSION 3.6.0) project(TestCTestProcess VERSION 0.1.0) -add_executable(GenerateOutputFile generate_output_file.cpp) -set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) - -add_library(TestUtils) +add_library(TestUtils SHARED) target_sources(TestUtils PRIVATE test_utils.cpp) target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) +set(TESTS_DIR "" CACHE STRING "Directory where the file generated by tests will be written") +configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY) +add_library(GetTestDir INTERFACE) +target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) + +add_executable(GenerateOutputFile generate_output_file.cpp) +target_link_libraries(GenerateOutputFile PRIVATE GetTestDir) +set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) + enable_testing() add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) @@ -21,4 +27,4 @@ set(TESTS_OUTPUT_FILES "" CACHE STRING "Output files generated by tests") set(TESTS_NAMES "" CACHE STRING "Names of the tests") set(TESTS_SUCCESS "" CACHE BOOL "Success of the tests") -register_tests(TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_PATH_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) +register_tests(TEST_DIRECTORY ${TESTS_DIR} TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_PATH_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index c4e59a9d3..0a80b3f59 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -8,7 +8,8 @@ "binaryDir": "${workspaceFolder}/build", "cacheVariables": { "CMAKE_BUILD_TYPE": "Debug", - "TESTS_OUTPUT_FILES": "/tmp/vscode-cmake-tools-tests/test_a.txt;/tmp/vscode-cmake-tools-tests/test_b.txt", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt", "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", "TESTS_SUCCESS": "true;true" } diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake index cd922c433..526bcccd3 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -60,7 +60,7 @@ endfunction() #-------------------------------------------------------------------- function(register_test) set(options) - set(oneValueArgs "TEST_NAME;TEST_OUTPUT_FILE_PATH;TEST_SUCCESS") + set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE_PATH;TEST_SUCCESS") ### PARSING ARGUMENTS cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if(DEFINED register_test_KEYWORDS_MISSING_VALUES) @@ -75,6 +75,9 @@ function(register_test) "Following arguments are unknown to register_test function: ${register_test_UNPARSED_ARGUMENTS}" ) endif() + if(NOT DEFINED register_test_TEST_DIR) + message(FATAL_ERROR "The function register_test is awaiting for TEST_DIR keyword") + endif() if(NOT DEFINED register_test_TEST_NAME) message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword") endif() @@ -85,13 +88,14 @@ function(register_test) message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword") endif() - message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${register_test_TEST_OUTPUT_FILE_PATH} returning as success: ${register_test_TEST_SUCCESS}") + set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE_PATH}") + message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${test_output_file_path} returning as success: ${register_test_TEST_SUCCESS}") ### GENERATE TEST - generate_test_source_file(${register_test_TEST_OUTPUT_FILE_PATH} ${register_test_TEST_SUCCESS}) # => returns test_source + generate_test_source_file(${test_output_file_path} ${register_test_TEST_SUCCESS}) # => returns test_source build_test_exe_name(${test_source}) # => returns test_exe message(STATUS "--> Creating test executable ${test_exe} with source ${test_source}") add_executable(${test_exe} ${test_source}) - target_link_libraries(${test_exe} PRIVATE TestUtils) + target_link_libraries(${test_exe} PRIVATE TestUtils GetTestDir) add_test(NAME "${register_test_TEST_NAME}" COMMAND "${test_exe}") set_tests_properties("${register_test_TEST_NAME}" PROPERTIES FIXTURES_REQUIRED GENOUT) endfunction() @@ -108,6 +112,7 @@ endfunction() #-------------------------------------------------------------------- function(register_tests) set(options) + set(oneValueArgs "TEST_DIRECTORY") set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_PATH_LIST;TEST_SUCCESS_LIST") ### PARSING ARGUMENTS cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) @@ -123,6 +128,9 @@ function(register_tests) "Following arguments are unknown to register_tests function: ${register_tests_UNPARSED_ARGUMENTS}" ) endif() + if(NOT DEFINED register_tests_TEST_DIRECTORY) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_DIRECTORY keyword") + endif() if(NOT DEFINED register_tests_TEST_NAME_LIST) message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword") endif() @@ -139,6 +147,6 @@ function(register_tests) list(GET register_tests_TEST_OUTPUT_FILE_PATH_LIST ${test_index} test_output) list(GET register_tests_TEST_NAME_LIST ${test_index} test_name) list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success) - register_test(TEST_NAME ${test_name} TEST_OUTPUT_FILE_PATH ${test_output} TEST_SUCCESS ${test_success}) + register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE_PATH ${test_output} TEST_SUCCESS ${test_success}) endforeach() endfunction() \ No newline at end of file diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp index 029ad3288..c84f6069a 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp +++ b/test/end-to-end-tests/single-root-ctest/project-folder/generate_output_file.cpp @@ -6,6 +6,8 @@ #include #include +#include "get_test_dir.h" + /********************************************************************************/ /** * @brief Dump the content of a file to a string. @@ -103,7 +105,7 @@ int generate_output_file(const std::vector &file_names) /*----------------------------------------------------------------------------*/ int main(int, char **) { - auto test_dir = std::filesystem::path{"/tmp/vscode-cmake-tools-tests"}; + auto test_dir = get_test_dir(); std::vector test_files{}; if (!std::filesystem::exists(test_dir)) { diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in b/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in new file mode 100644 index 000000000..35c274f28 --- /dev/null +++ b/test/end-to-end-tests/single-root-ctest/project-folder/get_test_dir.h.in @@ -0,0 +1,6 @@ +#include + +std::filesystem::path get_test_dir() +{ + return {"@TESTS_DIR@"}; +} diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in index c032f99c6..89fa928f9 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in +++ b/test/end-to-end-tests/single-root-ctest/project-folder/test.cpp.in @@ -1,8 +1,9 @@ #include "test_utils.h" +#include "get_test_dir.h" #include int main() { - auto test_dir = std::filesystem::path{"/tmp/vscode-cmake-tools-tests"}; + auto test_dir = get_test_dir(); if (!std::filesystem::exists(test_dir)) { std::filesystem::create_directory(test_dir); From b2909a2be26614e4c9ea0445bd31c398ada5be8a Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Tue, 19 Nov 2024 11:45:26 +0100 Subject: [PATCH 42/59] Add and fix doc. Renames variable. --- .../project-folder/CMakeLists.txt | 2 +- .../project-folder/cmake/RegisterTest.cmake | 32 ++++++++++--------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 1fdab60d0..adbb9d035 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -27,4 +27,4 @@ set(TESTS_OUTPUT_FILES "" CACHE STRING "Output files generated by tests") set(TESTS_NAMES "" CACHE STRING "Names of the tests") set(TESTS_SUCCESS "" CACHE BOOL "Success of the tests") -register_tests(TEST_DIRECTORY ${TESTS_DIR} TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_PATH_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) +register_tests(TEST_DIRECTORY ${TESTS_DIR} TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake index 526bcccd3..981334956 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake +++ b/test/end-to-end-tests/single-root-ctest/project-folder/cmake/RegisterTest.cmake @@ -52,15 +52,16 @@ endfunction() # Create and register a test # # Usage: -# register_test(TEST_NAME TEST_OUTPUT_FILE_PATH TEST_SUCCESS ) +# register_test(TEST_DIR TEST_NAME TEST_OUTPUT_FILE TEST_SUCCESS ) # Parameters: +# TEST_DIR: path to the directory where the test will write its output file # TEST_NAME: name of the test -# TEST_OUTPUT_FILE_PATH: path to the file the test should generate +# TEST_OUTPUT_FILE: name of the file the test should generate # TEST_SUCCESS: whether or not the test should end successfully #-------------------------------------------------------------------- function(register_test) set(options) - set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE_PATH;TEST_SUCCESS") + set(oneValueArgs "TEST_DIR;TEST_NAME;TEST_OUTPUT_FILE;TEST_SUCCESS") ### PARSING ARGUMENTS cmake_parse_arguments(register_test "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if(DEFINED register_test_KEYWORDS_MISSING_VALUES) @@ -81,14 +82,14 @@ function(register_test) if(NOT DEFINED register_test_TEST_NAME) message(FATAL_ERROR "The function register_test is awaiting for TEST_NAME keyword") endif() - if(NOT DEFINED register_test_TEST_OUTPUT_FILE_PATH) - message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE_PATH keyword") + if(NOT DEFINED register_test_TEST_OUTPUT_FILE) + message(FATAL_ERROR "The function register_test is awaiting for TEST_OUTPUT_FILE keyword") endif() if(NOT DEFINED register_test_TEST_SUCCESS) message(FATAL_ERROR "The function register_test is awaiting for TEST_SUCCESS keyword") endif() - set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE_PATH}") + set(test_output_file_path "${register_test_TEST_DIR}/${register_test_TEST_OUTPUT_FILE}") message(STATUS "Creating test named ${register_test_TEST_NAME} with result stored in ${test_output_file_path} returning as success: ${register_test_TEST_SUCCESS}") ### GENERATE TEST generate_test_source_file(${test_output_file_path} ${register_test_TEST_SUCCESS}) # => returns test_source @@ -104,16 +105,17 @@ endfunction() # Create and register tests in arguments # # Usage: -# register_tests(TEST_NAME_LIST TEST_OUTPUT_FILE_PATH_LIST TEST_SUCCESS_LIST ) +# register_tests(TEST_DIRECTORY TEST_NAME_LIST TEST_OUTPUT_FILE_LIST TEST_SUCCESS_LIST ) # Parameters: -# TEST_NAME_LIST: name of the test -# TEST_OUTPUT_FILE_PATH_LIST: path to the file the test should generate -# TEST_SUCCESS_LIST: whether or not the test should end successfully +# TEST_DIRECTORY: path to the directory where the tests will write their output files +# TEST_NAME_LIST: list of test names +# TEST_OUTPUT_FILE_LIST: list of file names the tests should generate +# TEST_SUCCESS_LIST: list of boolean values indicating whether or not the tests should end successfully #-------------------------------------------------------------------- function(register_tests) set(options) set(oneValueArgs "TEST_DIRECTORY") - set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_PATH_LIST;TEST_SUCCESS_LIST") + set(multiValueArgs "TEST_NAME_LIST;TEST_OUTPUT_FILE_LIST;TEST_SUCCESS_LIST") ### PARSING ARGUMENTS cmake_parse_arguments(register_tests "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) if(DEFINED register_tests_KEYWORDS_MISSING_VALUES) @@ -134,8 +136,8 @@ function(register_tests) if(NOT DEFINED register_tests_TEST_NAME_LIST) message(FATAL_ERROR "The function register_tests is awaiting for TEST_NAME_LIST keyword") endif() - if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_PATH_LIST) - message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_PATH_LIST keyword") + if(NOT DEFINED register_tests_TEST_OUTPUT_FILE_LIST) + message(FATAL_ERROR "The function register_tests is awaiting for TEST_OUTPUT_FILE_LIST keyword") endif() if(NOT DEFINED register_tests_TEST_SUCCESS_LIST) message(FATAL_ERROR "The function register_tests is awaiting for TEST_SUCCESS_LIST keyword") @@ -144,9 +146,9 @@ function(register_tests) list(LENGTH register_tests_TEST_NAME_LIST NB_TESTS) math(EXPR MAX_INDEX "${NB_TESTS}-1") foreach(test_index RANGE ${MAX_INDEX}) - list(GET register_tests_TEST_OUTPUT_FILE_PATH_LIST ${test_index} test_output) + list(GET register_tests_TEST_OUTPUT_FILE_LIST ${test_index} test_output) list(GET register_tests_TEST_NAME_LIST ${test_index} test_name) list(GET register_tests_TEST_SUCCESS_LIST ${test_index} test_success) - register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE_PATH ${test_output} TEST_SUCCESS ${test_success}) + register_test(TEST_DIR ${register_tests_TEST_DIRECTORY} TEST_NAME ${test_name} TEST_OUTPUT_FILE ${test_output} TEST_SUCCESS ${test_success}) endforeach() endfunction() \ No newline at end of file From d65b68540b34569f256fe3e3f52a9c20b06ede8c Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 13:20:53 +0100 Subject: [PATCH 43/59] First ry to retrieve CMakePresets name in the test --- .../test/ctest-run-tests.test.ts | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 5c9057c17..60991643c 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -9,8 +9,35 @@ import paths from '@cmt/paths'; suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "AllTestsSuccessfull"; + + async function getCMakePresetsAsJson() { + const preset_location: string = path.join(testEnv.projectFolder.location, "CMakePresets.json"); + expect(fs.existsSync(preset_location)).to.eq(true, `CMakePresets.json file ${ preset_location } was not found`); + const content = fs.readFile(preset_location); + expect(content.toLocaleString()).to.not.eq(''); + return JSON.parse(content.toString()); + } + + async function getSpecificPreset(presets_content: any, preset_name: string) { + expect(presets_content['configurePresets']).to.not.eq('', "Unable to find configurePresets section!"); + const all_conf_presets = presets_content['configurePresets']; + for (let index = 0; index < all_conf_presets.length; index++) { + const conf_preset = all_conf_presets[index]; + expect(conf_preset['name']).to.not.eq('', "Unable to find name of the current configure preset!"); + if (conf_preset['name'] === preset_name) { + return conf_preset; + } + } + return undefined; + } async function cleanUpTestResultFiles() { + const used_preset = await getSpecificPreset(await getCMakePresetsAsJson(), usedConfigPreset); + expect(used_preset['cacheVariables']['TESTS_DIR']).to.not.eq('', "Unable to find the TESTS_DIR cache variable in the configure preset!"); + expect(used_preset['cacheVariables']['TESTS_DIR'].split('/')[0]).to.eq(paths.tmpDir, "WARNING: The TEST_DIR variable does not seem to point to the temporary directory!"); + const tests_dir_name = used_preset['cacheVariables']['TESTS_DIR'].split('/')[-1]; + console.log("tests_dir_name is %s", tests_dir_name); const test_dir: string = path.join(paths.tmpDir, "vscode-cmake-tools-tests"); await fs.rmdir(test_dir); const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); @@ -32,7 +59,7 @@ suite('Ctest run tests', () => { await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); - await vscode.commands.executeCommand('cmake.setConfigurePreset', 'AllTestsSuccessfull'); + await vscode.commands.executeCommand('cmake.setConfigurePreset', usedConfigPreset); await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); From 465090e9d56db26488804476cc3f54de98f2107f Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 16:15:29 +0100 Subject: [PATCH 44/59] Remove tmp test dir, its location being read from CMakePresets. --- .../test/ctest-run-tests.test.ts | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 60991643c..564033f2b 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -11,14 +11,25 @@ suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; const usedConfigPreset: string = "AllTestsSuccessfull"; + /** + * + * @returns The content of the CMakePresets.json file as a JSON object + */ async function getCMakePresetsAsJson() { const preset_location: string = path.join(testEnv.projectFolder.location, "CMakePresets.json"); expect(fs.existsSync(preset_location)).to.eq(true, `CMakePresets.json file ${ preset_location } was not found`); - const content = fs.readFile(preset_location); + const content = await fs.readFile(preset_location); expect(content.toLocaleString()).to.not.eq(''); return JSON.parse(content.toString()); } + /** + * Given a CMakePresets.json content, this function will return the configure preset with the given name + * + * @param presets_content: The content of the CMakePresets.json file as a JSON object + * @param preset_name: The name of the configure preset to find + * @returns The configure preset with the given name or undefined if not found + */ async function getSpecificPreset(presets_content: any, preset_name: string) { expect(presets_content['configurePresets']).to.not.eq('', "Unable to find configurePresets section!"); const all_conf_presets = presets_content['configurePresets']; @@ -32,14 +43,15 @@ suite('Ctest run tests', () => { return undefined; } + /** + * This function removes the test result files and the test directory + */ async function cleanUpTestResultFiles() { const used_preset = await getSpecificPreset(await getCMakePresetsAsJson(), usedConfigPreset); expect(used_preset['cacheVariables']['TESTS_DIR']).to.not.eq('', "Unable to find the TESTS_DIR cache variable in the configure preset!"); - expect(used_preset['cacheVariables']['TESTS_DIR'].split('/')[0]).to.eq(paths.tmpDir, "WARNING: The TEST_DIR variable does not seem to point to the temporary directory!"); - const tests_dir_name = used_preset['cacheVariables']['TESTS_DIR'].split('/')[-1]; - console.log("tests_dir_name is %s", tests_dir_name); - const test_dir: string = path.join(paths.tmpDir, "vscode-cmake-tools-tests"); - await fs.rmdir(test_dir); + const test_dir_path = used_preset['cacheVariables']['TESTS_DIR']; + expect("/" + test_dir_path.split('/')[1]).to.eq(paths.tmpDir, `WARNING: The TESTS_DIR variable (${test_dir_path}) does not seem to point to the temporary directory (${paths.tmpDir})!`); + await fs.rmdir(test_dir_path); const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); if (await fs.exists(output_test_path)) { await fs.unlink(output_test_path); From d173bb502ef43193f0813d473d4fc52ed23b02ec Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 16:17:53 +0100 Subject: [PATCH 45/59] Reword test labels --- .../single-root-ctest/test/ctest-run-tests.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 564033f2b..01c8c7efe 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -94,7 +94,7 @@ suite('Ctest run tests', () => { } }); - test('Test of ctest without parallel jobs', async () => { + test('Run ctest without parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -105,7 +105,7 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); - test('Test of ctest without parallel jobs. Use test suite delimiter', async () => { + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -116,7 +116,7 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); - test('Test of ctest with parallel jobs', async () => { + test('Run ctest with parallel jobs', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -127,7 +127,7 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); - test('Test of ctest with parallel jobs. Use test suite delimiter', async () => { + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); From 28ee5080d8ce50c23f1df8d5329ae523767ac130 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 16:19:28 +0100 Subject: [PATCH 46/59] Format --- .../single-root-ctest/test/ctest-run-tests.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 01c8c7efe..f44b09202 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -12,7 +12,7 @@ suite('Ctest run tests', () => { const usedConfigPreset: string = "AllTestsSuccessfull"; /** - * + * * @returns The content of the CMakePresets.json file as a JSON object */ async function getCMakePresetsAsJson() { From aa5d49ee54d7b4d0278bdd12d7fd55bb642962d9 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 16:25:31 +0100 Subject: [PATCH 47/59] Turns method into free function to be reused in other test suites --- .../test/ctest-run-tests.test.ts | 99 ++++++++++--------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index f44b09202..d822b9bbe 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -6,58 +6,59 @@ import { import * as path from 'path'; import * as vscode from 'vscode'; import paths from '@cmt/paths'; +import { use } from 'chai'; + +/** + * Given a CMakePresets.json content, this function will return the configure preset with the given name + * + * @param presets_content: The content of the CMakePresets.json file as a JSON object + * @param preset_name: The name of the configure preset to find + * @returns The configure preset with the given name or undefined if not found + */ +async function getSpecificPreset(presets_content: any, preset_name: string) { + expect(presets_content['configurePresets']).to.not.eq('', "Unable to find configurePresets section!"); + const all_conf_presets = presets_content['configurePresets']; + for (let index = 0; index < all_conf_presets.length; index++) { + const conf_preset = all_conf_presets[index]; + expect(conf_preset['name']).to.not.eq('', "Unable to find name of the current configure preset!"); + if (conf_preset['name'] === preset_name) { + return conf_preset; + } + } + return undefined; +} + +/** + * This function removes the test result files and the test directory + */ +async function cleanUpTestResultFiles(test_env: DefaultEnvironment, configure_preset: string) { + const used_preset = await getSpecificPreset(await getCMakePresetsAsJson(test_env), configure_preset); + expect(used_preset['cacheVariables']['TESTS_DIR']).to.not.eq('', "Unable to find the TESTS_DIR cache variable in the configure preset!"); + const test_dir_path = used_preset['cacheVariables']['TESTS_DIR']; + expect("/" + test_dir_path.split('/')[1]).to.eq(paths.tmpDir, `WARNING: The TESTS_DIR variable (${test_dir_path}) does not seem to point to the temporary directory (${paths.tmpDir})!`); + await fs.rmdir(test_dir_path); + const output_test_path: string = path.join(test_env.projectFolder.location, test_env.buildLocation, test_env.executableResult); + if (await fs.exists(output_test_path)) { + await fs.unlink(output_test_path); + } +} + +/** + * + * @returns The content of the CMakePresets.json file as a JSON object + */ +async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { + const preset_location: string = path.join(test_env.projectFolder.location, "CMakePresets.json"); + expect(fs.existsSync(preset_location)).to.eq(true, `CMakePresets.json file ${preset_location} was not found`); + const content = await fs.readFile(preset_location); + expect(content.toLocaleString()).to.not.eq(''); + return JSON.parse(content.toString()); +} suite('Ctest run tests', () => { let testEnv: DefaultEnvironment; const usedConfigPreset: string = "AllTestsSuccessfull"; - /** - * - * @returns The content of the CMakePresets.json file as a JSON object - */ - async function getCMakePresetsAsJson() { - const preset_location: string = path.join(testEnv.projectFolder.location, "CMakePresets.json"); - expect(fs.existsSync(preset_location)).to.eq(true, `CMakePresets.json file ${ preset_location } was not found`); - const content = await fs.readFile(preset_location); - expect(content.toLocaleString()).to.not.eq(''); - return JSON.parse(content.toString()); - } - - /** - * Given a CMakePresets.json content, this function will return the configure preset with the given name - * - * @param presets_content: The content of the CMakePresets.json file as a JSON object - * @param preset_name: The name of the configure preset to find - * @returns The configure preset with the given name or undefined if not found - */ - async function getSpecificPreset(presets_content: any, preset_name: string) { - expect(presets_content['configurePresets']).to.not.eq('', "Unable to find configurePresets section!"); - const all_conf_presets = presets_content['configurePresets']; - for (let index = 0; index < all_conf_presets.length; index++) { - const conf_preset = all_conf_presets[index]; - expect(conf_preset['name']).to.not.eq('', "Unable to find name of the current configure preset!"); - if (conf_preset['name'] === preset_name) { - return conf_preset; - } - } - return undefined; - } - - /** - * This function removes the test result files and the test directory - */ - async function cleanUpTestResultFiles() { - const used_preset = await getSpecificPreset(await getCMakePresetsAsJson(), usedConfigPreset); - expect(used_preset['cacheVariables']['TESTS_DIR']).to.not.eq('', "Unable to find the TESTS_DIR cache variable in the configure preset!"); - const test_dir_path = used_preset['cacheVariables']['TESTS_DIR']; - expect("/" + test_dir_path.split('/')[1]).to.eq(paths.tmpDir, `WARNING: The TESTS_DIR variable (${test_dir_path}) does not seem to point to the temporary directory (${paths.tmpDir})!`); - await fs.rmdir(test_dir_path); - const output_test_path: string = path.join(testEnv.projectFolder.location, testEnv.buildLocation, testEnv.executableResult); - if (await fs.exists(output_test_path)) { - await fs.unlink(output_test_path); - } - } - suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); @@ -81,11 +82,11 @@ suite('Ctest run tests', () => { }); setup(async function (this: Mocha.Context) { - await cleanUpTestResultFiles(); + await cleanUpTestResultFiles(testEnv, usedConfigPreset); }); teardown(async function (this: Mocha.Context) { - await cleanUpTestResultFiles(); + await cleanUpTestResultFiles(testEnv, usedConfigPreset); }); suiteTeardown(async () => { From f25e4c6487998316b72995df0053befa68426d4b Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 16:30:38 +0100 Subject: [PATCH 48/59] Removes unused import --- .../single-root-ctest/test/ctest-run-tests.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index d822b9bbe..7e8d11055 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -6,7 +6,6 @@ import { import * as path from 'path'; import * as vscode from 'vscode'; import paths from '@cmt/paths'; -import { use } from 'chai'; /** * Given a CMakePresets.json content, this function will return the configure preset with the given name From 164b2acc46e72f09df7ca064e5ce71b421984714 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Thu, 21 Nov 2024 19:05:40 +0100 Subject: [PATCH 49/59] Adds another test suite to check test failure detection --- .../project-folder/CMakePresets.json | 15 ++- .../test/ctest-run-tests.test.ts | 93 ++++++++++++++++++- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index 0a80b3f59..7e11ceda7 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -2,7 +2,7 @@ "version": 2, "configurePresets": [ { - "name": "AllTestsSuccessfull", + "name": "2Successes", "description": "Sets generator, build and install directory, vcpkg", "generator": "Ninja", "binaryDir": "${workspaceFolder}/build", @@ -13,6 +13,19 @@ "TESTS_NAMES": "Suite1.TestA;Suite2.TestB", "TESTS_SUCCESS": "true;true" } + }, + { + "name": "2Successes1Failure", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", + "TESTS_SUCCESS": "true;false;true" + } } ] } diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 7e8d11055..c8312bea1 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -54,9 +54,9 @@ async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { return JSON.parse(content.toString()); } -suite('Ctest run tests', () => { +suite('Ctest: 2 successfull tests', () => { let testEnv: DefaultEnvironment; - const usedConfigPreset: string = "AllTestsSuccessfull"; + const usedConfigPreset: string = "2Successes"; suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); @@ -138,3 +138,92 @@ suite('Ctest run tests', () => { expect(result['test_b']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); }); + +suite('Ctest: 2 successfull tests 1 failing test', () => { + let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "2Successes1Failure"; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + + const build_loc = 'build'; + const exe_res = 'output_test.txt'; + + // CMakePresets.json exist so will use presets by default + testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); + testEnv.projectFolder.buildDirectory.clear(); + + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + await vscode.commands.executeCommand('cmake.setConfigurePreset', usedConfigPreset); + await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); + await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); + await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); + await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); + + await vscode.commands.executeCommand('cmake.build'); + }); + + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + teardown(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + }); + + test('Run ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + }).timeout(100000); +}); \ No newline at end of file From b1e0b35b829248b70b16a215cb6c5553077477a4 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 11:57:46 +0100 Subject: [PATCH 50/59] Typo --- .../single-root-ctest/project-folder/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index adbb9d035..7a0c0d477 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -6,7 +6,7 @@ target_sources(TestUtils PRIVATE test_utils.cpp) target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) -set(TESTS_DIR "" CACHE STRING "Directory where the file generated by tests will be written") +set(TESTS_DIR "" CACHE STRING "Directory where the files generated by tests will be written") configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY) add_library(GetTestDir INTERFACE) target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) From 5a1234f687ce7072dc0f0c2d5fa150fca7785a3f Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 12:01:05 +0100 Subject: [PATCH 51/59] Adds comments --- .../project-folder/CMakeLists.txt | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt index 7a0c0d477..b60c1ead6 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakeLists.txt @@ -4,27 +4,52 @@ project(TestCTestProcess VERSION 0.1.0) add_library(TestUtils SHARED) target_sources(TestUtils PRIVATE test_utils.cpp) target_include_directories(TestUtils PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) -set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) - -set(TESTS_DIR "" CACHE STRING "Directory where the files generated by tests will be written") +set_target_properties(TestUtils PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED + ON) + +set(TESTS_DIR + "" + CACHE STRING "Directory where the files generated by tests will be written") +# Will generate a file get_test_dir.h with the content of get_test_dir.h.in It's +# goal is to provide a function that returns the directory where the tests will +# be executed configure_file("get_test_dir.h.in" "get_test_dir.h" @ONLY) add_library(GetTestDir INTERFACE) target_include_directories(GetTestDir INTERFACE ${CMAKE_CURRENT_BINARY_DIR}) +# Adds an executable that will generate an output file by concatenating the +# content of each file in the test directory add_executable(GenerateOutputFile generate_output_file.cpp) target_link_libraries(GenerateOutputFile PRIVATE GetTestDir) -set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON) +set_target_properties(GenerateOutputFile PROPERTIES CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON) enable_testing() +# The generation of json readable output file must occur after each test run +# Declares a fixture that will be executed after each test run add_test(NAME "Generate_Output_File" COMMAND GenerateOutputFile) set_tests_properties("Generate_Output_File" PROPERTIES FIXTURES_CLEANUP GENOUT) list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake") include(RegisterTest) -set(TESTS_OUTPUT_FILES "" CACHE STRING "Output files generated by tests") -set(TESTS_NAMES "" CACHE STRING "Names of the tests") -set(TESTS_SUCCESS "" CACHE BOOL "Success of the tests") - -register_tests(TEST_DIRECTORY ${TESTS_DIR} TEST_NAME_LIST ${TESTS_NAMES} TEST_OUTPUT_FILE_LIST ${TESTS_OUTPUT_FILES} TEST_SUCCESS_LIST ${TESTS_SUCCESS}) +set(TESTS_OUTPUT_FILES + "" + CACHE STRING "Output files generated by tests") +set(TESTS_NAMES + "" + CACHE STRING "Names of the tests") +set(TESTS_SUCCESS + "" + CACHE BOOL "Success of the tests") + +register_tests( + TEST_DIRECTORY + ${TESTS_DIR} + TEST_NAME_LIST + ${TESTS_NAMES} + TEST_OUTPUT_FILE_LIST + ${TESTS_OUTPUT_FILES} + TEST_SUCCESS_LIST + ${TESTS_SUCCESS}) From f89b1660c5fee9642fe8c5b59c41f4b01467f982 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 19:59:34 +0100 Subject: [PATCH 52/59] Factorize code --- .../test/ctest-run-tests.test.ts | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index c8312bea1..8d239cbf0 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -54,6 +54,19 @@ async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { return JSON.parse(content.toString()); } +async function commonSetup(configure_preset: string) { + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + + await vscode.commands.executeCommand('cmake.setConfigurePreset', configure_preset); + await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); + await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); + await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); + await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); + + await vscode.commands.executeCommand('cmake.build'); +} + suite('Ctest: 2 successfull tests', () => { let testEnv: DefaultEnvironment; const usedConfigPreset: string = "2Successes"; @@ -68,16 +81,7 @@ suite('Ctest: 2 successfull tests', () => { testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); testEnv.projectFolder.buildDirectory.clear(); - await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); - await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); - - await vscode.commands.executeCommand('cmake.setConfigurePreset', usedConfigPreset); - await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); - await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); - await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); - await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); - - await vscode.commands.executeCommand('cmake.build'); + await commonSetup(usedConfigPreset); }); setup(async function (this: Mocha.Context) { @@ -153,16 +157,7 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); testEnv.projectFolder.buildDirectory.clear(); - await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); - await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); - - await vscode.commands.executeCommand('cmake.setConfigurePreset', usedConfigPreset); - await vscode.commands.executeCommand('cmake.setBuildPreset', '__defaultBuildPreset__'); - await vscode.commands.executeCommand('cmake.setTestPreset', '__defaultTestPreset__'); - await vscode.commands.executeCommand('cmake.setPackagePreset', '__defaultPackagePreset__'); - await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); - - await vscode.commands.executeCommand('cmake.build'); + await commonSetup(usedConfigPreset); }); setup(async function (this: Mocha.Context) { From 1bb015b3c8bea37419aa82c4b6a3a26b54da76e6 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 20:27:42 +0100 Subject: [PATCH 53/59] Add newline at eof --- .../single-root-ctest/test/ctest-run-tests.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 8d239cbf0..70e53202f 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -221,4 +221,4 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); }).timeout(100000); -}); \ No newline at end of file +}); From 8fcc6befa5c9948b7af0b8299142189c27fc8d21 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 20:38:13 +0100 Subject: [PATCH 54/59] Typos --- .../single-root-ctest/test/ctest-run-tests.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 70e53202f..6a250a590 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -183,7 +183,7 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { const result = await testEnv.result.getResultAsJson(); expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); - expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); }).timeout(100000); test('Run ctest without parallel jobs. Use test suite delimiter', async () => { @@ -195,7 +195,7 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { const result = await testEnv.result.getResultAsJson(); expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); - expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); }).timeout(100000); test('Run ctest with parallel jobs', async () => { @@ -207,7 +207,7 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { const result = await testEnv.result.getResultAsJson(); expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); - expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); }).timeout(100000); test('Run ctest with parallel jobs. Use test suite delimiter', async () => { @@ -219,6 +219,6 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { const result = await testEnv.result.getResultAsJson(); expect(result['test_a']).to.eq('OK', "Test_a result not found in output"); expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); - expect(result['test_c']).to.eq('OK', "Test_b result not found in output"); + expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); }).timeout(100000); }); From f6eeb39bebf28b19282a1a43b402e81f2959b5ce Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sat, 23 Nov 2024 20:51:13 +0100 Subject: [PATCH 55/59] Adds suite test to check when all tests fail --- .../project-folder/CMakePresets.json | 13 +++ .../test/ctest-run-tests.test.ts | 80 +++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json index 7e11ceda7..bd091181e 100644 --- a/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json +++ b/test/end-to-end-tests/single-root-ctest/project-folder/CMakePresets.json @@ -26,6 +26,19 @@ "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", "TESTS_SUCCESS": "true;false;true" } + }, + { + "name": "3Failures", + "description": "Sets generator, build and install directory, vcpkg", + "generator": "Ninja", + "binaryDir": "${workspaceFolder}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug", + "TESTS_DIR": "/tmp/vscode-cmake-tools-tests", + "TESTS_OUTPUT_FILES": "test_a.txt;test_b.txt;test_c.txt", + "TESTS_NAMES": "Suite1.TestA;Suite2.TestB;Suite2.TestC", + "TESTS_SUCCESS": "false;false;false" + } } ] } diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 6a250a590..63bba46a7 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -222,3 +222,83 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { expect(result['test_c']).to.eq('OK', "Test_c result not found in output"); }).timeout(100000); }); + +suite('Ctest: 3 failing tests', () => { + let testEnv: DefaultEnvironment; + const usedConfigPreset: string = "3Failures"; + + suiteSetup(async function (this: Mocha.Context) { + this.timeout(100000); + + const build_loc = 'build'; + const exe_res = 'output_test.txt'; + + // CMakePresets.json exist so will use presets by default + testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); + testEnv.projectFolder.buildDirectory.clear(); + + await commonSetup(usedConfigPreset); + }); + + setup(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + teardown(async function (this: Mocha.Context) { + await cleanUpTestResultFiles(testEnv, usedConfigPreset); + }); + + suiteTeardown(async () => { + if (testEnv) { + testEnv.teardown(); + } + }); + + test('Run ctest without parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest without parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', false); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', undefined); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); + + test('Run ctest with parallel jobs. Use test suite delimiter', async () => { + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('allowParallelJobs', true); + await vscode.workspace.getConfiguration('cmake.ctest', vscode.workspace.workspaceFolders![0].uri).update('testSuiteDelimiter', "\\."); + await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); + expect(await vscode.commands.executeCommand('cmake.ctest')).to.be.eq(-1); + + const result = await testEnv.result.getResultAsJson(); + expect(result['test_a']).to.eq('KO', "Test_a result not found in output"); + expect(result['test_b']).to.eq('KO', "Test_b result not found in output"); + expect(result['test_c']).to.eq('KO', "Test_c result not found in output"); + }).timeout(100000); +}); From fc734a629eca55a001a7f9cf94d9facba1134a42 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 24 Nov 2024 10:40:33 +0100 Subject: [PATCH 56/59] Adds doc. Removes comment --- .../test/ctest-run-tests.test.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 63bba46a7..72c182e87 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -1,3 +1,19 @@ +/** + * This test suite will test the ctest command with different test results. + * + * Each test suite is defined in the project-folder's CMakePresets.json file through CMake cache variables: + * - TESTS_DIR: The directory where the test results will be stored + * - TESTS_OUTPUT_FILES: A list of file names that will contain the test results + * - TESTS_NAMES: A list of names of the tests to run + * - TESTS_SUCCESS: A list of the expected results of the tests + * + * All of those lists should have the same size. + * + * Each invocation of the ctest command will run the tests defined in the TESTS_NAMES list, storing their TESTS_OUTPUT_FILES + * in the TESTS_DIR directory and will make the test command ends according to the TESTS_SUCCESS list. + * After each invocation of the ctest command, the test results will be concatenated in the output_test.txt file in JSon format so that + * the test suite can check the results. + */ import { fs } from '@cmt/pr'; import { DefaultEnvironment, @@ -54,6 +70,12 @@ async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { return JSON.parse(content.toString()); } +/** + * This function will setup the test environment by setting the configure, build, test, package and workflow presets + * before building the project + * + * @param configure_preset: The name of the configure preset to use + */ async function commonSetup(configure_preset: string) { await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -77,7 +99,6 @@ suite('Ctest: 2 successfull tests', () => { const build_loc = 'build'; const exe_res = 'output_test.txt'; - // CMakePresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); testEnv.projectFolder.buildDirectory.clear(); @@ -153,7 +174,6 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { const build_loc = 'build'; const exe_res = 'output_test.txt'; - // CMakePresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); testEnv.projectFolder.buildDirectory.clear(); @@ -233,7 +253,6 @@ suite('Ctest: 3 failing tests', () => { const build_loc = 'build'; const exe_res = 'output_test.txt'; - // CMakePresets.json exist so will use presets by default testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); testEnv.projectFolder.buildDirectory.clear(); From f904fcfc890405273df5839339354486ef7ed219 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 24 Nov 2024 11:02:14 +0100 Subject: [PATCH 57/59] Moves test env initialization into commonSetup method to avoid code duplication --- .../test/ctest-run-tests.test.ts | 36 +++++++------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 72c182e87..034e101f6 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -75,8 +75,15 @@ async function getCMakePresetsAsJson(test_env: DefaultEnvironment) { * before building the project * * @param configure_preset: The name of the configure preset to use + * @returns The test environment */ async function commonSetup(configure_preset: string) { + const build_loc = 'build'; + const exe_res = 'output_test.txt'; + + const test_env: DefaultEnvironment = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); + test_env.projectFolder.buildDirectory.clear(); + await vscode.workspace.getConfiguration('cmake', vscode.workspace.workspaceFolders![0].uri).update('useCMakePresets', 'always'); await vscode.commands.executeCommand('cmake.getSettingsChangePromise'); @@ -87,6 +94,8 @@ async function commonSetup(configure_preset: string) { await vscode.commands.executeCommand('cmake.setWorkflowPreset', '__defaultWorkflowPreset__'); await vscode.commands.executeCommand('cmake.build'); + + return test_env; } suite('Ctest: 2 successfull tests', () => { @@ -95,14 +104,7 @@ suite('Ctest: 2 successfull tests', () => { suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); - - const build_loc = 'build'; - const exe_res = 'output_test.txt'; - - testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); - testEnv.projectFolder.buildDirectory.clear(); - - await commonSetup(usedConfigPreset); + testEnv = await commonSetup(usedConfigPreset); }); setup(async function (this: Mocha.Context) { @@ -170,14 +172,7 @@ suite('Ctest: 2 successfull tests 1 failing test', () => { suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); - - const build_loc = 'build'; - const exe_res = 'output_test.txt'; - - testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); - testEnv.projectFolder.buildDirectory.clear(); - - await commonSetup(usedConfigPreset); + testEnv = await commonSetup(usedConfigPreset); }); setup(async function (this: Mocha.Context) { @@ -249,14 +244,7 @@ suite('Ctest: 3 failing tests', () => { suiteSetup(async function (this: Mocha.Context) { this.timeout(100000); - - const build_loc = 'build'; - const exe_res = 'output_test.txt'; - - testEnv = new DefaultEnvironment('test/end-to-end-tests/single-root-ctest/project-folder', build_loc, exe_res); - testEnv.projectFolder.buildDirectory.clear(); - - await commonSetup(usedConfigPreset); + testEnv = await commonSetup(usedConfigPreset); }); setup(async function (this: Mocha.Context) { From ad54d5ea959829e3741b4610a4c92dd8f412fb71 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 24 Nov 2024 11:05:30 +0100 Subject: [PATCH 58/59] Adds doc --- .../single-root-ctest/test/ctest-run-tests.test.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts index 034e101f6..85e59d453 100644 --- a/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts +++ b/test/end-to-end-tests/single-root-ctest/test/ctest-run-tests.test.ts @@ -13,6 +13,12 @@ * in the TESTS_DIR directory and will make the test command ends according to the TESTS_SUCCESS list. * After each invocation of the ctest command, the test results will be concatenated in the output_test.txt file in JSon format so that * the test suite can check the results. + * + * Each test suite will have the following tests: + * - Run ctest without parallel jobs + * - Run ctest with parallel jobs + * - Run ctest without parallel jobs. Use test suite delimiter + * - Run ctest with parallel jobs. Use test suite delimiter */ import { fs } from '@cmt/pr'; import { From 648eea60c8464d0cbf8d64327e94fd83f1cc7016 Mon Sep 17 00:00:00 2001 From: Guillaume Peillex Date: Sun, 24 Nov 2024 11:17:07 +0100 Subject: [PATCH 59/59] Remove dead code --- src/ctest.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ctest.ts b/src/ctest.ts index cad1a8b50..62e2815ca 100644 --- a/src/ctest.ts +++ b/src/ctest.ts @@ -710,11 +710,6 @@ export class CTestDriver implements vscode.Disposable { return -1; } - // if (util.isTestMode()) { - // // ProjectController can't be initialized in test mode, so we don't have a usable test explorer - // return 0; - // } - const initializedTestExplorer = this.ensureTestExplorerInitialized(); const sourceDir = util.platformNormalizePath(driver.sourceDir); const testExplorerRoot = initializedTestExplorer.items.get(sourceDir);