From 998b1cea12685ef55b3813693e2cbc2f1fd3c845 Mon Sep 17 00:00:00 2001 From: Virginie Grandgirard Date: Thu, 14 Dec 2023 14:57:42 +0000 Subject: [PATCH] Speed up CI --- .gitlab-ci.yml | 80 ++++++------------- ci_tools/reusable_ci_jobs.yml | 47 +++++++++++ .../2d_spline_interpolator/CMakeLists.txt | 1 + tests/geometryRTheta/CMakeLists.txt | 7 +- .../advection_2d_rp/CMakeLists.txt | 4 +- .../polar_poisson/CMakeLists.txt | 2 + tests/geometryXVx/bump_on_tail/CMakeLists.txt | 4 + tests/geometryXVx/landau/CMakeLists.txt | 4 + tests/geometryXVx/sheath/CMakeLists.txt | 1 + tests/geometryXYVxVy/landau/CMakeLists.txt | 1 + 10 files changed, 92 insertions(+), 59 deletions(-) create mode 100644 ci_tools/reusable_ci_jobs.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 934308c40..1d8e4f3d9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,113 +1,82 @@ + +include: + - local: 'ci_tools/reusable_ci_jobs.yml' + cmake_tests: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - changes: - - vendor/sll/**/* - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - variables: - SLL_BUILD_TESTING: "OFF" - stage: test - needs: [] + extends: .cmake_tests image: ghcr.io/gyselax/gyselalibxx_env timeout: 2 hours - variables: - GIT_SUBMODULE_STRATEGY: recursive - SLL_BUILD_TESTING: "ON" script: | + set -x git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos rm -rf build || true mkdir build cd build - cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DBUILD_BENCHMARKS=ON -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 .. + cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON $GENERAL_CMAKE_OPTIONS .. make -j 4 ctest -j 2 --output-on-failure --timeout 5 -LE test_on_Release_only --output-junit tests.xml artifacts: when: always + expire_in: 1 day reports: junit: ./build/tests.xml + cmake_tests_Release: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - changes: - - vendor/sll/**/* - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - variables: - SLL_BUILD_TESTING: "OFF" - stage: test - needs: [] + extends: .cmake_tests image: ghcr.io/gyselax/gyselalibxx_env - timeout: 2 hours - variables: - GIT_SUBMODULE_STRATEGY: recursive - SLL_BUILD_TESTING: "ON" script: | - set -e + set -x git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos rm -rf build || true mkdir build cd build - cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 .. + cmake -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare" -DBUILD_BENCHMARKS=ON -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS .. make -j 4 ctest -j 2 --output-on-failure --timeout 5 --output-junit tests.xml ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png artifacts: when: always + expire_in: 1 day paths: - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png reports: junit: ./build/tests.xml + cmake_tests_Release_GPU: - rules: - - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - changes: - - vendor/sll/**/* - - if: $CI_PIPELINE_SOURCE == "merge_request_event" - when: manual - variables: - SLL_BUILD_TESTING: "OFF" + extends: .cmake_tests tags: - persee - stage: test - needs: [] - timeout: 2 hours - variables: - GIT_SUBMODULE_STRATEGY: recursive - SLL_BUILD_TESTING: "ON" before_script: | . /data/gyselarunner/spack-0.20.0/share/spack/setup-env.sh spack load gcc@11 spack env activate voicexx-env-omp-cuda script: | - set -e + set -x + export OMP_PROC_BIND=spread + export OMP_PLACES=threads + export OMP_NUM_THREADS=8 git config --global --add safe.directory /builds/gysela-developpers/gyselalibxx/vendor/kokkos rm -rf build || true mkdir build cd build - cmake -DCMAKE_CXX_COMPILER=${CI_PROJECT_DIR}/vendor/kokkos/bin/nvcc_wrapper -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -Xcudafe --diag_suppress=unsigned_compare_with_zero -Xcudafe --diag_suppress=integer_sign_change" -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DCMAKE_BUILD_TYPE=Release -DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_CUDA=ON ../ + cmake -DCMAKE_CXX_COMPILER=${CI_PROJECT_DIR}/vendor/kokkos/bin/nvcc_wrapper \ + -DCMAKE_CXX_FLAGS="-Wall -Wno-sign-compare -Xcudafe --diag_suppress=unsigned_compare_with_zero -Xcudafe --diag_suppress=integer_sign_change" \ + -DCMAKE_BUILD_TYPE=Release $GENERAL_CMAKE_OPTIONS -DKokkos_ARCH_VOLTA70=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON -DKokkos_ENABLE_CUDA=ON .. make -j 8 - export OMP_PROC_BIND=spread - export OMP_PLACES=threads - export OMP_NUM_THREADS=8 ctest --output-on-failure --timeout 5 --output-junit tests.xml -LE ResultsNotTested ls ./tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png ls ./tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png artifacts: when: always + expire_in: 1 day paths: - ./build/tests/geometryXVx/landau/fft/growthrate_t0.0to45.0.png - ./build/tests/geometryXVx/landau/fft/frequency_t0.0to45.0.png reports: junit: ./build/tests.xml + indentation: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH @@ -144,6 +113,7 @@ python_best_practices: for f in $(git diff origin/main --name-only); do grep $f post_process_errors.txt || true; done | tee filtered_errors.txt # Raise an error if post-process in filtered errors ! grep "post-process" filtered_errors.txt >/dev/null + public_mirror: rules: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/ci_tools/reusable_ci_jobs.yml b/ci_tools/reusable_ci_jobs.yml new file mode 100644 index 000000000..8b81c5b4d --- /dev/null +++ b/ci_tools/reusable_ci_jobs.yml @@ -0,0 +1,47 @@ +.setup_env: + stage: .pre + image: ghcr.io/gyselax/gyselalibxx_env + needs: [] + timeout: 10 mins + script: + - FOUND_SLL=$(grep "vendor/sll/.*" -x changed_files.txt || true) + - if [ -z "$FOUND_SLL" ]; then echo "SLL_BUILD_TESTING=OFF" >> build.env; else echo "SLL_BUILD_TESTING=ON" >> build.env; fi + - FOUND_POISSON_2D=$(grep "src/geometryRTheta/poisson/.*" -x changed_files.txt || true) + - if [ -z "$FOUND_POISSON_2D" ]; then echo "POISSON_2D_BUILD_TESTING=OFF" >> build.env; else echo "POISSON_2D_BUILD_TESTING=ON" >> build.env; fi + artifacts: + expire_in: 1 week + reports: + dotenv: build.env + +setup_env_push: + extends: .setup_env + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + before_script: + - set -x + - git log -1 --format=%H | git diff --name-only" > changed_files.txt + +setup_env_mr: + extends: .setup_env + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + before_script: + - set -x + - ${GIT_DIFF_COMMAND} > changed_files.txt + - git config --global --add safe.directory $(pwd) + - git fetch --no-recurse-submodules origin ${CI_MERGE_REQUEST_DIFF_BASE_SHA} + - git diff --name-only ${CI_MERGE_REQUEST_DIFF_BASE_SHA} > changed_files.txt + + +.cmake_tests: + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + needs: [ setup_env_push ] + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + when: manual + needs: [ setup_env_mr ] + stage: test + timeout: 2 hours + variables: + GIT_SUBMODULE_STRATEGY: recursive + GENERAL_CMAKE_OPTIONS: "-DKokkos_ENABLE_DEPRECATED_CODE_3=OFF -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF -DSLL_SPLINES_TEST_DEGREE_MIN=1 -DSLL_SPLINES_TEST_DEGREE_MAX=7 -DSLL_BUILD_TESTING=$SLL_BUILD_TESTING -DPOISSON_2D_BUILD_TESTING=$POISSON_2D_BUILD_TESTING" diff --git a/tests/geometryRTheta/2d_spline_interpolator/CMakeLists.txt b/tests/geometryRTheta/2d_spline_interpolator/CMakeLists.txt index 1149a89b2..687ec123f 100644 --- a/tests/geometryRTheta/2d_spline_interpolator/CMakeLists.txt +++ b/tests/geometryRTheta/2d_spline_interpolator/CMakeLists.txt @@ -26,3 +26,4 @@ add_test(NAME Test_2d_spline_interpolator_Convergence "$") set_property(TEST Test_2d_spline_interpolator_Convergence PROPERTY TIMEOUT 200) +set_property(TEST Test_2d_spline_interpolator_Convergence PROPERTY COST 100) diff --git a/tests/geometryRTheta/CMakeLists.txt b/tests/geometryRTheta/CMakeLists.txt index 6e02494b2..273ff4a2d 100644 --- a/tests/geometryRTheta/CMakeLists.txt +++ b/tests/geometryRTheta/CMakeLists.txt @@ -1,6 +1,11 @@ # SPDX-License-Identifier: MIT -add_subdirectory(polar_poisson) +# Allow the possibility of deactivating slow polar poisson tests +option(POISSON_2D_BUILD_TESTING "Build polar poisson tests" ON) + +if("${POISSON_2D_BUILD_TESTING}") + add_subdirectory(polar_poisson) +endif() add_subdirectory(2d_spline_interpolator) add_subdirectory(quadrature) add_subdirectory(advection_2d_rp) diff --git a/tests/geometryRTheta/advection_2d_rp/CMakeLists.txt b/tests/geometryRTheta/advection_2d_rp/CMakeLists.txt index 4cb43e5c0..7b4863116 100644 --- a/tests/geometryRTheta/advection_2d_rp/CMakeLists.txt +++ b/tests/geometryRTheta/advection_2d_rp/CMakeLists.txt @@ -51,6 +51,7 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING_PHYSICAL" "CZARNY_MAPPIN COMMAND "$" "${CMAKE_CURRENT_SOURCE_DIR}/display_all_errors_for_gtest.py" "$") set_property(TEST TestAdvectionRPConvergence_${MAPPING_TYPE}__${TIME_METHOD}__${SIMULATION} PROPERTY TIMEOUT 20) + set_property(TEST TestAdvectionRPConvergence_${MAPPING_TYPE}__${TIME_METHOD}__${SIMULATION} PROPERTY COST 10) endforeach() endforeach() @@ -89,6 +90,3 @@ target_link_libraries(advection_ALL target_compile_definitions(advection_ALL PUBLIC) find_package(Python3 REQUIRED COMPONENTS Interpreter) - - - diff --git a/tests/geometryRTheta/polar_poisson/CMakeLists.txt b/tests/geometryRTheta/polar_poisson/CMakeLists.txt index 83d0c39e6..3233343dd 100644 --- a/tests/geometryRTheta/polar_poisson/CMakeLists.txt +++ b/tests/geometryRTheta/polar_poisson/CMakeLists.txt @@ -27,6 +27,7 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING") COMMAND "$" "${CMAKE_CURRENT_SOURCE_DIR}/test_poisson.py" "$") set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY TIMEOUT 200) + set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY COST 100) endforeach() endforeach() @@ -60,4 +61,5 @@ foreach(MAPPING_TYPE "CIRCULAR_MAPPING" "CZARNY_MAPPING") COMMAND "$" "${CMAKE_CURRENT_SOURCE_DIR}/test_vlasov_poisson.py" "$") set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY TIMEOUT 200) + set_property(TEST TestPoissonConvergence_${MAPPING_TYPE}_${SOLUTION} PROPERTY COST 100) endforeach() diff --git a/tests/geometryXVx/bump_on_tail/CMakeLists.txt b/tests/geometryXVx/bump_on_tail/CMakeLists.txt index 5bfe94a42..aebb1b9f8 100644 --- a/tests/geometryXVx/bump_on_tail/CMakeLists.txt +++ b/tests/geometryXVx/bump_on_tail/CMakeLists.txt @@ -11,6 +11,7 @@ add_test(NAME TestSimulationBumpontailFFT "$" "fft") set_property(TEST TestSimulationBumpontailFFT PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationBumpontailFFT PROPERTY COST 100) add_test(NAME TestSimulationBumpontailRestartFFT_XVx COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh" @@ -18,6 +19,7 @@ add_test(NAME TestSimulationBumpontailRestartFFT_XVx "$" "restart") set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationBumpontailRestartFFT_XVx PROPERTY COST 100) set_tests_properties(TestSimulationBumpontailRestartFFT_XVx PROPERTIES LABELS "ResultsNotTested" ) add_test(NAME TestSimulationBumpontailFemUniform_xperiod_vx @@ -27,6 +29,7 @@ add_test(NAME TestSimulationBumpontailFemUniform_xperiod_vx "$" "fem") set_property(TEST TestSimulationBumpontailFemUniform_xperiod_vx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationBumpontailFemUniform_xperiod_vx PROPERTY COST 100) add_test(NAME TestSimulationBumpontailRestartFemUniform_xperiod_vx COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_bumpontail_restart.sh" @@ -34,4 +37,5 @@ add_test(NAME TestSimulationBumpontailRestartFemUniform_xperiod_vx "$" "restart") set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTY COST 100) set_tests_properties(TestSimulationBumpontailRestartFemUniform_xperiod_vx PROPERTIES LABELS "ResultsNotTested" ) diff --git a/tests/geometryXVx/landau/CMakeLists.txt b/tests/geometryXVx/landau/CMakeLists.txt index 79f0c8da6..0e164c543 100644 --- a/tests/geometryXVx/landau/CMakeLists.txt +++ b/tests/geometryXVx/landau/CMakeLists.txt @@ -11,6 +11,7 @@ add_test(NAME TestSimulationLandauFFT_XVx "$" "fft") set_property(TEST TestSimulationLandauFFT_XVx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationLandauFFT_XVx PROPERTY COST 100) add_test(NAME TestSimulationLandauRestartFFT_XVx COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh" @@ -18,6 +19,7 @@ add_test(NAME TestSimulationLandauRestartFFT_XVx "$" "restart") set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationLandauRestartFFT_XVx PROPERTY COST 100) set_tests_properties(TestSimulationLandauRestartFFT_XVx PROPERTIES LABELS "ResultsNotTested" ) add_test(NAME TestSimulationLandauFemUniform_xperiod_vx @@ -27,6 +29,7 @@ add_test(NAME TestSimulationLandauFemUniform_xperiod_vx "$" "fem") set_property(TEST TestSimulationLandauFemUniform_xperiod_vx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationLandauFemUniform_xperiod_vx PROPERTY COST 100) add_test(NAME TestSimulationLandauRestartFemUniform_xperiod_vx COMMAND bash "${CMAKE_CURRENT_SOURCE_DIR}/test_landau_restart.sh" @@ -34,4 +37,5 @@ add_test(NAME TestSimulationLandauRestartFemUniform_xperiod_vx "$" "restart") set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTY COST 100) set_tests_properties(TestSimulationLandauRestartFemUniform_xperiod_vx PROPERTIES LABELS "ResultsNotTested" ) diff --git a/tests/geometryXVx/sheath/CMakeLists.txt b/tests/geometryXVx/sheath/CMakeLists.txt index d9f5431a8..4ddcdb165 100644 --- a/tests/geometryXVx/sheath/CMakeLists.txt +++ b/tests/geometryXVx/sheath/CMakeLists.txt @@ -8,4 +8,5 @@ add_test(NAME TestSimulationSheathRestart_xperiod_vx "$" "restart") set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationSheathRestart_xperiod_vx PROPERTY COST 100) set_tests_properties(TestSimulationSheathRestart_xperiod_vx PROPERTIES LABEL "ResultsNotTested" ) diff --git a/tests/geometryXYVxVy/landau/CMakeLists.txt b/tests/geometryXYVxVy/landau/CMakeLists.txt index 164ddb3c0..026b64680 100644 --- a/tests/geometryXYVxVy/landau/CMakeLists.txt +++ b/tests/geometryXYVxVy/landau/CMakeLists.txt @@ -11,3 +11,4 @@ add_test(NAME TestSimulationLandauFFT_XYVxVy "$" "fft") set_property(TEST TestSimulationLandauFFT_XYVxVy PROPERTY TIMEOUT 200) +set_property(TEST TestSimulationLandauFFT_XYVxVy PROPERTY COST 100)