diff --git a/example/decl_hdf5.yml b/example/decl_hdf5.yml index 2d4e28b5f..0779f8ce8 100644 --- a/example/decl_hdf5.yml +++ b/example/decl_hdf5.yml @@ -28,6 +28,7 @@ pdi: write: main_field: # the name of the data to write dataset: data + mpio: COLLECTIVE # or INDEPENDENT when: '$iter<10' # do only write the first 10 iterations (0...9) memory_selection: # exclude ghosts from the data in memory size: ['$dsize[0]-2', '$dsize[1]-2'] diff --git a/plugins/decl_hdf5/AUTHORS b/plugins/decl_hdf5/AUTHORS index 7aeb5fb44..c82136f02 100644 --- a/plugins/decl_hdf5/AUTHORS +++ b/plugins/decl_hdf5/AUTHORS @@ -11,6 +11,12 @@ Julien Bigot - CEA (julien.bigot@cea.fr) * Run tests that depend on the filesystem in their own temporary directory * Buildsystem +Jacques Morice - CEA (jacques.morice@cea.fr) +* contribution to feature improvement, validation + +Yacine Ould Rouis - CNRS (yacine.ould-rouis@inria.fr) +* contribution to feature design, validation + Thomas Padioleau - CEA (thomas.padioleau@cea.fr) * Fixed a bug with parallel file deletion diff --git a/plugins/decl_hdf5/CHANGELOG.md b/plugins/decl_hdf5/CHANGELOG.md index 4aef9cd26..f2d0ee2d6 100644 --- a/plugins/decl_hdf5/CHANGELOG.md +++ b/plugins/decl_hdf5/CHANGELOG.md @@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [Unreleased] ### Added +* Possibility to choose parallel MPI-I/O mode: either COLLECTIVE or INDEPENDENT + [#419](https://gitlab.maisondelasimulation.fr/pdidev/pdi/-/issues/419) ### Changed diff --git a/plugins/decl_hdf5/README.md b/plugins/decl_hdf5/README.md index acf02947e..a46db8676 100644 --- a/plugins/decl_hdf5/README.md +++ b/plugins/decl_hdf5/README.md @@ -145,6 +145,8 @@ The possible values for the keys are as follow: See https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetFletcher32 for more information. +* `mpio` : a string expression to define the type of MPI-I/O parallel pointer +for the operation among two choices : `COLLECTIVE` (default) and `INDEPENDENT`. ### SELECTION_DESC @@ -219,6 +221,7 @@ plugins: - dataset: data/array # a dataset name (default: the data name) when: "$iter>0&$iter<11" # an additional condition when to actually trigger the actions (default: always true) communicator: $MPI_COMM_SELF # the MPI communicator used for HDF5 parallel synchronized write (default: that of the file) + mpio: INDEPENDENT # the type of MPI-I/O parallel pointer (default: COLLECTIVE) memory_selection: size: [$width-2, $height-2] # number of elements to transfer in each dimension (default: size of the full data) start: [1, 1] # coordinate of the start point in memory relative to the shared data (default: 0 in each dimensions) diff --git a/plugins/decl_hdf5/dataset_op.cxx b/plugins/decl_hdf5/dataset_op.cxx index 7da392e9e..3e627067a 100644 --- a/plugins/decl_hdf5/dataset_op.cxx +++ b/plugins/decl_hdf5/dataset_op.cxx @@ -1,6 +1,6 @@ /******************************************************************************* * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA) - * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * Copyright (C) 2021-2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -164,6 +164,14 @@ Dataset_op::Dataset_op(Direction dir, string name, Expression default_when, PC_t m_fletcher = value; } else if (key == "attributes") { // pass + } else if (key == "mpio") { + if (to_string(value) == "INDEPENDENT") { + m_mpio = H5FD_MPIO_INDEPENDENT; + } else if (to_string(value) == "COLLECTIVE") { + m_mpio = H5FD_MPIO_COLLECTIVE; + } else { + throw Config_error{key_tree, "Not valid mpio value: `{}'. Expecting INDEPENDENT or COLLECTIVE.", to_string(value)}; + } } else if (key == "collision_policy") { m_collision_policy = to_collision_policy(to_string(value)); } else { @@ -203,8 +211,12 @@ void Dataset_op::fletcher(Context& ctx, Expression value) } } -void Dataset_op::execute(Context& ctx, hid_t h5_file, hid_t xfer_lst, const unordered_map& dsets) +void Dataset_op::execute(Context& ctx, hid_t h5_file, bool use_mpio, const unordered_map& dsets) { + Raii_hid xfer_lst = make_raii_hid(H5Pcreate(H5P_DATASET_XFER), H5Pclose); + if (use_mpio) { + if (0 > H5Pset_dxpl_mpio(xfer_lst, m_mpio)) handle_hdf5_err(); + } if (m_direction == READ) do_read(ctx, h5_file, xfer_lst); else @@ -328,7 +340,7 @@ hid_t Dataset_op::dataset_creation_plist(Context& ctx, const Datatype* dataset_t return dset_plist; } -void Dataset_op::do_write(Context& ctx, hid_t h5_file, hid_t write_lst, const unordered_map& dsets) +void Dataset_op::do_write(Context& ctx, hid_t h5_file, hid_t write_lst, const unordered_map& dsets) { string dataset_name = m_dataset.to_string(ctx); ctx.logger().trace("Preparing for writing `{}' dataset", dataset_name); diff --git a/plugins/decl_hdf5/dataset_op.h b/plugins/decl_hdf5/dataset_op.h index b5d3be40e..aaa63bec8 100644 --- a/plugins/decl_hdf5/dataset_op.h +++ b/plugins/decl_hdf5/dataset_op.h @@ -1,6 +1,6 @@ /******************************************************************************* * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA) - * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * Copyright (C) 2021-2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -67,6 +67,9 @@ class Dataset_op /// direction of the transfer (read or write) Direction m_direction; + /// the type of MPI-I/O parallel pointer (default = COLLECTIVE) + H5FD_mpio_xfer_t m_mpio = H5FD_MPIO_COLLECTIVE; + /// the name of the dataset where to transfer PDI::Expression m_dataset; @@ -182,11 +185,10 @@ class Dataset_op * * \param ctx the context in which to operate * \param h5_file the already opened HDF5 file id - * \param xfer_lst the already created transfer property list including any - * parallel HDF5 required property. + * \param use_mpio whether the hdf5 read/write is parallel * \param dsets the type of the explicitly typed datasets */ - void execute(PDI::Context& ctx, hid_t h5_file, hid_t xfer_lst, const std::unordered_map& dsets); + void execute(PDI::Context& ctx, hid_t h5_file, bool use_mpio, const std::unordered_map& dsets); private: void do_read(PDI::Context& ctx, hid_t h5_file, hid_t read_lst); diff --git a/plugins/decl_hdf5/file_op.cxx b/plugins/decl_hdf5/file_op.cxx index 1f084b9cd..61f296823 100644 --- a/plugins/decl_hdf5/file_op.cxx +++ b/plugins/decl_hdf5/file_op.cxx @@ -1,6 +1,6 @@ /******************************************************************************* * Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA) - * Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * Copyright (C) 2021-2022 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -280,7 +280,7 @@ void File_op::execute(Context& ctx) std::string filename = m_file.to_string(ctx); Raii_hid file_lst = make_raii_hid(H5Pcreate(H5P_FILE_ACCESS), H5Pclose); - Raii_hid xfer_lst = make_raii_hid(H5Pcreate(H5P_DATASET_XFER), H5Pclose); + bool use_mpio = false; #ifdef H5_HAVE_PARALLEL MPI_Comm comm = MPI_COMM_SELF; if (communicator()) { @@ -288,7 +288,7 @@ void File_op::execute(Context& ctx) } if (comm != MPI_COMM_SELF) { if (0 > H5Pset_fapl_mpio(file_lst, comm, MPI_INFO_NULL)) handle_hdf5_err(); - if (0 > H5Pset_dxpl_mpio(xfer_lst, H5FD_MPIO_COLLECTIVE)) handle_hdf5_err(); + use_mpio = true; ctx.logger().debug("Opening `{}' file in parallel mode", filename); } #endif @@ -337,10 +337,10 @@ void File_op::execute(Context& ctx) Raii_hid h5_file = make_raii_hid(h5_file_raw, H5Fclose, ("Cannot open `" + filename + "' file").c_str()); for (auto&& one_dset_op: dset_writes) { - one_dset_op.execute(ctx, h5_file, xfer_lst, m_datasets); + one_dset_op.execute(ctx, h5_file, use_mpio, m_datasets); } for (auto&& one_dset_op: dset_reads) { - one_dset_op.execute(ctx, h5_file, xfer_lst, m_datasets); + one_dset_op.execute(ctx, h5_file, use_mpio, m_datasets); } for (auto&& one_attr_op: attr_writes) { one_attr_op.execute(ctx, h5_file); diff --git a/plugins/decl_hdf5/tests/CMakeLists.txt b/plugins/decl_hdf5/tests/CMakeLists.txt index ed58f39f1..d67503cb6 100644 --- a/plugins/decl_hdf5/tests/CMakeLists.txt +++ b/plugins/decl_hdf5/tests/CMakeLists.txt @@ -1,7 +1,7 @@ #============================================================================= # Copyright (C) 2015-2024 Commissariat a l'energie atomique et aux energies alternatives (CEA) # Copyright (C) 2021 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) -# +# Copyright (C) 2022 Centre National de Recherche Scientifique (CNRS) # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -95,7 +95,7 @@ if("${BUILD_HDF5_PARALLEL}") set_property(TEST decl_hdf5_mpi_04_C PROPERTY PROCESSORS 4) endif() -# bcommunicator as a reference +# communicator as a reference if("${BUILD_HDF5_PARALLEL}") add_executable(decl_hdf5_mpi_05_C decl_hdf5_mpi_test_05.c) target_link_libraries(decl_hdf5_mpi_05_C PDI::PDI_C MPI::MPI_C) @@ -113,6 +113,15 @@ if("${BUILD_HDF5_PARALLEL}") set_property(TEST decl_hdf5_mpi_06_C PROPERTY PROCESSORS 4) endif() +# mpio dataset attribute +if("${BUILD_HDF5_PARALLEL}") + add_executable(decl_hdf5_mpi_07_C decl_hdf5_mpi_test_07.c) + target_link_libraries(decl_hdf5_mpi_07_C PDI::PDI_C MPI::MPI_C) + add_test(NAME decl_hdf5_mpi_07_C COMMAND "${RUNTEST_DIR}" "${MPIEXEC}" "${MPIEXEC_NUMPROC_FLAG}" 4 ${MPIEXEC_PREFLAGS} "$" ${MPIEXEC_POSTFLAGS}) + set_property(TEST decl_hdf5_mpi_07_C PROPERTY TIMEOUT 15) + set_property(TEST decl_hdf5_mpi_07_C PROPERTY PROCESSORS 4) +endif() + add_executable(decl_hdf5_IO_options_C decl_hdf5_test_IO_options.c) target_link_libraries(decl_hdf5_IO_options_C PDI::PDI_C ${HDF5_DEPS}) add_test(NAME decl_hdf5_IO_options_C COMMAND "${RUNTEST_DIR}" "$") diff --git a/plugins/decl_hdf5/tests/compatibility_tests/CMakeLists.txt b/plugins/decl_hdf5/tests/compatibility_tests/CMakeLists.txt index 582c4361d..a390af759 100644 --- a/plugins/decl_hdf5/tests/compatibility_tests/CMakeLists.txt +++ b/plugins/decl_hdf5/tests/compatibility_tests/CMakeLists.txt @@ -29,7 +29,7 @@ set(RUNTEST_DIR "${CMAKE_CURRENT_LIST_DIR}/../../cmake/runtest-dir") set(ALL_TEST_NAMES array dataset group struct variables) if("${BUILD_HDF5_PARALLEL}") - list(APPEND ALL_TEST_NAMES mpi) + list(APPEND ALL_TEST_NAMES mpi mpi_independent) endif() foreach(test_name ${ALL_TEST_NAMES}) @@ -39,7 +39,7 @@ foreach(test_name ${ALL_TEST_NAMES}) add_executable("PDI_C_${test_name}_${io_operation}_test" "PDI_C/${test_name}_${io_operation}_test.c") target_link_libraries("PDI_C_${test_name}_${io_operation}_test" PDI::PDI_C) - if ("${test_name}" STREQUAL "mpi") + if ( ("${test_name}" STREQUAL "mpi") OR ("${test_name}" STREQUAL "mpi_independent") ) target_link_libraries("HDF5_C_${test_name}_${io_operation}_test" MPI::MPI_C) target_link_libraries("PDI_C_${test_name}_${io_operation}_test" MPI::MPI_C) endif() @@ -49,7 +49,7 @@ foreach(test_name ${ALL_TEST_NAMES}) foreach(read_prefix HDF5_C_ PDI_C_) set(write_test_name "${write_prefix}${test_name}_write_test") set(read_test_name "${read_prefix}${test_name}_read_test") - if ("${test_name}" STREQUAL "mpi") + if ( ("${test_name}" STREQUAL "mpi") OR ("${test_name}" STREQUAL "mpi_independent") ) set(MPI_CASE_CMD "${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_PREFLAGS}") set(MPI_POST_CMD " ${MPIEXEC_POSTFLAGS}") else() diff --git a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_read_test.c b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_read_test.c new file mode 100644 index 000000000..f7b9da615 --- /dev/null +++ b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_read_test.c @@ -0,0 +1,144 @@ +/******************************************************************************* + * Copyright (C) 2020 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of CEA nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include +#include +#include +#include + +#define FILE "mpi_independent_test.h5" + +/** +* Test : Read a file using hdf5 parallel version with the option independent parallel pointer. +*/ + +int main(int argc, char* argv[]) +{ + printf("HDF5 mpi_independent_read_test started\n"); + MPI_Init(&argc, &argv); + int mpi_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + herr_t status = H5open(); + if (status < 0) { + return 1; + } + + hid_t fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); + hid_t file_id = H5Fopen(FILE, H5F_ACC_RDONLY, fapl_id); + if (file_id < 0) { + return 1; + } + + int dset_data[5][5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + dset_data[i][j] = 0; + } + } + + hsize_t coords[2] = {5, 10}; + hid_t dataspace_id = H5Screate_simple(2, coords, NULL); + if (dataspace_id < 0) { + return 1; + } + + hid_t dataset_id = H5Dopen2(file_id, "array_data", H5P_DEFAULT); + if (dataset_id < 0) { + return 1; + } + + hsize_t count[2] = {5, 5}; + hsize_t stride[2] = {1, 1}; + hsize_t dataset_offset[2] = {0, 5 * mpi_rank}; + hsize_t memory_offset[2] = {0, 0}; + hsize_t block[2] = {1, 1}; + + status = H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, dataset_offset, stride, count, block); + if (status < 0) { + return 1; + } + hid_t memory_dataspace_id = H5Screate_simple(2, count, NULL); + if (memory_dataspace_id < 0) { + return 1; + } + status = H5Sselect_hyperslab(memory_dataspace_id, H5S_SELECT_SET, memory_offset, stride, count, block); + if (status < 0) { + return 1; + } + + hid_t dxpl_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_INDEPENDENT); + + status = H5Dread(dataset_id, H5T_NATIVE_INT, memory_dataspace_id, dataspace_id, dxpl_id, dset_data); + if (status < 0) { + return 1; + } + + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + if (dset_data[i][j] != i * 10 + j + (5 * mpi_rank)) { + fprintf(stderr, "[%d][%d] %d != %d\n ", i, j, dset_data[i][j], i * 10 + j + (5 * mpi_rank)); + return 1; + } + } + } + + status = H5Dclose(dataset_id); + if (status < 0) { + return 1; + } + status = H5Pclose(dxpl_id); + if (status != 0) { + return status; + } + status = H5Sclose(memory_dataspace_id); + if (status < 0) { + return 1; + } + status = H5Sclose(dataspace_id); + if (status < 0) { + return 1; + } + status = H5Fclose(file_id); + if (file_id < 0) { + return 1; + } + if (status != 0) { + return status; + } + status = H5Pclose(fapl_id); + if (status != 0) { + return status; + } + + H5close(); + if (status < 0) { + return 1; + } + MPI_Finalize(); + + printf("[Rank: %d] HDF5 mpi_read_test finalized\n", mpi_rank); + return 0; +} diff --git a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_write_test.c b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_write_test.c new file mode 100644 index 000000000..af6469a6c --- /dev/null +++ b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_independent_write_test.c @@ -0,0 +1,135 @@ +/******************************************************************************* + * Copyright (C) 2020 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of CEA nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include +#include +#include +#include + +#define FILE "mpi_independent_test.h5" + +/** +* Test : Write a file using hdf5 parallel version with the option independent parallel pointer. +*/ + +int main(int argc, char* argv[]) +{ + printf("HDF5 mpi_independent_write_test started\n"); + MPI_Init(&argc, &argv); + int mpi_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + herr_t status = H5open(); + if (status < 0) { + return 1; + } + + hid_t fapl_id = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); + hid_t file_id = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id < 0) { + return 1; + } + + hsize_t coords[2] = {5, 10}; + hid_t dataspace_id = H5Screate_simple(2, coords, NULL); + if (dataspace_id < 0) { + return 1; + } + + int dset_data[5][5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + dset_data[i][j] = i * 10 + j + (5 * mpi_rank); + } + } + + hid_t dataset_id = H5Dcreate2(file_id, "/array_data", H5T_STD_I32BE, dataspace_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dataset_id < 0) { + return 1; + } + + hsize_t count[2] = {5, 5}; + hsize_t stride[2] = {1, 1}; + hsize_t dataset_offset[2] = {0, 5 * mpi_rank}; + hsize_t memory_offset[2] = {0, 0}; + hsize_t block[2] = {1, 1}; + + status = H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, dataset_offset, stride, count, block); + if (status < 0) { + return 1; + } + hid_t memory_dataspace_id = H5Screate_simple(2, count, NULL); + if (memory_dataspace_id < 0) { + return 1; + } + status = H5Sselect_hyperslab(memory_dataspace_id, H5S_SELECT_SET, memory_offset, stride, count, block); + if (status < 0) { + return 1; + } + + hid_t dxpl_id = H5Pcreate(H5P_DATASET_XFER); + H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_INDEPENDENT); + + + status = H5Dwrite(dataset_id, H5T_NATIVE_INT, memory_dataspace_id, dataspace_id, dxpl_id, dset_data); + if (status < 0) { + return 1; + } + + status = H5Sclose(memory_dataspace_id); + if (status < 0) { + return 1; + } + status = H5Sclose(dataspace_id); + if (status < 0) { + return 1; + } + status = H5Dclose(dataset_id); + if (status < 0) { + return 1; + } + status = H5Pclose(dxpl_id); + if (status != 0) { + return status; + } + status = H5Fclose(file_id); + if (file_id < 0) { + return 1; + } + if (status != 0) { + return status; + } + status = H5Pclose(fapl_id); + if (status != 0) { + return status; + } + + H5close(); + if (status < 0) { + return 1; + } + MPI_Finalize(); + printf("[Rank: %d] HDF5 mpi_write_test finalized\n", mpi_rank); + return 0; +} diff --git a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_read_test.c b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_read_test.c index b8cf6c592..730e29589 100644 --- a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_read_test.c +++ b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_read_test.c @@ -29,10 +29,14 @@ #define FILE "mpi_test.h5" -int main() +/** +* Test : Read a file using hdf5 parallel version with the option collective parallel pointer. +*/ + +int main(int argc, char* argv[]) { printf("HDF5 mpi_read_test started\n"); - MPI_Init(NULL, NULL); + MPI_Init(&argc, &argv); int mpi_rank; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); herr_t status = H5open(); diff --git a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_write_test.c b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_write_test.c index 0c5da2633..cc3cc6ed9 100644 --- a/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_write_test.c +++ b/plugins/decl_hdf5/tests/compatibility_tests/HDF5_C/mpi_write_test.c @@ -29,10 +29,14 @@ #define FILE "mpi_test.h5" -int main() +/** +* Test : Write a file using hdf5 parallel version with the option collective parallel pointer. +*/ + +int main(int argc, char* argv[]) { printf("HDF5 mpi_write_test started\n"); - MPI_Init(NULL, NULL); + MPI_Init(&argc, &argv); int mpi_rank; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); herr_t status = H5open(); diff --git a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_read_test.c b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_read_test.c new file mode 100644 index 000000000..284f0c88e --- /dev/null +++ b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_read_test.c @@ -0,0 +1,95 @@ +/******************************************************************************* + * Copyright (C) 2020 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * Copyright (C) 2020 Commissariat a l'energie atomique et aux energies alternatives (CEA) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of CEA nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include +#include +#include +#include + +#define FILE "mpi_independent_test.h5" + +/** +* Test : Read a file using PDI with the plugin decl_hdf5 in parallel with the option mpio: INDEPENDENT. +*/ + +int main(int argc, char* argv[]) +{ + printf("PDI mpi_independent_read_test started\n"); + MPI_Init(&argc, &argv); + int mpi_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + const char* CONFIG_YAML + = "logging: trace \n" + "metadata: \n" + " mpi_rank: int \n" + "data: \n" + " array_data: { size: [5, 5], type: array, subtype: int } \n" + "plugins: \n" + " mpi: ~ \n" + " decl_hdf5: \n" + " communicator: $MPI_COMM_WORLD \n" + " file: mpi_independent_test.h5 \n" + " datasets: \n" + " array_data: {type: array, subtype: int, size: [5, 10]} \n" + " read: \n" + " mpio: INDEPENDENT \n" + " array_data: \n" + " - memory_selection: \n" + " size: [5, 5] \n" + " dataset_selection: \n" + " size: [5, 5] \n" + " start: [0, $mpi_rank * 5] \n"; + + PC_tree_t conf = PC_parse_string(CONFIG_YAML); + PDI_init(conf); + + PDI_expose("mpi_rank", &mpi_rank, PDI_OUT); + + int test_array[5][5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + test_array[i][j] = 0; + } + } + PDI_expose("array_data", test_array, PDI_IN); + + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + if (test_array[i][j] != i * 10 + j + 5 * mpi_rank) { + fprintf(stderr, "[%d][%d] %d != %d\n ", i, j, test_array[i][j], i * 10 + j + 5 * mpi_rank); + return 1; + } + } + } + + PDI_finalize(); + PC_tree_destroy(&conf); + + MPI_Finalize(); + + printf("[Rank: %d] PDI mpi_read_test finalized\n", mpi_rank); + return 0; +} diff --git a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_write_test.c b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_write_test.c new file mode 100644 index 000000000..7e34810e5 --- /dev/null +++ b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_independent_write_test.c @@ -0,0 +1,87 @@ +/******************************************************************************* + * Copyright (C) 2020 Institute of Bioorganic Chemistry Polish Academy of Science (PSNC) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of CEA nor the names of its contributors may be used to + * endorse or promote products derived from this software without specific + * prior written permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include +#include +#include +#include + +#define FILE "mpi_independent_test.h5" + +/** +* Test : Write a file using PDI with the plugin decl_hdf5 in parallel with the option mpio: INDEPENDENT. +*/ + +int main(int argc, char* argv[]) +{ + printf("PDI mpi_independent_write_test started\n"); + MPI_Init(&argc, &argv); + int mpi_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + const char* CONFIG_YAML + = "logging: trace \n" + "metadata: \n" + " mpi_rank: int \n" + "data: \n" + " array_data: { size: [5, 5], type: array, subtype: int } \n" + "plugins: \n" + " mpi: ~ \n" + " decl_hdf5: \n" + " communicator: $MPI_COMM_WORLD \n" + " file: mpi_independent_test.h5 \n" + " datasets: \n" + " array_data: {type: array, subtype: int, size: [5, 10]} \n" + " write: \n" + " mpio: INDEPENDENT \n" + " array_data: \n" + " - memory_selection: \n" + " size: [5, 5] \n" + " dataset_selection: \n" + " size: [5, 5] \n" + " start: [0, $mpi_rank * 5] \n"; + + PC_tree_t conf = PC_parse_string(CONFIG_YAML); + PDI_init(conf); + + PDI_expose("mpi_rank", &mpi_rank, PDI_OUT); + + int test_array[5][5]; + + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + test_array[i][j] = i * 10 + j + 5 * mpi_rank; + } + } + + PDI_expose("array_data", test_array, PDI_OUT); + + PDI_finalize(); + PC_tree_destroy(&conf); + + MPI_Finalize(); + + printf("[Rank: %d] HDF5 mpi_write_test finalized\n", mpi_rank); + return 0; +} diff --git a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_read_test.c b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_read_test.c index 9b440f998..0871cc50c 100644 --- a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_read_test.c +++ b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_read_test.c @@ -30,10 +30,10 @@ #define FILE "mpi_test.h5" -int main() +int main(int argc, char* argv[]) { printf("PDI mpi_read_test started\n"); - MPI_Init(NULL, NULL); + MPI_Init(&argc, &argv); int mpi_rank; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); diff --git a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_write_test.c b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_write_test.c index f126451ce..054fb11f9 100644 --- a/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_write_test.c +++ b/plugins/decl_hdf5/tests/compatibility_tests/PDI_C/mpi_write_test.c @@ -29,10 +29,14 @@ #define FILE "mpi_test.h5" -int main() +/** +* Test : Write a file using PDI with the plugin decl_hdf5 in parallel with the option mpio: COLLECTIVE(DEFAULT). +*/ + +int main(int argc, char* argv[]) { printf("PDI mpi_write_test started\n"); - MPI_Init(NULL, NULL); + MPI_Init(&argc, &argv); int mpi_rank; MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); diff --git a/plugins/decl_hdf5/tests/decl_hdf5_mpi_test_07.c b/plugins/decl_hdf5/tests/decl_hdf5_mpi_test_07.c new file mode 100644 index 000000000..df20f5252 --- /dev/null +++ b/plugins/decl_hdf5/tests/decl_hdf5_mpi_test_07.c @@ -0,0 +1,232 @@ +/******************************************************************************* + * Copyright (C) 2015-2019 Commissariat a l'energie atomique et aux energies alternatives (CEA) + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * forcumentation and/or other materials provided with the distribution. + * * Neither the name of CEA nor the names of its contributors may be used to + * enforrse or promote products derived from this software without specific + * prior written permission. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + ******************************************************************************/ + +#include +#include +#include + +#define IMX 5 +#define JMX 4 +#define NI_GHOST 1 +#define NJ_GHOST 2 +#define DIM 2 + +const char* CONFIG_YAML + = "logging: trace \n" + "metadata: \n" + " input: int \n" + " ni: int \n" + " nj: int \n" + " nig: int \n" + " njg: int \n" + " nit: int \n" + " njt: int \n" + " istart: int \n" + " jstart: int \n" + "data: \n" + " reals: \n" + " type: array \n" + " subtype: double \n" + " size: [$nj + 2*$njg, $ni + 2*$nig] \n" + " subsize: [$nj, $ni] \n" + " start: [$njg, $nig] \n" + " values: \n" + " type: array \n" + " subtype: int \n" + " size: [$nj + 2*$njg, $ni + 2*$nig] \n" + " subsize: [$nj, $ni] \n" + " start: [$njg, $nig] \n" + "plugins: \n" + " mpi: \n" + " decl_hdf5: \n" + " file: decl_hdf5_mpi_test_07_C.h5 \n" + " communicator: $MPI_COMM_WORLD \n" + " datasets: \n" + " reals: {type: array, subtype: double, size: [$njt, $nit]} \n" + " values: {type: array, subtype: int, size: [$njt, $nit]} \n" + " write: \n" + " reals: \n" + " when: $input=0 \n" + " mpio: INDEPENDENT \n" + " dataset_selection: {start: [$jstart, $istart]} \n" + " values: \n" + " mpio: COLLECTIVE \n" + " when: $input=0 \n" + " dataset_selection: {start: [$jstart, $istart]} \n" + " read: \n" + " reals: \n" + " mpio: COLLECTIVE \n" + " when: $input=1 \n" + " dataset_selection: {start: [$jstart, $istart]} \n" + " values: \n" + " mpio: INDEPENDENT \n" + " when: $input=1 \n" + " dataset_selection: {start: [$jstart, $istart]} \n"; + +int main(int argc, char* argv[]) +{ + const int icst = -1; /// constants values in the ghost nodes + const double rcst = -1.01; + + int nig = NI_GHOST, njg = NJ_GHOST; + int ni = IMX, nj = JMX; + int values[JMX + 2 * NJ_GHOST][IMX + NI_GHOST * 2] = {{0}}, cp_values[JMX + 2 * NJ_GHOST][IMX + NI_GHOST * 2] = {{0}}; + double reals[JMX + 2 * NJ_GHOST][IMX + NI_GHOST * 2] = {{0}}, cp_reals[JMX + 2 * NJ_GHOST][IMX + NI_GHOST * 2] = {{0}}; + int i, j, input; + int nit, njt; + + /// MPI and parallel data or info + int dims[DIM], coord[DIM], periodic[DIM]; + int istart, jstart; + MPI_Comm comm2D; + periodic[0] = 0; + periodic[1] = 0; + dims[0] = 2; + dims[1] = 2; + + + MPI_Init(&argc, &argv); + PC_tree_t conf = PC_parse_string(CONFIG_YAML); + MPI_Comm world = MPI_COMM_WORLD; + PDI_init(conf); + int rank; + MPI_Comm_rank(world, &rank); + + if (0 == rank) { + remove("decl_hdf5_mpi_test_07_C.h5"); + } + + { + /// setting nb of procs. + int size; + MPI_Comm_size(world, &size); + if (size != 4) { + printf("Run on 4 procs only."); + MPI_Abort(MPI_COMM_WORLD, -1); + } + PDI_expose("nproc", &size, PDI_OUT); + } + + + MPI_Cart_create(world, DIM, dims, periodic, 0, &comm2D); + MPI_Cart_coords(comm2D, rank, DIM, coord); + + istart = coord[1] * ni; + jstart = coord[0] * nj; + + nit = 2 * ni; + njt = 2 * nj; + + PDI_expose("nig", &nig, PDI_OUT); /// Ghost cells + PDI_expose("njg", &njg, PDI_OUT); + + PDI_expose("ni", &ni, PDI_OUT); /// Size of the portion of the array for a given MPI task + PDI_expose("nj", &nj, PDI_OUT); + + PDI_expose("nit", &nit, PDI_OUT); /// size of the distributed array + PDI_expose("njt", &njt, PDI_OUT); + + PDI_expose("istart", &istart, PDI_OUT); /// offset + PDI_expose("jstart", &jstart, PDI_OUT); + + // Fill arrays + for (j = 0; j < nj + 2 * njg; ++j) { + for (i = 0; i < ni + 2 * nig; ++i) { + cp_values[j][i] = icst; + cp_reals[j][i] = rcst; /// array initialized with const values + } + } + /// Values and reals == 0 in the ghost. + double cst = -rcst; + for (j = njg; j < nj + njg; ++j) { + for (i = nig; i < ni + nig; ++i) { + values[j][i] = (i + coord[1] * ni - nig) + (j + coord[0] * nj - njg) * 10; + reals[j][i] = (i + coord[1] * ni - nig) * cst + (j + coord[0] * nj - njg) * 10 * cst; + } + } + + input = 0; + PDI_expose("rank", &rank, PDI_OUT); + PDI_expose("input", &input, PDI_OUT); + + /// Test that export/exchange works + PDI_expose("input", &input, PDI_OUT); + PDI_expose("reals", &reals, PDI_OUT); // output real + PDI_expose("values", &values, PDI_INOUT); // output integers + + input = 1; + /// Import should also work + PDI_expose("input", &input, PDI_OUT); // update metadata => HDF5 now import only + PDI_expose("reals", &cp_reals, PDI_IN); // input real + PDI_expose("values", &cp_values, PDI_INOUT); // input integers + + /// So the data should be the same + fprintf(stderr, "Data exported | Data imported\n"); + + for (int j = njg; j < nj + njg; ++j) { // Should be the same inside + for (int i = nig; i < ni + nig; i++) { + if ((values[j][i] != cp_values[j][i]) || (reals[j][i] != cp_reals[j][i])) { + fprintf(stderr, "Ghost: integer (export) / integer(imported) :: %3d %3d\n", values[j][i], cp_values[j][i]); + fprintf(stderr, "Ghost: reals (export) / reals (imported) :: %6f %6f\n", reals[j][i], cp_reals[j][i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + for (int j = 0; j < njg; j++) { // and should be icst/rcst outside + for (int i = 0; i < nig; i++) { + if ((icst != cp_values[j][i]) || (rcst != cp_reals[j][i])) { + fprintf(stderr, "Ghost: integer (export) / integer(imported) :: %3d %3d\n", icst, cp_values[j][i]); + fprintf(stderr, "Ghost: reals (export) / reals (imported) :: %6f %6f\n", rcst, cp_reals[j][i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + for (int i = ni + nig; i < ni + 2 * nig; ++i) { + if ((icst != cp_values[j][i]) || (rcst != cp_reals[j][i])) { + fprintf(stderr, "Ghost: integer (export) / integer(imported) :: %3d %3d\n", icst, cp_values[j][i]); + fprintf(stderr, "Ghost: reals (export) / reals (imported) :: %6f %6f\n", rcst, cp_reals[j][i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + for (int j = nj + njg; j < nj + 2 * njg; ++j) { + for (int i = 0; i < nig; i++) { + if ((icst != cp_values[j][i]) || (rcst != cp_reals[j][i])) { + fprintf(stderr, "Ghost: integer (export) / integer(imported) :: %3d %3d\n", icst, cp_values[j][i]); + fprintf(stderr, "Ghost: reals (export) / reals (imported) :: %6f %6f\n", rcst, cp_reals[j][i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + for (int i = ni + nig; i < ni + 2 * nig; ++i) { + if ((icst != cp_values[j][i]) || (rcst != cp_reals[j][i])) { + fprintf(stderr, "Ghost: integer (export) / integer(imported) :: %3d %3d\n", icst, cp_values[j][i]); + fprintf(stderr, "Ghost: reals (export) / reals (imported) :: %6f %6f\n", rcst, cp_reals[j][i]); + MPI_Abort(MPI_COMM_WORLD, -1); + } + } + } + + PDI_finalize(); + PC_tree_destroy(&conf); + MPI_Finalize(); +}