From 0f92dd83048c3590ee69b76251740d23c3fd5c4a Mon Sep 17 00:00:00 2001 From: Ben Hourahine Date: Sat, 7 Nov 2020 00:58:04 +0000 Subject: [PATCH 01/11] Start of ctest for mpifx --- CMakeLists.txt | 1 + test/CMakeLists.txt | 27 ++++++++++++++++++++++--- test/test_allgather.f90 | 43 +++++++++++++++++++++++++++------------- test/test_allgatherv.f90 | 14 +++++++++++++ test/testhelper.f90 | 36 +++++++++++++++++++++++++++++++++ 5 files changed, 104 insertions(+), 17 deletions(-) create mode 100644 test/testhelper.f90 diff --git a/CMakeLists.txt b/CMakeLists.txt index a9581ff..ba190b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -27,6 +27,7 @@ include(GNUInstallDirs) add_subdirectory(lib) if(NOT BUILD_EXPORTED_TARGETS_ONLY) + enable_testing() add_subdirectory(test) endif() diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 33983d7..568429b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,6 +1,8 @@ -set(targets +set(tested test_allgather test_allgatherv + ) +set(targets test_allreduce test_bcast test_comm_split @@ -8,9 +10,28 @@ set(targets test_gatherv test_reduce test_scatter - test_scatterv) + test_scatterv + ) +list(APPEND targets ${tested}) foreach(target IN LISTS targets) - add_executable(${target} ${target}.f90) + add_executable(${target} ${target}.f90 testhelper.f90) target_link_libraries(${target} MpiFx) endforeach() + +foreach(target IN LISTS tested) + add_test(NAME ${target} + COMMAND ${MPIEXEC_EXECUTABLE} + ${MPIEXEC_NUMPROC_FLAG} + ${MPIEXEC_MAX_NUMPROCS} + ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/${target} + ${MPIEXEC_POSTFLAGS} + ) + set_tests_properties(${target} PROPERTIES + PASS_REGULAR_EXPRESSION "TestPASSED" + ) + set_tests_properties(${target} PROPERTIES + FAIL_REGULAR_EXPRESSION "TestFAILED" + ) +endforeach() diff --git a/test/test_allgather.f90 b/test/test_allgather.f90 index 7b58d3e..437734c 100644 --- a/test/test_allgather.f90 +++ b/test/test_allgather.f90 @@ -1,5 +1,7 @@ +!> Test various patterns of allgather program test_allgather use libmpifx_module + use testhelper implicit none type(mpifx_comm) :: mycomm @@ -8,6 +10,7 @@ program test_allgather integer, allocatable :: recv1(:), recv2(:,:) character(100) :: formstr character(*), parameter :: label = "(I2.2,'-',I3.3,'|',1X" + logical :: tPassed call mpifx_init() call mycomm%init() @@ -16,12 +19,16 @@ program test_allgather send0 = mycomm%rank * 2 allocate(recv1(1 * mycomm%size)) recv1(:) = 0 - write(*, label // ",A,1X,I0)") 1, mycomm%rank, & - & "Send0 buffer:", send0 + write(*, label // ",A,1X,I0)") 1, mycomm%rank, "Send0 buffer:", send0 call mpifx_allgather(mycomm, send0, recv1) write(formstr, "(A,I0,A)") "A,", size(recv1), "(1X,I0))" - write(*, label // formstr) 2, mycomm%rank, & - & "Recv1 buffer:", recv1(:) + write(*, label // formstr) 2, mycomm%rank, "Recv1 buffer:", recv1(:) + if (sum(recv1) /= mycomm%size * (mycomm%size-1)) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) deallocate(recv1) ! I1 -> I1 @@ -30,25 +37,33 @@ program test_allgather recv1(:) = 0 send1(:) = [ mycomm%rank, mycomm%rank + 1 ] write(formstr, "(A,I0,A)") "A,", size(send1), "(1X,I0))" - write(*, label // formstr) 3, mycomm%rank, & - & "Send1 buffer:", send1(:) + write(*, label // formstr) 3, mycomm%rank, "Send1 buffer:", send1(:) call mpifx_allgather(mycomm, send1, recv1) write(formstr, "(A,I0,A)") "A,", size(recv1), "(1X,I0))" - write(*, label // formstr) 4, mycomm%rank, & - & "Recv1 buffer:", recv1 + write(*, label // formstr) 4, mycomm%rank, "Recv1 buffer:", recv1 + if (sum(recv1) /= mycomm%size**2) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) ! I1 -> I2 allocate(recv2(size(send1), mycomm%size)) recv2(:,:) = 0 send1(:) = [ mycomm%rank, mycomm%rank + 1 ] write(formstr, "(A,I0,A)") "A,", size(send1), "(1X,I0))" - write(*, label // formstr) 5, mycomm%rank, & - & "Send1 buffer:", send1(:) + write(*, label // formstr) 5, mycomm%rank, "Send1 buffer:", send1(:) call mpifx_allgather(mycomm, send1, recv2) write(formstr, "(A,I0,A)") "A,", size(recv2), "(1X,I0))" - write(*, label // formstr) 6, mycomm%rank, & - & "Recv2 buffer:", recv2 - + write(*, label // formstr) 6, mycomm%rank, "Recv2 buffer:", recv2 + if (sum(recv1) /= mycomm%size**2) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) + call mpifx_finalize() - + end program test_allgather diff --git a/test/test_allgatherv.f90 b/test/test_allgatherv.f90 index e3c831d..bb8bf0d 100644 --- a/test/test_allgatherv.f90 +++ b/test/test_allgatherv.f90 @@ -1,5 +1,7 @@ +!> Test various patterns of allgatherv program test_allgatherv use libmpifx_module + use testhelper implicit none type(mpifx_comm) :: mycomm @@ -12,6 +14,8 @@ program test_allgatherv integer :: ii, nrecv, nCol character(100) :: formstr character(*), parameter :: label = "(I2.2,'-',I3.3,'|',1X" + logical :: tPassed + integer :: iCount call mpifx_init() call mycomm%init() @@ -33,6 +37,16 @@ program test_allgatherv if (mycomm%rank == mycomm%size - 1) then write(*, *) "Recv1 buffer:", recv1 end if + iCount = 0 + do ii = 1, mycomm%size + iCount = iCount + ii**2 + end do + if (nint(sum(recv1)) /= iCount) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) deallocate(recvcounts) deallocate(recv1) diff --git a/test/testhelper.f90 b/test/testhelper.f90 new file mode 100644 index 0000000..4e8878d --- /dev/null +++ b/test/testhelper.f90 @@ -0,0 +1,36 @@ +!> Helper routines for testers +module testhelper + use libmpifx_module, only : mpifx_comm, mpifx_barrier, mpifx_finalize + implicit none + + private + public :: testReturn + +contains + + !> Return expected labels for ctest + subroutine testReturn(mycomm, tPassed) + + type(mpifx_comm), intent(in) :: mycomm + + logical, intent(in) :: tPassed + + call mpifx_barrier(mycomm) + + if (tPassed) then + if (mycomm%rank == 0) then + ! label for ctest regex + write(*,*)'TestPASSED' + end if + else + if (mycomm%rank == 0) then + ! label for ctest regex + write(*,*)'TestFAILED' + end if + call mpifx_finalize() + stop + end if + + end subroutine testReturn + +end module testhelper From a014bfb0c2e80b2c50c0535b60d9d78c1aae6f90 Mon Sep 17 00:00:00 2001 From: Ben Hourahine Date: Sat, 7 Nov 2020 10:44:35 +0000 Subject: [PATCH 02/11] Fixed helper related dependencies in test --- test/CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 568429b..dff899d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,9 +14,15 @@ set(targets ) list(APPEND targets ${tested}) +set(sources-helper + testhelper.f90 + ) +add_library(mpifxtesthelp ${sources-helper}) +target_link_libraries(mpifxtesthelp PRIVATE MPI::MPI_Fortran MpiFx) + foreach(target IN LISTS targets) - add_executable(${target} ${target}.f90 testhelper.f90) - target_link_libraries(${target} MpiFx) + add_executable(${target} ${target}.f90) + target_link_libraries(${target} MpiFx mpifxtesthelp) endforeach() foreach(target IN LISTS tested) From f477589ed49b5c0a0e790bd20818ca26bb094d10 Mon Sep 17 00:00:00 2001 From: Ben Hourahine Date: Sat, 7 Nov 2020 13:38:46 +0000 Subject: [PATCH 03/11] Finish off allgatherv tests --- test/CMakeLists.txt | 2 ++ test/test_allgatherv.f90 | 23 +++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index dff899d..16f753d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -35,9 +35,11 @@ foreach(target IN LISTS tested) ${MPIEXEC_POSTFLAGS} ) set_tests_properties(${target} PROPERTIES + # test cases generate this on stdOut PASS_REGULAR_EXPRESSION "TestPASSED" ) set_tests_properties(${target} PROPERTIES + # test cases generate this on stdOut FAIL_REGULAR_EXPRESSION "TestFAILED" ) endforeach() diff --git a/test/test_allgatherv.f90 b/test/test_allgatherv.f90 index bb8bf0d..ae46762 100644 --- a/test/test_allgatherv.f90 +++ b/test/test_allgatherv.f90 @@ -37,15 +37,14 @@ program test_allgatherv if (mycomm%rank == mycomm%size - 1) then write(*, *) "Recv1 buffer:", recv1 end if - iCount = 0 - do ii = 1, mycomm%size - iCount = iCount + ii**2 - end do + ! test what has been gathered + iCount = (2*mycomm%size**3+3*mycomm%size**2+mycomm%size)/6 if (nint(sum(recv1)) /= iCount) then tPassed = .false. else tPassed = .true. end if + tPassed = tPassed .and. (abs(sum(recv1)-nint(sum(recv1))) < epsilon(1.0_sp)) call testReturn(mycomm, tPassed) deallocate(recvcounts) deallocate(recv1) @@ -73,6 +72,14 @@ program test_allgatherv write(*,*)recv2(:,ii) end do end if + iCount = 5*mycomm%size*(mycomm%size+1)*(2*mycomm%size+1)/6 + if (nint(sum(recv2)) /= iCount) then + tPassed = .false. + else + tPassed = .true. + end if + tPassed = tPassed .and. (abs(sum(recv2)-nint(sum(recv2))) < epsilon(1.0_sp)) + call testReturn(mycomm, tPassed) deallocate(recvcounts) @@ -95,6 +102,14 @@ program test_allgatherv if (mycomm%rank == mycomm%size - 1) then write(*, *) "Recv1 buffer:", recv1 end if + ! test what has been gathered + if (nint(sum(recv1)) /= (mycomm%size*(mycomm%size+1))/2) then + tPassed = .false. + else + tPassed = .true. + end if + tPassed = tPassed .and. (abs(sum(recv1)-nint(sum(recv1))) < epsilon(1.0_sp)) + call testReturn(mycomm, tPassed) call mpifx_finalize() From aa15bed6c0aff8edc2698efbcbb830967a3d6927 Mon Sep 17 00:00:00 2001 From: Tobias Melson Date: Mon, 21 Feb 2022 10:33:12 +0100 Subject: [PATCH 04/11] Implemented mpifx_comm_split_type Calling mpi_comm_split_type is especially necessary if you want to split the global communicator into groups that share a common memory. On a cluster, that would mean you get a new communicator for each node. --- lib/mpifx_comm.fpp | 51 +++++++++++++++++++++++++++++++++++ lib/mpifx_constants.fpp | 1 + test/CMakeLists.txt | 1 + test/meson.build | 1 + test/test_comm_split_type.f90 | 13 +++++++++ 5 files changed, 67 insertions(+) create mode 100644 test/test_comm_split_type.f90 diff --git a/lib/mpifx_comm.fpp b/lib/mpifx_comm.fpp index 79fbcf2..1d7b01a 100644 --- a/lib/mpifx_comm.fpp +++ b/lib/mpifx_comm.fpp @@ -21,6 +21,9 @@ module mpifx_comm_module !> Creates a new communicator by splitting the old one. procedure :: split => mpifx_comm_split + !> Creates a new communicator by splitting the old one given a split type. + procedure :: split_type => mpifx_comm_split_type + !> Frees the communicator. The communicator should not be used after this. procedure :: free => mpifx_comm_free @@ -111,6 +114,54 @@ contains end subroutine mpifx_comm_split + !> Creates a new communicator by splitting the old one applying a given split type. + !! + !! \param self Communicator instance. + !! \param splittype Determines which ranks to be grouped together. In MPI 3.0, + !! this can only be MPI_COMM_TYPE_SHARED grouping all MPI ranks together + !! that can share memory (usually on a node). + !! \param rankkey Is used to determine the rank of the process in its new + !! communicator. Processes calling the routine with a higher value will + !! have a higher rank in the new communicator. + !! \param newcomm New communicator for the given process. + !! \param error Optional error code on return. + !! + !! Example: + !! + !! program test_split_type + !! use libmpifx_module + !! implicit none + !! + !! type(mpifx_comm) :: allproc, splitproc + !! + !! call mpifx_init() + !! call allproc%init() + !! call allproc%split_type(MPI_COMM_TYPE_SHARED, allproc%rank, splitproc) + !! write(*, "(2(A,1X,I0,1X))") "ID:", allproc%rank, "SPLIT ID", splitproc%rank + !! call mpifx_finalize() + !! + !! end program test_split_type + !! + !! \see MPI documentation (\c MPI_COMM_SPLIT_TYPE) + !! + subroutine mpifx_comm_split_type(self, splittype, rankkey, newcomm, error) + class(mpifx_comm), intent(inout) :: self + integer, intent(in) :: splittype, rankkey + class(mpifx_comm), intent(out) :: newcomm + integer, intent(out), optional :: error + + integer :: error0, newcommid + + call mpi_comm_split_type(self%id, splittype, rankkey, MPI_INFO_NULL, newcommid, error0) + call handle_errorflag(error0, "mpi_comm_split_type() in mpifx_comm_split_type()", error) + if (error0 /= 0) then + return + end if + call newcomm%init(newcommid, error) + + end subroutine mpifx_comm_split_type + + !> Frees the MPI communicator. !> !> After this call, the passed communicator should not be used any more. diff --git a/lib/mpifx_constants.fpp b/lib/mpifx_constants.fpp index c133034..2a0a22f 100644 --- a/lib/mpifx_constants.fpp +++ b/lib/mpifx_constants.fpp @@ -8,6 +8,7 @@ module mpifx_constants_module public :: MPI_LAND, MPI_BAND, MPI_LOR, MPI_BOR, MPI_LXOR ,MPI_BXOR public :: MPI_MAXLOC, MPI_MINLOC public :: MPI_THREAD_SINGLE, MPI_THREAD_FUNNELED, MPI_THREAD_SERIALIZED, MPI_THREAD_MULTIPLE + public :: MPI_COMM_TYPE_SHARED public :: MPIFX_UNHANDLED_ERROR, MPIFX_ASSERT_FAILED diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 33983d7..29760dd 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -4,6 +4,7 @@ set(targets test_allreduce test_bcast test_comm_split + test_comm_split_type test_gather test_gatherv test_reduce diff --git a/test/meson.build b/test/meson.build index eb813e3..cbfc239 100644 --- a/test/meson.build +++ b/test/meson.build @@ -6,6 +6,7 @@ tests = [ 'allreduce', 'bcast', 'comm_split', + 'comm_split_type', 'gather', 'gatherv', 'reduce', diff --git a/test/test_comm_split_type.f90 b/test/test_comm_split_type.f90 new file mode 100644 index 0000000..c40491c --- /dev/null +++ b/test/test_comm_split_type.f90 @@ -0,0 +1,13 @@ +program test_split_type + use libmpifx_module + implicit none + + type(mpifx_comm) :: allproc, splitproc + + call mpifx_init() + call allproc%init() + call allproc%split_type(MPI_COMM_TYPE_SHARED, allproc%rank, splitproc) + write(*, "(2(A,1X,I0,1X))") "ID:", allproc%rank, "SPLIT ID", splitproc%rank + call mpifx_finalize() + +end program test_split_type From e6be7389c9da0e1eb0126faf06044eaee5a11cc9 Mon Sep 17 00:00:00 2001 From: Tobias Melson Date: Mon, 21 Feb 2022 16:16:07 +0100 Subject: [PATCH 05/11] Introduced routines for MPI shared memory MPI shared memory is a concept for allocating memory accessibly by ranks that are connected to the same physical memory, i.e. on the same node in a cluster. In this basic implementation here, we can use this concept to reduce the memory usage of an application if large arrays are stored only once per node and not once per MPI task. See test/test_shared_memory.f90 for a short example. --- lib/CMakeLists.txt | 3 +- lib/module.fpp | 1 + lib/mpifx_shared_memory.fpp | 150 ++++++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 3 +- test/meson.build | 1 + test/test_shared_memory.f90 | 34 ++++++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 lib/mpifx_shared_memory.fpp create mode 100644 test/test_shared_memory.f90 diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index b5250a0..43c83aa 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -19,7 +19,8 @@ set(sources-fpp mpifx_reduce.fpp mpifx_scatter.fpp mpifx_scatterv.fpp - mpifx_send.fpp) + mpifx_send.fpp + mpifx_shared_memory.fpp) fypp_preprocess("${sources-fpp}" sources-f90) diff --git a/lib/module.fpp b/lib/module.fpp index a121129..7e4cec4 100644 --- a/lib/module.fpp +++ b/lib/module.fpp @@ -30,6 +30,7 @@ module libmpifx_module use mpifx_allgatherv_module use mpifx_scatter_module use mpifx_scatterv_module + use mpifx_shared_memory_module implicit none public diff --git a/lib/mpifx_shared_memory.fpp b/lib/mpifx_shared_memory.fpp new file mode 100644 index 0000000..2948669 --- /dev/null +++ b/lib/mpifx_shared_memory.fpp @@ -0,0 +1,150 @@ +#:include 'mpifx.fypp' +#:set TYPES = NUMERIC_TYPES + +!> Contains utilities for handling MPI shared memory +module mpifx_shared_memory_module + use mpifx_common_module + use iso_c_binding, only : c_ptr, c_f_pointer + implicit none + private + + public :: mpifx_allocate_shared, mpifx_free_shared, mpifx_lock_shared, mpifx_unlock_shared, mpifx_sync_shared + + interface mpifx_allocate_shared +#:for TYPE in TYPES + #:set TYPEABBREV = TYPE_ABBREVS[TYPE] + module procedure mpifx_allocate_shared_${TYPEABBREV}$ +#:endfor + end interface mpifx_allocate_shared + +contains + +#:def mpifx_allocate_shared_template(SUFFIX, TYPE) + + !> Returns a window handle and a pointer to the address associated with a shared memory segment. + !! + !! \param mycomm MPI communicator. + !! \param length Number of elements of type ${TYPE}$ in the shared memory window. + !! \param win Handle of the shared memory window on return. + !! \param shared_data Pointer to the shared data array of length 'length' on return. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_ALLOCATE_SHARED) + !! + subroutine mpifx_allocate_shared_${SUFFIX}$(mycomm, length, win, shared_data, error) + type(mpifx_comm), intent(in) :: mycomm + integer, intent(in) :: length + integer, intent(out) :: win + ${TYPE}$, pointer, intent(out) :: shared_data(:) + integer, intent(out), optional :: error + + integer :: disp_unit, error0, error1 + integer(MPI_ADDRESS_KIND) :: local_length + type(c_ptr) :: baseptr + + disp_unit = kind(shared_data) + + local_length = 0 + if (mycomm%lead) then + local_length = length * disp_unit + end if + + call mpi_win_allocate_shared(local_length, disp_unit, MPI_INFO_NULL, mycomm%id, baseptr, win, error0) + call handle_errorflag(error0, "MPI_WIN_ALLOCATE_SHARED in mpifx_allocate_shared_${SUFFIX}$", error) + + call mpi_win_shared_query(win, 0, local_length, disp_unit, baseptr, error1) + call handle_errorflag(error1, "MPI_WIN_SHARED_QUERY in mpifx_allocate_shared_${SUFFIX}$", error) + + call c_f_pointer(baseptr, shared_data, [length]) + + end subroutine mpifx_allocate_shared_${SUFFIX}$ + +#:enddef mpifx_allocate_shared_template + + !> Deallocates a memory associated with a shared memory segment. + !! + !! \param win Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_FREE) + !! + subroutine mpifx_free_shared(win, error) + integer, intent(inout) :: win + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_free(win, error0) + call handle_errorflag(error0, "MPI_WIN_FREE in mpifx_free_shared", error) + + end subroutine mpifx_free_shared + + !> Locks a shared memory segment. + !! + !! \param win Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_LOCK_ALL) + !! + subroutine mpifx_lock_shared(win, error) + integer, intent(inout) :: win + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_lock_all(MPI_MODE_NOCHECK, win, error0) + call handle_errorflag(error0, "MPI_WIN_LOCK_ALL in mpifx_lock_shared", error) + + end subroutine mpifx_lock_shared + + !> Unlocks a shared memory segment. + !! + !! \param win Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_UNLOCK_ALL) + !! + subroutine mpifx_unlock_shared(win, error) + integer, intent(inout) :: win + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_unlock_all(win, error0) + call handle_errorflag(error0, "MPI_WIN_UNLOCK_ALL in mpifx_unlock_shared", error) + + end subroutine mpifx_unlock_shared + + !> Synchronizes shared memory across MPI ranks. + !! + !! \param mycomm MPI communicator. + !! \param win Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_SYNC) + !! + subroutine mpifx_sync_shared(mycomm, win, error) + type(mpifx_comm), intent(in) :: mycomm + integer, intent(inout) :: win + integer, intent(out), optional :: error + + integer :: error0, error1 + + call mpi_win_sync(win, error0) + call handle_errorflag(error0, "MPI_WIN_SYNC in mpifx_sync_shared", error) + + call mpi_barrier(mycomm%id, error1) + call handle_errorflag(error1, "MPI_BARRIER in mpifx_sync_shared", error) + + end subroutine mpifx_sync_shared + + +#:for TYPE in TYPES + #:set FTYPE = FORTRAN_TYPES[TYPE] + #:set SUFFIX = TYPE_ABBREVS[TYPE] + + $:mpifx_allocate_shared_template(SUFFIX, FTYPE) + +#:endfor + +end module mpifx_shared_memory_module diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 29760dd..0be19fc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,7 +9,8 @@ set(targets test_gatherv test_reduce test_scatter - test_scatterv) + test_scatterv + test_shared_memory) foreach(target IN LISTS targets) add_executable(${target} ${target}.f90) diff --git a/test/meson.build b/test/meson.build index cbfc239..04fc3bb 100644 --- a/test/meson.build +++ b/test/meson.build @@ -12,6 +12,7 @@ tests = [ 'reduce', 'scatter', 'scatterv', + 'shared_memory', ] foreach t : tests diff --git a/test/test_shared_memory.f90 b/test/test_shared_memory.f90 new file mode 100644 index 0000000..ebce3cc --- /dev/null +++ b/test/test_shared_memory.f90 @@ -0,0 +1,34 @@ +program test_shared_memory + use libmpifx_module + implicit none + + type(mpifx_comm) :: globalcomm, nodecomm + integer, parameter :: length = 7 + integer :: win + integer, pointer :: data_pointer(:) + + call mpifx_init() + call globalcomm%init() + + ! Create a new communicator for all ranks on a node first + call globalcomm%split_type(MPI_COMM_TYPE_SHARED, globalcomm%rank, nodecomm) + + call mpifx_allocate_shared(nodecomm, length, win, data_pointer) + + call mpifx_lock_shared(win) + + ! Only rank 0 writes data into the array + if (nodecomm%lead) then + data_pointer(:) = 42 + end if + + call mpifx_sync_shared(nodecomm, win) + call mpifx_unlock_shared(win) + + ! All ranks on the node will read the same value + write(*, "(2(A,1X,I0,1X))") "ID:", nodecomm%rank, "VALUE:", data_pointer(1) + + call mpifx_free_shared(win) + call mpifx_finalize() + +end program test_shared_memory From d104cd863338f753824f8c2e774c4aac58271625 Mon Sep 17 00:00:00 2001 From: Tobias Melson Date: Wed, 23 Feb 2022 14:09:54 +0100 Subject: [PATCH 06/11] Created win type with shared memory functionality The new type mpifx_win can now be used to handle MPI shared memory conveniently. --- lib/CMakeLists.txt | 2 +- lib/meson.build | 1 + lib/module.fpp | 2 +- lib/mpifx_shared_memory.fpp | 150 --------------- lib/mpifx_win.fpp | 172 ++++++++++++++++++ test/CMakeLists.txt | 2 +- test/meson.build | 2 +- ...red_memory.f90 => test_win_shared_mem.f90} | 16 +- 8 files changed, 185 insertions(+), 162 deletions(-) delete mode 100644 lib/mpifx_shared_memory.fpp create mode 100644 lib/mpifx_win.fpp rename test/{test_shared_memory.f90 => test_win_shared_mem.f90} (68%) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 43c83aa..0cdf659 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -20,7 +20,7 @@ set(sources-fpp mpifx_scatter.fpp mpifx_scatterv.fpp mpifx_send.fpp - mpifx_shared_memory.fpp) + mpifx_win.fpp) fypp_preprocess("${sources-fpp}" sources-f90) diff --git a/lib/meson.build b/lib/meson.build index db50325..c590372 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -22,6 +22,7 @@ sources_fpp = files( 'mpifx_scatter.fpp', 'mpifx_scatterv.fpp', 'mpifx_send.fpp', + 'mpifx_win.fpp', ) sources_f90 = [] foreach src : sources_fpp diff --git a/lib/module.fpp b/lib/module.fpp index 7e4cec4..a605e24 100644 --- a/lib/module.fpp +++ b/lib/module.fpp @@ -30,7 +30,7 @@ module libmpifx_module use mpifx_allgatherv_module use mpifx_scatter_module use mpifx_scatterv_module - use mpifx_shared_memory_module + use mpifx_win_module implicit none public diff --git a/lib/mpifx_shared_memory.fpp b/lib/mpifx_shared_memory.fpp deleted file mode 100644 index 2948669..0000000 --- a/lib/mpifx_shared_memory.fpp +++ /dev/null @@ -1,150 +0,0 @@ -#:include 'mpifx.fypp' -#:set TYPES = NUMERIC_TYPES - -!> Contains utilities for handling MPI shared memory -module mpifx_shared_memory_module - use mpifx_common_module - use iso_c_binding, only : c_ptr, c_f_pointer - implicit none - private - - public :: mpifx_allocate_shared, mpifx_free_shared, mpifx_lock_shared, mpifx_unlock_shared, mpifx_sync_shared - - interface mpifx_allocate_shared -#:for TYPE in TYPES - #:set TYPEABBREV = TYPE_ABBREVS[TYPE] - module procedure mpifx_allocate_shared_${TYPEABBREV}$ -#:endfor - end interface mpifx_allocate_shared - -contains - -#:def mpifx_allocate_shared_template(SUFFIX, TYPE) - - !> Returns a window handle and a pointer to the address associated with a shared memory segment. - !! - !! \param mycomm MPI communicator. - !! \param length Number of elements of type ${TYPE}$ in the shared memory window. - !! \param win Handle of the shared memory window on return. - !! \param shared_data Pointer to the shared data array of length 'length' on return. - !! \param error Optional error code on return. - !! - !! \see MPI documentation (\c MPI_WIN_ALLOCATE_SHARED) - !! - subroutine mpifx_allocate_shared_${SUFFIX}$(mycomm, length, win, shared_data, error) - type(mpifx_comm), intent(in) :: mycomm - integer, intent(in) :: length - integer, intent(out) :: win - ${TYPE}$, pointer, intent(out) :: shared_data(:) - integer, intent(out), optional :: error - - integer :: disp_unit, error0, error1 - integer(MPI_ADDRESS_KIND) :: local_length - type(c_ptr) :: baseptr - - disp_unit = kind(shared_data) - - local_length = 0 - if (mycomm%lead) then - local_length = length * disp_unit - end if - - call mpi_win_allocate_shared(local_length, disp_unit, MPI_INFO_NULL, mycomm%id, baseptr, win, error0) - call handle_errorflag(error0, "MPI_WIN_ALLOCATE_SHARED in mpifx_allocate_shared_${SUFFIX}$", error) - - call mpi_win_shared_query(win, 0, local_length, disp_unit, baseptr, error1) - call handle_errorflag(error1, "MPI_WIN_SHARED_QUERY in mpifx_allocate_shared_${SUFFIX}$", error) - - call c_f_pointer(baseptr, shared_data, [length]) - - end subroutine mpifx_allocate_shared_${SUFFIX}$ - -#:enddef mpifx_allocate_shared_template - - !> Deallocates a memory associated with a shared memory segment. - !! - !! \param win Handle of the shared memory window. - !! \param error Optional error code on return. - !! - !! \see MPI documentation (\c MPI_WIN_FREE) - !! - subroutine mpifx_free_shared(win, error) - integer, intent(inout) :: win - integer, intent(out), optional :: error - - integer :: error0 - - call mpi_win_free(win, error0) - call handle_errorflag(error0, "MPI_WIN_FREE in mpifx_free_shared", error) - - end subroutine mpifx_free_shared - - !> Locks a shared memory segment. - !! - !! \param win Handle of the shared memory window. - !! \param error Optional error code on return. - !! - !! \see MPI documentation (\c MPI_WIN_LOCK_ALL) - !! - subroutine mpifx_lock_shared(win, error) - integer, intent(inout) :: win - integer, intent(out), optional :: error - - integer :: error0 - - call mpi_win_lock_all(MPI_MODE_NOCHECK, win, error0) - call handle_errorflag(error0, "MPI_WIN_LOCK_ALL in mpifx_lock_shared", error) - - end subroutine mpifx_lock_shared - - !> Unlocks a shared memory segment. - !! - !! \param win Handle of the shared memory window. - !! \param error Optional error code on return. - !! - !! \see MPI documentation (\c MPI_WIN_UNLOCK_ALL) - !! - subroutine mpifx_unlock_shared(win, error) - integer, intent(inout) :: win - integer, intent(out), optional :: error - - integer :: error0 - - call mpi_win_unlock_all(win, error0) - call handle_errorflag(error0, "MPI_WIN_UNLOCK_ALL in mpifx_unlock_shared", error) - - end subroutine mpifx_unlock_shared - - !> Synchronizes shared memory across MPI ranks. - !! - !! \param mycomm MPI communicator. - !! \param win Handle of the shared memory window. - !! \param error Optional error code on return. - !! - !! \see MPI documentation (\c MPI_WIN_SYNC) - !! - subroutine mpifx_sync_shared(mycomm, win, error) - type(mpifx_comm), intent(in) :: mycomm - integer, intent(inout) :: win - integer, intent(out), optional :: error - - integer :: error0, error1 - - call mpi_win_sync(win, error0) - call handle_errorflag(error0, "MPI_WIN_SYNC in mpifx_sync_shared", error) - - call mpi_barrier(mycomm%id, error1) - call handle_errorflag(error1, "MPI_BARRIER in mpifx_sync_shared", error) - - end subroutine mpifx_sync_shared - - -#:for TYPE in TYPES - #:set FTYPE = FORTRAN_TYPES[TYPE] - #:set SUFFIX = TYPE_ABBREVS[TYPE] - - $:mpifx_allocate_shared_template(SUFFIX, FTYPE) - -#:endfor - -end module mpifx_shared_memory_module diff --git a/lib/mpifx_win.fpp b/lib/mpifx_win.fpp new file mode 100644 index 0000000..2d9ece3 --- /dev/null +++ b/lib/mpifx_win.fpp @@ -0,0 +1,172 @@ +#:include 'mpifx.fypp' +#:set TYPES = NUMERIC_TYPES + +!> Contains routined for MPI shared memory. +module mpifx_win_module + use mpifx_common_module + use iso_c_binding, only : c_ptr, c_f_pointer + implicit none + private + + public :: mpifx_win + + !> MPI shared memory window with some additional information. + type mpifx_win + integer :: id !< Window id. + integer :: comm_id !< Communicator id. + contains + !> Initializes an MPI shared memory window. + generic :: allocate_shared => & +#:for TYPE in TYPES[:-1] + & mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$,& +#:endfor + & mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPES[-1]]}$ + +#:for TYPE in TYPES + procedure, private :: mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$ +#:endfor + + !> Locks a shared memory segment. + procedure :: lock => mpifx_win_lock + + !> Unlocks a shared memory segment. + procedure :: unlock => mpifx_win_unlock + + !> Synchronizes shared memory across MPI ranks. + procedure :: sync => mpifx_win_sync + + !> Deallocates memory associated with a shared memory segment. + procedure :: free => mpifx_win_free + + end type mpifx_win + +contains + +#:def mpifx_win_allocate_shared_template(SUFFIX, TYPE) + + !> Initialized a window handle and returns a pointer to the address associated with a shared memory segment. + !! + !! \param self Handle of the shared memory window on return. + !! \param mycomm MPI communicator. + !! \param length Number of elements of type ${TYPE}$ in the shared memory window. + !! \param shared_data Pointer to the shared data array of length 'length' on return. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_ALLOCATE_SHARED) + !! + subroutine mpifx_win_allocate_shared_${SUFFIX}$(self, mycomm, length, shared_data, error) + class(mpifx_win), intent(out) :: self + class(mpifx_comm), intent(in) :: mycomm + integer, intent(in) :: length + ${TYPE}$, pointer, intent(out) :: shared_data(:) + integer, intent(out), optional :: error + + integer :: disp_unit, error0, error1 + integer(MPI_ADDRESS_KIND) :: local_length + type(c_ptr) :: baseptr + + disp_unit = kind(shared_data) + + local_length = 0 + if (mycomm%lead) then + local_length = length * disp_unit + end if + + call mpi_win_allocate_shared(local_length, disp_unit, MPI_INFO_NULL, mycomm%id, baseptr, self%id, error0) + call handle_errorflag(error0, "MPI_WIN_ALLOCATE_SHARED in mpifx_win_allocate_shared_${SUFFIX}$", error) + + call mpi_win_shared_query(self%id, 0, local_length, disp_unit, baseptr, error1) + call handle_errorflag(error1, "MPI_WIN_SHARED_QUERY in mpifx_win_allocate_shared_${SUFFIX}$", error) + + self%comm_id = mycomm%id + call c_f_pointer(baseptr, shared_data, [length]) + + end subroutine mpifx_win_allocate_shared_${SUFFIX}$ + +#:enddef mpifx_win_allocate_shared_template + + !> Locks a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_LOCK_ALL) + !! + subroutine mpifx_win_lock(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_lock_all(MPI_MODE_NOCHECK, self%id, error0) + call handle_errorflag(error0, "MPI_WIN_LOCK_ALL in mpifx_win_lock", error) + + end subroutine mpifx_win_lock + + !> Unlocks a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_UNLOCK_ALL) + !! + subroutine mpifx_win_unlock(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_unlock_all(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_UNLOCK_ALL in mpifx_win_unlock", error) + + end subroutine mpifx_win_unlock + + !> Synchronizes shared memory across MPI ranks. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_SYNC) + !! + subroutine mpifx_win_sync(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0, error1 + + call mpi_win_sync(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_SYNC in mpifx_win_sync", error) + + call mpi_barrier(self%comm_id, error1) + call handle_errorflag(error1, "MPI_BARRIER in mpifx_win_sync", error) + + end subroutine mpifx_win_sync + + !> Deallocates memory associated with a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_FREE) + !! + subroutine mpifx_win_free(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_free(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_FREE in mpifx_win_free", error) + + end subroutine mpifx_win_free + + +#:for TYPE in TYPES + #:set FTYPE = FORTRAN_TYPES[TYPE] + #:set SUFFIX = TYPE_ABBREVS[TYPE] + + $:mpifx_win_allocate_shared_template(SUFFIX, FTYPE) + +#:endfor + +end module mpifx_win_module diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0be19fc..0b6785c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -10,7 +10,7 @@ set(targets test_reduce test_scatter test_scatterv - test_shared_memory) + test_win_shared_mem) foreach(target IN LISTS targets) add_executable(${target} ${target}.f90) diff --git a/test/meson.build b/test/meson.build index 04fc3bb..fd98ff5 100644 --- a/test/meson.build +++ b/test/meson.build @@ -12,7 +12,7 @@ tests = [ 'reduce', 'scatter', 'scatterv', - 'shared_memory', + 'win_shared_mem', ] foreach t : tests diff --git a/test/test_shared_memory.f90 b/test/test_win_shared_mem.f90 similarity index 68% rename from test/test_shared_memory.f90 rename to test/test_win_shared_mem.f90 index ebce3cc..048fda8 100644 --- a/test/test_shared_memory.f90 +++ b/test/test_win_shared_mem.f90 @@ -1,10 +1,10 @@ -program test_shared_memory +program test_win_shared_mem use libmpifx_module implicit none type(mpifx_comm) :: globalcomm, nodecomm + type(mpifx_win) :: win integer, parameter :: length = 7 - integer :: win integer, pointer :: data_pointer(:) call mpifx_init() @@ -13,22 +13,22 @@ program test_shared_memory ! Create a new communicator for all ranks on a node first call globalcomm%split_type(MPI_COMM_TYPE_SHARED, globalcomm%rank, nodecomm) - call mpifx_allocate_shared(nodecomm, length, win, data_pointer) + call win%allocate_shared(nodecomm, length, data_pointer) - call mpifx_lock_shared(win) + call win%lock() ! Only rank 0 writes data into the array if (nodecomm%lead) then data_pointer(:) = 42 end if - call mpifx_sync_shared(nodecomm, win) - call mpifx_unlock_shared(win) + call win%sync() + call win%unlock() ! All ranks on the node will read the same value write(*, "(2(A,1X,I0,1X))") "ID:", nodecomm%rank, "VALUE:", data_pointer(1) - call mpifx_free_shared(win) + call win%free() call mpifx_finalize() -end program test_shared_memory +end program test_win_shared_mem From f9e03274f5a335cfc8e90209e3ccf4a8077924b0 Mon Sep 17 00:00:00 2001 From: Tobias Melson Date: Wed, 23 Feb 2022 14:12:34 +0100 Subject: [PATCH 07/11] Use storage_size to determine the required bytes kind() does not always return the number of bytes for the data type. Instead, storage_size() does what we want here. --- lib/mpifx_win.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mpifx_win.fpp b/lib/mpifx_win.fpp index 2d9ece3..1bd2d63 100644 --- a/lib/mpifx_win.fpp +++ b/lib/mpifx_win.fpp @@ -65,7 +65,7 @@ contains integer(MPI_ADDRESS_KIND) :: local_length type(c_ptr) :: baseptr - disp_unit = kind(shared_data) + disp_unit = storage_size(shared_data) / 8 local_length = 0 if (mycomm%lead) then From 3dce2ea873a2568d8d63ce5c3029c28012dfbc50 Mon Sep 17 00:00:00 2001 From: Tobias Melson Date: Fri, 25 Feb 2022 14:39:16 +0100 Subject: [PATCH 08/11] Avoid integer overflow when calculating length It may happen that the product of the two 4-byte integers overflows before assigning to the 8-byte integer. We need to explicitly convert to 8 bytes first before multiplying. --- lib/mpifx_win.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mpifx_win.fpp b/lib/mpifx_win.fpp index 1bd2d63..f54efd2 100644 --- a/lib/mpifx_win.fpp +++ b/lib/mpifx_win.fpp @@ -69,7 +69,7 @@ contains local_length = 0 if (mycomm%lead) then - local_length = length * disp_unit + local_length = int(length, kind=MPI_ADDRESS_KIND) * disp_unit end if call mpi_win_allocate_shared(local_length, disp_unit, MPI_INFO_NULL, mycomm%id, baseptr, self%id, error0) From 6d96ce0b3b75c181c94d10b35770b74e587836fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Mon, 28 Feb 2022 11:55:09 +0100 Subject: [PATCH 09/11] Make comm id private, simplify fypp-template --- lib/mpifx_win.fpp | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/mpifx_win.fpp b/lib/mpifx_win.fpp index f54efd2..1d96892 100644 --- a/lib/mpifx_win.fpp +++ b/lib/mpifx_win.fpp @@ -12,15 +12,14 @@ module mpifx_win_module !> MPI shared memory window with some additional information. type mpifx_win - integer :: id !< Window id. + private + integer, public :: id !< Window id. integer :: comm_id !< Communicator id. contains !> Initializes an MPI shared memory window. - generic :: allocate_shared => & -#:for TYPE in TYPES[:-1] - & mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$,& +#:for TYPE in TYPES + generic :: allocate_shared => mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$ #:endfor - & mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPES[-1]]}$ #:for TYPE in TYPES procedure, private :: mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$ From 5446b049cea2535ce1155c49e35c4680efef1530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Mon, 2 May 2022 17:13:17 +0200 Subject: [PATCH 10/11] Bump version number, fix version bumper --- CHANGELOG.rst | 13 +++++++++++++ VERSION | 2 +- doc/doxygen/Doxyfile | 2 +- doc/sphinx/conf.py | 4 ++-- utils/srcmanip/set_version | 4 ++++ 5 files changed, 21 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cac490b..7913ac3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,19 @@ Change Log Notable project changes in various releases. +1.3 +=== + +Added +------ + +* Grid splitting based on type (e.g. MPI_COMM_TYPE_SHARED) + +* Wrappers for accessing MPI shared memory window + +* Some tests accessible via ctest + + 1.2 === diff --git a/VERSION b/VERSION index 26aaba0..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/doc/doxygen/Doxyfile b/doc/doxygen/Doxyfile index 0439a1e..bd83227 100644 --- a/doc/doxygen/Doxyfile +++ b/doc/doxygen/Doxyfile @@ -32,7 +32,7 @@ PROJECT_NAME = "MpiFx" # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = "1.2.0" +PROJECT_NUMBER = "1.3.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 4ec24c2..9df2efe 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -45,10 +45,10 @@ # built documents. # # The short X.Y version. -version = '1.2' +version = '1.3' # The full version, including alpha/beta/rc tags. -release = '1.2.0' +release = '1.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/utils/srcmanip/set_version b/utils/srcmanip/set_version index 00be0d0..b5247c6 100755 --- a/utils/srcmanip/set_version +++ b/utils/srcmanip/set_version @@ -15,6 +15,10 @@ _VERSION_PATTERN = r'\d+\.\d+(?:\.\d+)?(?:-\w+)?' _FILES_AND_PATTERNS = [ # + ('VERSION', + r'{}'.format(_VERSION_PATTERN), + r'{version}'), + # ('doc/doxygen/Doxyfile', r'^PROJECT_NUMBER\s*=\s*([\'"]){}\1\s*$'.format(_VERSION_PATTERN), 'PROJECT_NUMBER = "{version}"\n'), From 80b13e5f5e9d0f5ce642aaa57f8235552928c71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A1lint=20Aradi?= Date: Fri, 6 May 2022 20:39:38 +0200 Subject: [PATCH 11/11] Minor renaming in test/test_allgather.f90 Co-authored-by: Ben Hourahine --- test/test_allgather.f90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_allgather.f90 b/test/test_allgather.f90 index 437734c..42933b0 100644 --- a/test/test_allgather.f90 +++ b/test/test_allgather.f90 @@ -10,7 +10,7 @@ program test_allgather integer, allocatable :: recv1(:), recv2(:,:) character(100) :: formstr character(*), parameter :: label = "(I2.2,'-',I3.3,'|',1X" - logical :: tPassed + logical :: isPassed call mpifx_init() call mycomm%init()