diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cac490b..7913ac3 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,6 +5,19 @@ Change Log Notable project changes in various releases. +1.3 +=== + +Added +------ + +* Grid splitting based on type (e.g. MPI_COMM_TYPE_SHARED) + +* Wrappers for accessing MPI shared memory window + +* Some tests accessible via ctest + + 1.2 === diff --git a/CMakeLists.txt b/CMakeLists.txt index 13c4c3b..a7cffc6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,7 @@ include(GNUInstallDirs) add_subdirectory(lib) if(NOT BUILD_EXPORTED_TARGETS_ONLY) + enable_testing() add_subdirectory(test) endif() diff --git a/VERSION b/VERSION index 26aaba0..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.0 +1.3.0 diff --git a/doc/doxygen/Doxyfile b/doc/doxygen/Doxyfile index 0439a1e..bd83227 100644 --- a/doc/doxygen/Doxyfile +++ b/doc/doxygen/Doxyfile @@ -32,7 +32,7 @@ PROJECT_NAME = "MpiFx" # This could be handy for archiving the generated documentation or # if some version control system is used. -PROJECT_NUMBER = "1.2.0" +PROJECT_NUMBER = "1.3.0" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer diff --git a/doc/sphinx/conf.py b/doc/sphinx/conf.py index 4ec24c2..9df2efe 100644 --- a/doc/sphinx/conf.py +++ b/doc/sphinx/conf.py @@ -45,10 +45,10 @@ # built documents. # # The short X.Y version. -version = '1.2' +version = '1.3' # The full version, including alpha/beta/rc tags. -release = '1.2.0' +release = '1.3.0' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index b5250a0..0cdf659 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -19,7 +19,8 @@ set(sources-fpp mpifx_reduce.fpp mpifx_scatter.fpp mpifx_scatterv.fpp - mpifx_send.fpp) + mpifx_send.fpp + mpifx_win.fpp) fypp_preprocess("${sources-fpp}" sources-f90) diff --git a/lib/meson.build b/lib/meson.build index db50325..c590372 100644 --- a/lib/meson.build +++ b/lib/meson.build @@ -22,6 +22,7 @@ sources_fpp = files( 'mpifx_scatter.fpp', 'mpifx_scatterv.fpp', 'mpifx_send.fpp', + 'mpifx_win.fpp', ) sources_f90 = [] foreach src : sources_fpp diff --git a/lib/module.fpp b/lib/module.fpp index a121129..a605e24 100644 --- a/lib/module.fpp +++ b/lib/module.fpp @@ -30,6 +30,7 @@ module libmpifx_module use mpifx_allgatherv_module use mpifx_scatter_module use mpifx_scatterv_module + use mpifx_win_module implicit none public diff --git a/lib/mpifx_comm.fpp b/lib/mpifx_comm.fpp index 79fbcf2..1d7b01a 100644 --- a/lib/mpifx_comm.fpp +++ b/lib/mpifx_comm.fpp @@ -21,6 +21,9 @@ module mpifx_comm_module !> Creates a new communicator by splitting the old one. procedure :: split => mpifx_comm_split + !> Creates a new communicator by splitting the old one given a split type. + procedure :: split_type => mpifx_comm_split_type + !> Frees the communicator. The communicator should not be used after this. procedure :: free => mpifx_comm_free @@ -111,6 +114,54 @@ contains end subroutine mpifx_comm_split + !> Creates a new communicator by splitting the old one applying a given split type. + !! + !! \param self Communicator instance. + !! \param splittype Determines which ranks to be grouped together. In MPI 3.0, + !! this can only be MPI_COMM_TYPE_SHARED grouping all MPI ranks together + !! that can share memory (usually on a node). + !! \param rankkey Is used to determine the rank of the process in its new + !! communicator. Processes calling the routine with a higher value will + !! have a higher rank in the new communicator. + !! \param newcomm New communicator for the given process. + !! \param error Optional error code on return. + !! + !! Example: + !! + !! program test_split_type + !! use libmpifx_module + !! implicit none + !! + !! type(mpifx_comm) :: allproc, splitproc + !! + !! call mpifx_init() + !! call allproc%init() + !! call allproc%split_type(MPI_COMM_TYPE_SHARED, allproc%rank, splitproc) + !! write(*, "(2(A,1X,I0,1X))") "ID:", allproc%rank, "SPLIT ID", splitproc%rank + !! call mpifx_finalize() + !! + !! end program test_split_type + !! + !! \see MPI documentation (\c MPI_COMM_SPLIT_TYPE) + !! + subroutine mpifx_comm_split_type(self, splittype, rankkey, newcomm, error) + class(mpifx_comm), intent(inout) :: self + integer, intent(in) :: splittype, rankkey + class(mpifx_comm), intent(out) :: newcomm + integer, intent(out), optional :: error + + integer :: error0, newcommid + + call mpi_comm_split_type(self%id, splittype, rankkey, MPI_INFO_NULL, newcommid, error0) + call handle_errorflag(error0, "mpi_comm_split_type() in mpifx_comm_split_type()", error) + if (error0 /= 0) then + return + end if + call newcomm%init(newcommid, error) + + end subroutine mpifx_comm_split_type + + !> Frees the MPI communicator. !> !> After this call, the passed communicator should not be used any more. diff --git a/lib/mpifx_constants.fpp b/lib/mpifx_constants.fpp index c133034..2a0a22f 100644 --- a/lib/mpifx_constants.fpp +++ b/lib/mpifx_constants.fpp @@ -8,6 +8,7 @@ module mpifx_constants_module public :: MPI_LAND, MPI_BAND, MPI_LOR, MPI_BOR, MPI_LXOR ,MPI_BXOR public :: MPI_MAXLOC, MPI_MINLOC public :: MPI_THREAD_SINGLE, MPI_THREAD_FUNNELED, MPI_THREAD_SERIALIZED, MPI_THREAD_MULTIPLE + public :: MPI_COMM_TYPE_SHARED public :: MPIFX_UNHANDLED_ERROR, MPIFX_ASSERT_FAILED diff --git a/lib/mpifx_win.fpp b/lib/mpifx_win.fpp new file mode 100644 index 0000000..1d96892 --- /dev/null +++ b/lib/mpifx_win.fpp @@ -0,0 +1,171 @@ +#:include 'mpifx.fypp' +#:set TYPES = NUMERIC_TYPES + +!> Contains routined for MPI shared memory. +module mpifx_win_module + use mpifx_common_module + use iso_c_binding, only : c_ptr, c_f_pointer + implicit none + private + + public :: mpifx_win + + !> MPI shared memory window with some additional information. + type mpifx_win + private + integer, public :: id !< Window id. + integer :: comm_id !< Communicator id. + contains + !> Initializes an MPI shared memory window. +#:for TYPE in TYPES + generic :: allocate_shared => mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$ +#:endfor + +#:for TYPE in TYPES + procedure, private :: mpifx_win_allocate_shared_${TYPE_ABBREVS[TYPE]}$ +#:endfor + + !> Locks a shared memory segment. + procedure :: lock => mpifx_win_lock + + !> Unlocks a shared memory segment. + procedure :: unlock => mpifx_win_unlock + + !> Synchronizes shared memory across MPI ranks. + procedure :: sync => mpifx_win_sync + + !> Deallocates memory associated with a shared memory segment. + procedure :: free => mpifx_win_free + + end type mpifx_win + +contains + +#:def mpifx_win_allocate_shared_template(SUFFIX, TYPE) + + !> Initialized a window handle and returns a pointer to the address associated with a shared memory segment. + !! + !! \param self Handle of the shared memory window on return. + !! \param mycomm MPI communicator. + !! \param length Number of elements of type ${TYPE}$ in the shared memory window. + !! \param shared_data Pointer to the shared data array of length 'length' on return. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_ALLOCATE_SHARED) + !! + subroutine mpifx_win_allocate_shared_${SUFFIX}$(self, mycomm, length, shared_data, error) + class(mpifx_win), intent(out) :: self + class(mpifx_comm), intent(in) :: mycomm + integer, intent(in) :: length + ${TYPE}$, pointer, intent(out) :: shared_data(:) + integer, intent(out), optional :: error + + integer :: disp_unit, error0, error1 + integer(MPI_ADDRESS_KIND) :: local_length + type(c_ptr) :: baseptr + + disp_unit = storage_size(shared_data) / 8 + + local_length = 0 + if (mycomm%lead) then + local_length = int(length, kind=MPI_ADDRESS_KIND) * disp_unit + end if + + call mpi_win_allocate_shared(local_length, disp_unit, MPI_INFO_NULL, mycomm%id, baseptr, self%id, error0) + call handle_errorflag(error0, "MPI_WIN_ALLOCATE_SHARED in mpifx_win_allocate_shared_${SUFFIX}$", error) + + call mpi_win_shared_query(self%id, 0, local_length, disp_unit, baseptr, error1) + call handle_errorflag(error1, "MPI_WIN_SHARED_QUERY in mpifx_win_allocate_shared_${SUFFIX}$", error) + + self%comm_id = mycomm%id + call c_f_pointer(baseptr, shared_data, [length]) + + end subroutine mpifx_win_allocate_shared_${SUFFIX}$ + +#:enddef mpifx_win_allocate_shared_template + + !> Locks a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_LOCK_ALL) + !! + subroutine mpifx_win_lock(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_lock_all(MPI_MODE_NOCHECK, self%id, error0) + call handle_errorflag(error0, "MPI_WIN_LOCK_ALL in mpifx_win_lock", error) + + end subroutine mpifx_win_lock + + !> Unlocks a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_UNLOCK_ALL) + !! + subroutine mpifx_win_unlock(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_unlock_all(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_UNLOCK_ALL in mpifx_win_unlock", error) + + end subroutine mpifx_win_unlock + + !> Synchronizes shared memory across MPI ranks. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_SYNC) + !! + subroutine mpifx_win_sync(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0, error1 + + call mpi_win_sync(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_SYNC in mpifx_win_sync", error) + + call mpi_barrier(self%comm_id, error1) + call handle_errorflag(error1, "MPI_BARRIER in mpifx_win_sync", error) + + end subroutine mpifx_win_sync + + !> Deallocates memory associated with a shared memory segment. + !! + !! \param self Handle of the shared memory window. + !! \param error Optional error code on return. + !! + !! \see MPI documentation (\c MPI_WIN_FREE) + !! + subroutine mpifx_win_free(self, error) + class(mpifx_win), intent(inout) :: self + integer, intent(out), optional :: error + + integer :: error0 + + call mpi_win_free(self%id, error0) + call handle_errorflag(error0, "MPI_WIN_FREE in mpifx_win_free", error) + + end subroutine mpifx_win_free + + +#:for TYPE in TYPES + #:set FTYPE = FORTRAN_TYPES[TYPE] + #:set SUFFIX = TYPE_ABBREVS[TYPE] + + $:mpifx_win_allocate_shared_template(SUFFIX, FTYPE) + +#:endfor + +end module mpifx_win_module diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 33983d7..394cd1e 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,16 +1,43 @@ -set(targets +set(tested test_allgather - test_allgatherv + test_allgatherv) + +set(targets + ${tested} test_allreduce test_bcast test_comm_split + test_comm_split_type test_gather test_gatherv test_reduce test_scatter - test_scatterv) + test_scatterv + test_win_shared_mem) + +set(sources-helper + testhelper.f90) + +add_library(mpifxtesthelp ${sources-helper}) +target_link_libraries(mpifxtesthelp PRIVATE MPI::MPI_Fortran MpiFx) foreach(target IN LISTS targets) add_executable(${target} ${target}.f90) - target_link_libraries(${target} MpiFx) + target_link_libraries(${target} MpiFx mpifxtesthelp) +endforeach() + +foreach(target IN LISTS tested) + add_test(NAME ${target} + COMMAND ${MPIEXEC_EXECUTABLE} + ${MPIEXEC_NUMPROC_FLAG} + ${MPIEXEC_MAX_NUMPROCS} + ${MPIEXEC_PREFLAGS} + ${CMAKE_CURRENT_BINARY_DIR}/${target} + ${MPIEXEC_POSTFLAGS}) + set_tests_properties(${target} PROPERTIES + # test cases generate this on stdOut + PASS_REGULAR_EXPRESSION "TestPASSED") + set_tests_properties(${target} PROPERTIES + # test cases generate this on stdOut + FAIL_REGULAR_EXPRESSION "TestFAILED") endforeach() diff --git a/test/meson.build b/test/meson.build index eb813e3..fd98ff5 100644 --- a/test/meson.build +++ b/test/meson.build @@ -6,11 +6,13 @@ tests = [ 'allreduce', 'bcast', 'comm_split', + 'comm_split_type', 'gather', 'gatherv', 'reduce', 'scatter', 'scatterv', + 'win_shared_mem', ] foreach t : tests diff --git a/test/test_allgather.f90 b/test/test_allgather.f90 index 7b58d3e..42933b0 100644 --- a/test/test_allgather.f90 +++ b/test/test_allgather.f90 @@ -1,5 +1,7 @@ +!> Test various patterns of allgather program test_allgather use libmpifx_module + use testhelper implicit none type(mpifx_comm) :: mycomm @@ -8,6 +10,7 @@ program test_allgather integer, allocatable :: recv1(:), recv2(:,:) character(100) :: formstr character(*), parameter :: label = "(I2.2,'-',I3.3,'|',1X" + logical :: isPassed call mpifx_init() call mycomm%init() @@ -16,12 +19,16 @@ program test_allgather send0 = mycomm%rank * 2 allocate(recv1(1 * mycomm%size)) recv1(:) = 0 - write(*, label // ",A,1X,I0)") 1, mycomm%rank, & - & "Send0 buffer:", send0 + write(*, label // ",A,1X,I0)") 1, mycomm%rank, "Send0 buffer:", send0 call mpifx_allgather(mycomm, send0, recv1) write(formstr, "(A,I0,A)") "A,", size(recv1), "(1X,I0))" - write(*, label // formstr) 2, mycomm%rank, & - & "Recv1 buffer:", recv1(:) + write(*, label // formstr) 2, mycomm%rank, "Recv1 buffer:", recv1(:) + if (sum(recv1) /= mycomm%size * (mycomm%size-1)) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) deallocate(recv1) ! I1 -> I1 @@ -30,25 +37,33 @@ program test_allgather recv1(:) = 0 send1(:) = [ mycomm%rank, mycomm%rank + 1 ] write(formstr, "(A,I0,A)") "A,", size(send1), "(1X,I0))" - write(*, label // formstr) 3, mycomm%rank, & - & "Send1 buffer:", send1(:) + write(*, label // formstr) 3, mycomm%rank, "Send1 buffer:", send1(:) call mpifx_allgather(mycomm, send1, recv1) write(formstr, "(A,I0,A)") "A,", size(recv1), "(1X,I0))" - write(*, label // formstr) 4, mycomm%rank, & - & "Recv1 buffer:", recv1 + write(*, label // formstr) 4, mycomm%rank, "Recv1 buffer:", recv1 + if (sum(recv1) /= mycomm%size**2) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) ! I1 -> I2 allocate(recv2(size(send1), mycomm%size)) recv2(:,:) = 0 send1(:) = [ mycomm%rank, mycomm%rank + 1 ] write(formstr, "(A,I0,A)") "A,", size(send1), "(1X,I0))" - write(*, label // formstr) 5, mycomm%rank, & - & "Send1 buffer:", send1(:) + write(*, label // formstr) 5, mycomm%rank, "Send1 buffer:", send1(:) call mpifx_allgather(mycomm, send1, recv2) write(formstr, "(A,I0,A)") "A,", size(recv2), "(1X,I0))" - write(*, label // formstr) 6, mycomm%rank, & - & "Recv2 buffer:", recv2 - + write(*, label // formstr) 6, mycomm%rank, "Recv2 buffer:", recv2 + if (sum(recv1) /= mycomm%size**2) then + tPassed = .false. + else + tPassed = .true. + end if + call testReturn(mycomm, tPassed) + call mpifx_finalize() - + end program test_allgather diff --git a/test/test_allgatherv.f90 b/test/test_allgatherv.f90 index e3c831d..ae46762 100644 --- a/test/test_allgatherv.f90 +++ b/test/test_allgatherv.f90 @@ -1,5 +1,7 @@ +!> Test various patterns of allgatherv program test_allgatherv use libmpifx_module + use testhelper implicit none type(mpifx_comm) :: mycomm @@ -12,6 +14,8 @@ program test_allgatherv integer :: ii, nrecv, nCol character(100) :: formstr character(*), parameter :: label = "(I2.2,'-',I3.3,'|',1X" + logical :: tPassed + integer :: iCount call mpifx_init() call mycomm%init() @@ -33,6 +37,15 @@ program test_allgatherv if (mycomm%rank == mycomm%size - 1) then write(*, *) "Recv1 buffer:", recv1 end if + ! test what has been gathered + iCount = (2*mycomm%size**3+3*mycomm%size**2+mycomm%size)/6 + if (nint(sum(recv1)) /= iCount) then + tPassed = .false. + else + tPassed = .true. + end if + tPassed = tPassed .and. (abs(sum(recv1)-nint(sum(recv1))) < epsilon(1.0_sp)) + call testReturn(mycomm, tPassed) deallocate(recvcounts) deallocate(recv1) @@ -59,6 +72,14 @@ program test_allgatherv write(*,*)recv2(:,ii) end do end if + iCount = 5*mycomm%size*(mycomm%size+1)*(2*mycomm%size+1)/6 + if (nint(sum(recv2)) /= iCount) then + tPassed = .false. + else + tPassed = .true. + end if + tPassed = tPassed .and. (abs(sum(recv2)-nint(sum(recv2))) < epsilon(1.0_sp)) + call testReturn(mycomm, tPassed) deallocate(recvcounts) @@ -81,6 +102,14 @@ program test_allgatherv if (mycomm%rank == mycomm%size - 1) then write(*, *) "Recv1 buffer:", recv1 end if + ! test what has been gathered + if (nint(sum(recv1)) /= (mycomm%size*(mycomm%size+1))/2) then + tPassed = .false. + else + tPassed = .true. + end if + tPassed = tPassed .and. (abs(sum(recv1)-nint(sum(recv1))) < epsilon(1.0_sp)) + call testReturn(mycomm, tPassed) call mpifx_finalize() diff --git a/test/test_comm_split_type.f90 b/test/test_comm_split_type.f90 new file mode 100644 index 0000000..c40491c --- /dev/null +++ b/test/test_comm_split_type.f90 @@ -0,0 +1,13 @@ +program test_split_type + use libmpifx_module + implicit none + + type(mpifx_comm) :: allproc, splitproc + + call mpifx_init() + call allproc%init() + call allproc%split_type(MPI_COMM_TYPE_SHARED, allproc%rank, splitproc) + write(*, "(2(A,1X,I0,1X))") "ID:", allproc%rank, "SPLIT ID", splitproc%rank + call mpifx_finalize() + +end program test_split_type diff --git a/test/test_win_shared_mem.f90 b/test/test_win_shared_mem.f90 new file mode 100644 index 0000000..048fda8 --- /dev/null +++ b/test/test_win_shared_mem.f90 @@ -0,0 +1,34 @@ +program test_win_shared_mem + use libmpifx_module + implicit none + + type(mpifx_comm) :: globalcomm, nodecomm + type(mpifx_win) :: win + integer, parameter :: length = 7 + integer, pointer :: data_pointer(:) + + call mpifx_init() + call globalcomm%init() + + ! Create a new communicator for all ranks on a node first + call globalcomm%split_type(MPI_COMM_TYPE_SHARED, globalcomm%rank, nodecomm) + + call win%allocate_shared(nodecomm, length, data_pointer) + + call win%lock() + + ! Only rank 0 writes data into the array + if (nodecomm%lead) then + data_pointer(:) = 42 + end if + + call win%sync() + call win%unlock() + + ! All ranks on the node will read the same value + write(*, "(2(A,1X,I0,1X))") "ID:", nodecomm%rank, "VALUE:", data_pointer(1) + + call win%free() + call mpifx_finalize() + +end program test_win_shared_mem diff --git a/test/testhelper.f90 b/test/testhelper.f90 new file mode 100644 index 0000000..4e8878d --- /dev/null +++ b/test/testhelper.f90 @@ -0,0 +1,36 @@ +!> Helper routines for testers +module testhelper + use libmpifx_module, only : mpifx_comm, mpifx_barrier, mpifx_finalize + implicit none + + private + public :: testReturn + +contains + + !> Return expected labels for ctest + subroutine testReturn(mycomm, tPassed) + + type(mpifx_comm), intent(in) :: mycomm + + logical, intent(in) :: tPassed + + call mpifx_barrier(mycomm) + + if (tPassed) then + if (mycomm%rank == 0) then + ! label for ctest regex + write(*,*)'TestPASSED' + end if + else + if (mycomm%rank == 0) then + ! label for ctest regex + write(*,*)'TestFAILED' + end if + call mpifx_finalize() + stop + end if + + end subroutine testReturn + +end module testhelper diff --git a/utils/srcmanip/set_version b/utils/srcmanip/set_version index 00be0d0..b5247c6 100755 --- a/utils/srcmanip/set_version +++ b/utils/srcmanip/set_version @@ -15,6 +15,10 @@ _VERSION_PATTERN = r'\d+\.\d+(?:\.\d+)?(?:-\w+)?' _FILES_AND_PATTERNS = [ # + ('VERSION', + r'{}'.format(_VERSION_PATTERN), + r'{version}'), + # ('doc/doxygen/Doxyfile', r'^PROJECT_NUMBER\s*=\s*([\'"]){}\1\s*$'.format(_VERSION_PATTERN), 'PROJECT_NUMBER = "{version}"\n'),