diff --git a/src/mpid/ch4/netmod/ofi/ofi_coll.h b/src/mpid/ch4/netmod/ofi/ofi_coll.h index 61043ac76be..49be1c26fb5 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_coll.h +++ b/src/mpid/ch4/netmod/ofi/ofi_coll.h @@ -45,22 +45,6 @@ static inline int MPIDI_NM_mpi_barrier(MPIR_Comm * comm_ptr, MPIR_Errflag_t * er goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_Barrier_recursive_doubling -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Barrier_recursive_doubling(comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_NM_mpi_bcast #undef FCNAME @@ -92,66 +76,6 @@ static inline int MPIDI_NM_mpi_bcast(void *buffer, int count, MPI_Datatype datat goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_Bcast_binomial -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_binomial(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_binomial(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_Bcast_scatter_doubling_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_scatter_doubling_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_Bcast_scatter_ring_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_scatter_ring_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_NM_mpi_allreduce #undef FCNAME @@ -182,42 +106,6 @@ static inline int MPIDI_NM_mpi_allreduce(const void *sendbuf, void *recvbuf, int goto fn_exit; } - -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_allreduce_recursive_doubling -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_allreduce_recursive_doubling(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allreduce_recursive_doubling(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_allreduce_reduce_scatter_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_allreduce_reduce_scatter_allgather(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allreduce_reduce_scatter_allgather(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); - - return mpi_errno; -} - - #undef FUNCNAME #define FUNCNAME MPIDI_NM_mpi_allgather #undef FCNAME @@ -427,40 +315,6 @@ static inline int MPIDI_NM_mpi_reduce(const void *sendbuf, void *recvbuf, int co goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_reduce_redscat_gather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_reduce_redscat_gather(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_redscat_gather(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_OFI_reduce_binomial -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_reduce_binomial(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_binomial(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_NM_mpi_reduce_scatter #undef FCNAME diff --git a/src/mpid/ch4/netmod/ofi/ofi_coll_impl.h b/src/mpid/ch4/netmod/ofi/ofi_coll_impl.h new file mode 100644 index 00000000000..5cb085248b8 --- /dev/null +++ b/src/mpid/ch4/netmod/ofi/ofi_coll_impl.h @@ -0,0 +1,160 @@ +/* -*- Mode: C ; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2006 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + * + * Portions of this code were written by Intel Corporation. + * Copyright (C) 2011-2016 Intel Corporation. Intel provides this material + * to Argonne National Laboratory subject to Software Grant and Corporate + * Contributor License Agreement dated February 8, 2012. + */ +#ifndef OFI_COLL_IMPL_H_INCLUDED +#define OFI_COLL_IMPL_H_INCLUDED + +#include "ofi_coll_params.h" + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_Barrier_recursive_doubling +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Barrier_recursive_doubling(comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_Bcast_binomial +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_Bcast_binomial(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_binomial(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_Bcast_scatter_doubling_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_Bcast_scatter_doubling_allgather(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_Bcast_scatter_ring_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_Bcast_scatter_ring_allgather(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_allreduce_recursive_doubling +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_allreduce_recursive_doubling(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Allreduce_recursive_doubling(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_allreduce_reduce_scatter_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_allreduce_reduce_scatter_allgather(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Allreduce_reduce_scatter_allgather(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_reduce_redscat_gather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_reduce_redscat_gather(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Reduce_redscat_gather(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_OFI_reduce_binomial +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_OFI_reduce_binomial(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_OFI_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Reduce_binomial(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); + + return mpi_errno; +} + +#endif /* OFI_COLL_IMPL_H_INCLUDED */ diff --git a/src/mpid/ch4/netmod/ofi/ofi_coll_select.h b/src/mpid/ch4/netmod/ofi/ofi_coll_select.h index 95eebadb8fa..eaf12a53833 100644 --- a/src/mpid/ch4/netmod/ofi/ofi_coll_select.h +++ b/src/mpid/ch4/netmod/ofi/ofi_coll_select.h @@ -3,76 +3,7 @@ #include "ofi_impl.h" #include "coll_algo_params.h" - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_binomial(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_scatter_doubling_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_Bcast_scatter_ring_allgather(void *buffer, int count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_allreduce_recursive_doubling(const void *sendbuf, - void *recvbuf, int count, - MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_allreduce_reduce_scatter_allgather(const void *sendbuf, - void *recvbuf, int count, - MPI_Datatype datatype, - MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_reduce_redscat_gather(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_OFI_reduce_binomial(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, MPI_Op op, - int root, MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_OFI_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; +#include "ofi_coll_impl.h" MPL_STATIC_INLINE_PREFIX MPIDI_OFI_coll_algo_container_t *MPIDI_OFI_Barrier_select(MPIR_Comm * comm_ptr, diff --git a/src/mpid/ch4/shm/posix/posix_coll.h b/src/mpid/ch4/shm/posix/posix_coll.h index f9ae364a891..d76144f8366 100644 --- a/src/mpid/ch4/shm/posix/posix_coll.h +++ b/src/mpid/ch4/shm/posix/posix_coll.h @@ -47,22 +47,6 @@ static inline int MPIDI_POSIX_mpi_barrier(MPIR_Comm * comm_ptr, MPIR_Errflag_t * goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_Barrier_recursive_doubling -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Barrier_recursive_doubling(comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_POSIX_mpi_bcast #undef FCNAME @@ -94,66 +78,6 @@ static inline int MPIDI_POSIX_mpi_bcast(void *buffer, int count, MPI_Datatype da goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_Bcast_binomial -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_binomial(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_binomial(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_Bcast_scatter_doubling_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_scatter_doubling_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_Bcast_scatter_doubling_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_scatter_ring_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_POSIX_mpi_allreduce #undef FCNAME @@ -186,40 +110,6 @@ static inline int MPIDI_POSIX_mpi_allreduce(const void *sendbuf, void *recvbuf, goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_allreduce_recursive_doubling -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_allreduce_recursive_doubling(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allreduce_recursive_doubling(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_allreduce_reduce_scatter_allgather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_allreduce_reduce_scatter_allgather(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allreduce_reduce_scatter_allgather(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_POSIX_mpi_allgather #undef FCNAME @@ -430,40 +320,6 @@ static inline int MPIDI_POSIX_mpi_reduce(const void *sendbuf, void *recvbuf, int goto fn_exit; } -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_reduce_redscat_gather -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_reduce_redscat_gather(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_redscat_gather(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); - - return mpi_errno; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_POSIX_reduce_binomial -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_reduce_binomial(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_binomial(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); - - return mpi_errno; -} - #undef FUNCNAME #define FUNCNAME MPIDI_POSIX_mpi_reduce_scatter #undef FCNAME diff --git a/src/mpid/ch4/shm/posix/posix_coll_impl.h b/src/mpid/ch4/shm/posix/posix_coll_impl.h new file mode 100644 index 00000000000..90c4bc76559 --- /dev/null +++ b/src/mpid/ch4/shm/posix/posix_coll_impl.h @@ -0,0 +1,160 @@ +/* -*- Mode: C ; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2006 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + * + * Portions of this code were written by Intel Corporation. + * Copyright (C) 2011-2016 Intel Corporation. Intel provides this material + * to Argonne National Laboratory subject to Software Grant and Corporate + * Contributor License Agreement dated February 8, 2012. + */ +#ifndef POSIX_COLL_IMPL_H_INCLUDED +#define POSIX_COLL_IMPL_H_INCLUDED + +#include "posix_coll_params.h" + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_Barrier_recursive_doubling +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Barrier_recursive_doubling(comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_Bcast_binomial +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_Bcast_binomial(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_binomial(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_Bcast_scatter_doubling_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_Bcast_scatter_doubling_allgather(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_Bcast_scatter_doubling_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_Bcast_scatter_ring_allgather(void *buffer, + int count, + MPI_Datatype datatype, + int root, + MPIR_Comm * comm_ptr, + MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_scatter_ring_allgather(buffer, count, datatype, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_allreduce_recursive_doubling +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_allreduce_recursive_doubling(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Allreduce_recursive_doubling(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_allreduce_reduce_scatter_allgather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_allreduce_reduce_scatter_allgather(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Allreduce_reduce_scatter_allgather(sendbuf, recvbuf, count, datatype, op, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_reduce_redscat_gather +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_reduce_redscat_gather(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Reduce_redscat_gather(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); + + return mpi_errno; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_POSIX_reduce_binomial +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX +int MPIDI_POSIX_reduce_binomial(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm_ptr, MPIR_Errflag_t * errflag, + MPIDI_POSIX_coll_algo_container_t * params_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Reduce_binomial(sendbuf, recvbuf, count, datatype, op, root, comm_ptr, errflag); + + return mpi_errno; +} + +#endif /* POSIX_COLL_IMPL_H_INCLUDED */ diff --git a/src/mpid/ch4/shm/posix/posix_coll_select.h b/src/mpid/ch4/shm/posix/posix_coll_select.h index 762f5cfdf2e..12ba823e03f 100644 --- a/src/mpid/ch4/shm/posix/posix_coll_select.h +++ b/src/mpid/ch4/shm/posix/posix_coll_select.h @@ -4,79 +4,7 @@ #include "posix_impl.h" #include "ch4_impl.h" #include "coll_algo_params.h" - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Barrier_recursive_doubling(MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_binomial(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_scatter_doubling_allgather(void *buffer, - int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_Bcast_scatter_ring_allgather(void *buffer, int count, - MPI_Datatype datatype, - int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_allreduce_recursive_doubling(const void *sendbuf, - void *recvbuf, int count, - MPI_Datatype datatype, - MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_allreduce_reduce_scatter_allgather(const void *sendbuf, - void *recvbuf, - int count, - MPI_Datatype datatype, - MPI_Op op, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_reduce_redscat_gather(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, int root, - MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_POSIX_reduce_binomial(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm_ptr, - MPIR_Errflag_t * errflag, - MPIDI_POSIX_coll_algo_container_t * params_container) -MPL_STATIC_INLINE_SUFFIX; +#include "posix_coll_impl.h" MPL_STATIC_INLINE_PREFIX MPIDI_POSIX_coll_algo_container_t *MPIDI_POSIX_Barrier_select(MPIR_Comm * comm_ptr, diff --git a/src/mpid/ch4/src/ch4_coll.h b/src/mpid/ch4/src/ch4_coll.h index 4e00c3c7454..434013d6383 100644 --- a/src/mpid/ch4/src/ch4_coll.h +++ b/src/mpid/ch4/src/ch4_coll.h @@ -14,7 +14,6 @@ #include "ch4_impl.h" #include "ch4r_proc.h" #include "ch4_coll_select.h" -#include "ch4_coll_params.h" MPL_STATIC_INLINE_PREFIX int MPID_Barrier(MPIR_Comm * comm, MPIR_Errflag_t * errflag) { @@ -53,271 +52,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Bcast(void *buffer, int count, MPI_Datatype da } -#undef FUNCNAME -#define FUNCNAME MPIDI_Barrier_composition_alpha -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_composition_alpha(MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * barrier_node_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - void * barrier_roots_container = MPIDI_coll_get_next_container(barrier_node_container); - void * bcast_node_container = MPIDI_coll_get_next_container(barrier_roots_container); - - /* do the intranode barrier on all nodes */ - if (comm->node_comm != NULL) - { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_barrier(comm->node_comm, errflag, barrier_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = - MPIDI_NM_mpi_barrier(comm->node_comm, errflag, barrier_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - - /* do the barrier across roots of all nodes */ - if (comm->node_roots_comm != NULL) { - mpi_errno = - MPIDI_NM_mpi_barrier(comm->node_roots_comm, errflag, barrier_roots_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - - /* release the local processes on each node with a 1-byte - broadcast (0-byte broadcast just returns without doing - anything) */ - if (comm->node_comm != NULL) - { - int i=0; -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_bcast(&i, 1, MPI_BYTE, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = - MPIDI_NM_mpi_bcast(&i, 1, MPI_BYTE, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Barrier_composition_beta -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_composition_beta(MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * barrier_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - -#ifndef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_NM_mpi_barrier(comm, errflag, barrier_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - if(comm->is_single_node) { - mpi_errno = - MPIDI_SHM_mpi_barrier(comm, errflag, barrier_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } else{ - mpi_errno = - MPIDI_NM_mpi_barrier(comm, errflag, barrier_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } -#endif/*MPIDI_BUILD_CH4_SHM*/ - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Barrier_intercomm -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_intercomm(MPIR_Comm * comm, MPIR_Errflag_t *errflag, - MPIDI_coll_algo_container_t *ch4_algo_parameters_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Barrier_inter(comm, errflag); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Bcast_composition_alpha -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_alpha(void *buffer, int count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * bcast_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - void * bcast_node_container = MPIDI_coll_get_next_container(bcast_roots_container); - - if (comm->node_roots_comm == NULL && comm->rank == root) { - mpi_errno = MPIC_Send(buffer, count, datatype, 0, MPIR_BCAST_TAG, comm->node_comm, errflag); - } - - if (comm->node_roots_comm != NULL && comm->rank != root && MPIR_Get_intranode_rank(comm, root) != -1){ - mpi_errno = MPIC_Recv(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), MPIR_BCAST_TAG, - comm->node_comm, MPI_STATUS_IGNORE, errflag); - } - - if (comm->node_roots_comm != NULL) { - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_internode_rank(comm, root), - comm->node_roots_comm, errflag, bcast_roots_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - if (comm->node_comm != NULL) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Bcast_composition_beta -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_beta(void *buffer, int count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * bcast_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - void * bcast_node_container_root_local = MPIDI_coll_get_next_container(bcast_roots_container); - void * bcast_node_container_root_remote = MPIDI_coll_get_next_container(bcast_node_container_root_local); - - if (comm->node_comm != NULL && - MPIR_Get_intranode_rank(comm, root) > 0) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_bcast(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), - comm->node_comm, errflag, bcast_node_container_root_local); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), - comm->node_comm, errflag, bcast_node_container_root_local); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - if (comm->node_roots_comm != NULL) { - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_internode_rank(comm, root), - comm->node_roots_comm, errflag, bcast_roots_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - if (comm->node_comm != NULL && - MPIR_Get_intranode_rank(comm, root) <= 0){ -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container_root_remote); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container_root_remote); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Bcast_composition_gamma -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_gamma(void *buffer, int count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * bcast_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - -#ifndef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, root, - comm, errflag, bcast_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - if(comm->is_single_node){ - mpi_errno = - MPIDI_SHM_mpi_bcast(buffer, count, datatype, root, - comm, errflag, bcast_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - else{ - mpi_errno = - MPIDI_NM_mpi_bcast(buffer, count, datatype, root, - comm, errflag, bcast_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } -#endif/*MPIDI_BUILD_CH4_SHM*/ - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Bcast_intercomm -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Bcast_inter(buffer, count, datatype, root, comm, errflag); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, MPIR_Errflag_t * errflag) @@ -338,130 +72,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Allreduce(const void *sendbuf, void *recvbuf, return ret; } -#undef FUNCNAME -#define FUNCNAME MPIDI_Allreduce_composition_alpha -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_composition_alpha(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * reduce_node_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - void * allred_roots_container = MPIDI_coll_get_next_container(reduce_node_container); - void * bcast_node_container = MPIDI_coll_get_next_container(allred_roots_container); - - if(comm->node_comm != NULL) { - if((sendbuf == MPI_IN_PLACE) && (comm->node_comm->rank != 0)) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = MPIDI_SHM_mpi_reduce(recvbuf, NULL, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = MPIDI_NM_mpi_reduce(recvbuf, NULL, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } else { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif/* MPIDI_BUILD_CH4_SHM */ - } - } else { - if (sendbuf != MPI_IN_PLACE) { - mpi_errno = MPIR_Localcopy(sendbuf, count, datatype, recvbuf, count, datatype); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - } - - if (comm->node_roots_comm != NULL) { - mpi_errno = MPIDI_NM_mpi_allreduce(MPI_IN_PLACE, recvbuf, count, datatype, op, comm->node_roots_comm, - errflag, allred_roots_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } - - if (comm->node_comm != NULL) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = MPIDI_SHM_mpi_bcast(recvbuf, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - mpi_errno = MPIDI_NM_mpi_bcast(recvbuf, count, datatype, 0, comm->node_comm, errflag, - bcast_node_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#endif - } - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Allreduce_composition_gamma -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_composition_beta(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * allred_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - -#ifndef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_NM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, - allred_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - if(comm->is_single_node) { - mpi_errno = - MPIDI_SHM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, - allred_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } else { - mpi_errno = - MPIDI_NM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, - allred_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } -#endif/*MPIDI_BUILD_CH4_SHM*/ - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Allreduce_intercomm -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Allreduce_inter(sendbuf, recvbuf, count, datatype, op, comm, errflag); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX int MPID_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, void *recvbuf, int recvcount, MPI_Datatype recvtype, MPIR_Comm * comm, @@ -639,204 +249,6 @@ MPL_STATIC_INLINE_PREFIX int MPID_Reduce(const void *sendbuf, void *recvbuf, return ret; } -#undef FUNCNAME -#define FUNCNAME MPIDI_Reduce_composition_alpha -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_composition_alpha(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - int mpi_errno_ret = MPI_SUCCESS; - MPI_Aint true_lb, true_extent, extent; - void * reduce_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - void * reduce_node_container = MPIDI_coll_get_next_container(reduce_roots_container); - - MPIR_CHKLMEM_DECL(1); - - void *tmp_buf = NULL; - - /* Create a temporary buffer on local roots of all nodes */ - if (comm->node_roots_comm != NULL) { - - MPIR_Type_get_true_extent_impl(datatype, &true_lb, &true_extent); - MPIR_Datatype_get_extent_macro(datatype, extent); - - MPIR_Ensure_Aint_fits_in_pointer(count * MPL_MAX(extent, true_extent)); - - MPIR_CHKLMEM_MALLOC(tmp_buf, void *, count*(MPL_MAX(extent,true_extent)), - mpi_errno, "temporary buffer"); - /* adjust for potential negative lower bound in datatype */ - tmp_buf = (void *)((char*)tmp_buf - true_lb); - } - - /* do the intranode reduce on all nodes other than the root's node */ - if (comm->node_comm != NULL && - MPIR_Get_intranode_rank(comm, root) == -1) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_reduce(sendbuf, tmp_buf, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); -#else - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, tmp_buf, count, datatype, op, 0, comm->node_comm, errflag, - reduce_node_container); -#endif/* MPIDI_BUILD_CH4_SHM */ - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = MPIR_ERR_GET_CLASS(mpi_errno); - MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); - MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - - /* do the internode reduce to the root's node */ - if (comm->node_roots_comm != NULL) { - if (comm->node_roots_comm->rank != MPIR_Get_internode_rank(comm, root)) { - /* I am not on root's node. Use tmp_buf if we - participated in the first reduce, otherwise use sendbuf */ - const void *buf = (comm->node_comm == NULL ? sendbuf : tmp_buf); - mpi_errno = - MPIDI_NM_mpi_reduce(buf, NULL, count, datatype, - op, MPIR_Get_internode_rank(comm, root), - comm->node_roots_comm, errflag, reduce_roots_container); - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = MPIR_ERR_GET_CLASS(mpi_errno); - MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); - MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - else { /* I am on root's node. I have not participated in the earlier reduce. */ - if (comm->rank != root) { - /* I am not the root though. I don't have a valid recvbuf. - Use tmp_buf as recvbuf. */ - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, tmp_buf, count, datatype, - op, MPIR_Get_internode_rank(comm, root), - comm->node_roots_comm, errflag, reduce_roots_container); - - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = MPIR_ERR_GET_CLASS(mpi_errno); - MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); - MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); - } - - /* point sendbuf at tmp_buf to make final intranode reduce easy */ - sendbuf = tmp_buf; - } - else { - /* I am the root. in_place is automatically handled. */ - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, - op, MPIR_Get_internode_rank(comm, root), - comm->node_roots_comm, errflag, reduce_roots_container); - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = MPIR_ERR_GET_CLASS(mpi_errno); - MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); - MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); - } - - /* set sendbuf to MPI_IN_PLACE to make final intranode reduce easy. */ - sendbuf = MPI_IN_PLACE; - } - } - - } - - /* do the intranode reduce on the root's node */ - if (comm->node_comm != NULL && - MPIR_Get_intranode_rank(comm, root) != -1) { -#ifdef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, - op, MPIR_Get_intranode_rank(comm, root), - comm->node_comm, errflag, reduce_node_container); -#else - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, - op, MPIR_Get_intranode_rank(comm, root), - comm->node_comm, errflag, reduce_node_container); -#endif/* MPIDI_BUILD_CH4_SHM */ - if (mpi_errno) { - /* for communication errors, just record the error but continue */ - *errflag = MPIR_ERR_GET_CLASS(mpi_errno); - MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); - MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); - } - } - -fn_exit: - MPIR_CHKLMEM_FREEALL(); - return mpi_errno; -fn_fail: - goto fn_exit; - -} - - -#undef FUNCNAME -#define FUNCNAME MPIDI_Reduce_composition_beta -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_composition_beta(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -{ - int mpi_errno = MPI_SUCCESS; - void * reduce_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); - -#ifndef MPIDI_BUILD_CH4_SHM - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, - comm, errflag, reduce_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); -#else - if(comm->is_single_node) { - mpi_errno = - MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, - comm, errflag, reduce_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } else { - mpi_errno = - MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, - comm, errflag, reduce_container); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - } -#endif/*MPIDI_BUILD_CH4_SHM*/ - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - -#undef FUNCNAME -#define FUNCNAME MPIDI_Reduce_intercomm -#undef FCNAME -#define FCNAME MPL_QUOTE(FUNCNAME) -MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) -{ - int mpi_errno = MPI_SUCCESS; - - mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, - op, root, comm, errflag); - if (mpi_errno) MPIR_ERR_POP(mpi_errno); - -fn_exit: - return mpi_errno; -fn_fail: - goto fn_exit; -} - MPL_STATIC_INLINE_PREFIX int MPID_Reduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts[], MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm_ptr, diff --git a/src/mpid/ch4/src/ch4_coll_impl.h b/src/mpid/ch4/src/ch4_coll_impl.h new file mode 100644 index 00000000000..5e16ff8cf71 --- /dev/null +++ b/src/mpid/ch4/src/ch4_coll_impl.h @@ -0,0 +1,604 @@ +/* -*- Mode: C ; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */ +/* + * (C) 2006 by Argonne National Laboratory. + * See COPYRIGHT in top-level directory. + * + * Portions of this code were written by Intel Corporation. + * Copyright (C) 2011-2016 Intel Corporation. Intel provides this material + * to Argonne National Laboratory subject to Software Grant and Corporate + * Contributor License Agreement dated February 8, 2012. + */ +#ifndef CH4_COLL_IMPL_H_INCLUDED +#define CH4_COLL_IMPL_H_INCLUDED + +#include "ch4_coll_params.h" +#include "coll_algo_params.h" + +#undef FUNCNAME +#define FUNCNAME MPIDI_Barrier_composition_alpha +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_composition_alpha(MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * barrier_node_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + void * barrier_roots_container = MPIDI_coll_get_next_container(barrier_node_container); + void * bcast_node_container = MPIDI_coll_get_next_container(barrier_roots_container); + + /* do the intranode barrier on all nodes */ + if (comm->node_comm != NULL) + { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_barrier(comm->node_comm, errflag, barrier_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = + MPIDI_NM_mpi_barrier(comm->node_comm, errflag, barrier_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + + /* do the barrier across roots of all nodes */ + if (comm->node_roots_comm != NULL) { + mpi_errno = + MPIDI_NM_mpi_barrier(comm->node_roots_comm, errflag, barrier_roots_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + + /* release the local processes on each node with a 1-byte + broadcast (0-byte broadcast just returns without doing + anything) */ + if (comm->node_comm != NULL) + { + int i=0; +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_bcast(&i, 1, MPI_BYTE, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = + MPIDI_NM_mpi_bcast(&i, 1, MPI_BYTE, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Barrier_composition_beta +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_composition_beta(MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * barrier_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + +#ifndef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_NM_mpi_barrier(comm, errflag, barrier_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + if(comm->is_single_node) { + mpi_errno = + MPIDI_SHM_mpi_barrier(comm, errflag, barrier_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } else{ + mpi_errno = + MPIDI_NM_mpi_barrier(comm, errflag, barrier_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } +#endif/*MPIDI_BUILD_CH4_SHM*/ + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Barrier_intercomm +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Barrier_intercomm(MPIR_Comm * comm, MPIR_Errflag_t *errflag, + MPIDI_coll_algo_container_t *ch4_algo_parameters_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Barrier_inter(comm, errflag); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Bcast_composition_alpha +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_alpha(void *buffer, int count, MPI_Datatype datatype, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * bcast_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + void * bcast_node_container = MPIDI_coll_get_next_container(bcast_roots_container); + + if (comm->node_roots_comm == NULL && comm->rank == root) { + mpi_errno = MPIC_Send(buffer, count, datatype, 0, MPIR_BCAST_TAG, comm->node_comm, errflag); + } + + if (comm->node_roots_comm != NULL && comm->rank != root && MPIR_Get_intranode_rank(comm, root) != -1){ + mpi_errno = MPIC_Recv(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), MPIR_BCAST_TAG, + comm->node_comm, MPI_STATUS_IGNORE, errflag); + } + + if (comm->node_roots_comm != NULL) { + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_internode_rank(comm, root), + comm->node_roots_comm, errflag, bcast_roots_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + if (comm->node_comm != NULL) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Bcast_composition_beta +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_beta(void *buffer, int count, MPI_Datatype datatype, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * bcast_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + void * bcast_node_container_root_local = MPIDI_coll_get_next_container(bcast_roots_container); + void * bcast_node_container_root_remote = MPIDI_coll_get_next_container(bcast_node_container_root_local); + + if (comm->node_comm != NULL && + MPIR_Get_intranode_rank(comm, root) > 0) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_bcast(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), + comm->node_comm, errflag, bcast_node_container_root_local); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_intranode_rank(comm, root), + comm->node_comm, errflag, bcast_node_container_root_local); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + if (comm->node_roots_comm != NULL) { + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, MPIR_Get_internode_rank(comm, root), + comm->node_roots_comm, errflag, bcast_roots_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + if (comm->node_comm != NULL && + MPIR_Get_intranode_rank(comm, root) <= 0){ +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container_root_remote); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container_root_remote); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Bcast_composition_gamma +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_composition_gamma(void *buffer, int count, MPI_Datatype datatype, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * bcast_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + +#ifndef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, root, + comm, errflag, bcast_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + if(comm->is_single_node){ + mpi_errno = + MPIDI_SHM_mpi_bcast(buffer, count, datatype, root, + comm, errflag, bcast_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + else{ + mpi_errno = + MPIDI_NM_mpi_bcast(buffer, count, datatype, root, + comm, errflag, bcast_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } +#endif/*MPIDI_BUILD_CH4_SHM*/ + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Bcast_intercomm +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, + int root, MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Bcast_inter(buffer, count, datatype, root, comm, errflag); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Allreduce_composition_alpha +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_composition_alpha(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, + MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * reduce_node_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + void * allred_roots_container = MPIDI_coll_get_next_container(reduce_node_container); + void * bcast_node_container = MPIDI_coll_get_next_container(allred_roots_container); + + if(comm->node_comm != NULL) { + if((sendbuf == MPI_IN_PLACE) && (comm->node_comm->rank != 0)) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = MPIDI_SHM_mpi_reduce(recvbuf, NULL, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = MPIDI_NM_mpi_reduce(recvbuf, NULL, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } else { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif/* MPIDI_BUILD_CH4_SHM */ + } + } else { + if (sendbuf != MPI_IN_PLACE) { + mpi_errno = MPIR_Localcopy(sendbuf, count, datatype, recvbuf, count, datatype); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + } + + if (comm->node_roots_comm != NULL) { + mpi_errno = MPIDI_NM_mpi_allreduce(MPI_IN_PLACE, recvbuf, count, datatype, op, comm->node_roots_comm, + errflag, allred_roots_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } + + if (comm->node_comm != NULL) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = MPIDI_SHM_mpi_bcast(recvbuf, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + mpi_errno = MPIDI_NM_mpi_bcast(recvbuf, count, datatype, 0, comm->node_comm, errflag, + bcast_node_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#endif + } + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Allreduce_composition_gamma +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_composition_beta(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, + MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * allred_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + +#ifndef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_NM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, + allred_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + if(comm->is_single_node) { + mpi_errno = + MPIDI_SHM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, + allred_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } else { + mpi_errno = + MPIDI_NM_mpi_allreduce(sendbuf, recvbuf, count, datatype, op, comm, errflag, + allred_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } +#endif/*MPIDI_BUILD_CH4_SHM*/ + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Allreduce_intercomm +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, MPIR_Comm * comm, + MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Allreduce_inter(sendbuf, recvbuf, count, datatype, op, comm, errflag); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Reduce_composition_alpha +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_composition_alpha(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + int mpi_errno_ret = MPI_SUCCESS; + MPI_Aint true_lb, true_extent, extent; + void * reduce_roots_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + void * reduce_node_container = MPIDI_coll_get_next_container(reduce_roots_container); + + MPIR_CHKLMEM_DECL(1); + + void *tmp_buf = NULL; + + /* Create a temporary buffer on local roots of all nodes */ + if (comm->node_roots_comm != NULL) { + + MPIR_Type_get_true_extent_impl(datatype, &true_lb, &true_extent); + MPIR_Datatype_get_extent_macro(datatype, extent); + + MPIR_Ensure_Aint_fits_in_pointer(count * MPL_MAX(extent, true_extent)); + + MPIR_CHKLMEM_MALLOC(tmp_buf, void *, count*(MPL_MAX(extent,true_extent)), + mpi_errno, "temporary buffer"); + /* adjust for potential negative lower bound in datatype */ + tmp_buf = (void *)((char*)tmp_buf - true_lb); + } + + /* do the intranode reduce on all nodes other than the root's node */ + if (comm->node_comm != NULL && + MPIR_Get_intranode_rank(comm, root) == -1) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_reduce(sendbuf, tmp_buf, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); +#else + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, tmp_buf, count, datatype, op, 0, comm->node_comm, errflag, + reduce_node_container); +#endif/* MPIDI_BUILD_CH4_SHM */ + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = MPIR_ERR_GET_CLASS(mpi_errno); + MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); + MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + + /* do the internode reduce to the root's node */ + if (comm->node_roots_comm != NULL) { + if (comm->node_roots_comm->rank != MPIR_Get_internode_rank(comm, root)) { + /* I am not on root's node. Use tmp_buf if we + participated in the first reduce, otherwise use sendbuf */ + const void *buf = (comm->node_comm == NULL ? sendbuf : tmp_buf); + mpi_errno = + MPIDI_NM_mpi_reduce(buf, NULL, count, datatype, + op, MPIR_Get_internode_rank(comm, root), + comm->node_roots_comm, errflag, reduce_roots_container); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = MPIR_ERR_GET_CLASS(mpi_errno); + MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); + MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + else { /* I am on root's node. I have not participated in the earlier reduce. */ + if (comm->rank != root) { + /* I am not the root though. I don't have a valid recvbuf. + Use tmp_buf as recvbuf. */ + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, tmp_buf, count, datatype, + op, MPIR_Get_internode_rank(comm, root), + comm->node_roots_comm, errflag, reduce_roots_container); + + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = MPIR_ERR_GET_CLASS(mpi_errno); + MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); + MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); + } + + /* point sendbuf at tmp_buf to make final intranode reduce easy */ + sendbuf = tmp_buf; + } + else { + /* I am the root. in_place is automatically handled. */ + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, + op, MPIR_Get_internode_rank(comm, root), + comm->node_roots_comm, errflag, reduce_roots_container); + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = MPIR_ERR_GET_CLASS(mpi_errno); + MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); + MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); + } + + /* set sendbuf to MPI_IN_PLACE to make final intranode reduce easy. */ + sendbuf = MPI_IN_PLACE; + } + } + + } + + /* do the intranode reduce on the root's node */ + if (comm->node_comm != NULL && + MPIR_Get_intranode_rank(comm, root) != -1) { +#ifdef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, + op, MPIR_Get_intranode_rank(comm, root), + comm->node_comm, errflag, reduce_node_container); +#else + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, + op, MPIR_Get_intranode_rank(comm, root), + comm->node_comm, errflag, reduce_node_container); +#endif/* MPIDI_BUILD_CH4_SHM */ + if (mpi_errno) { + /* for communication errors, just record the error but continue */ + *errflag = MPIR_ERR_GET_CLASS(mpi_errno); + MPIR_ERR_SET(mpi_errno, *errflag, "**fail"); + MPIR_ERR_ADD(mpi_errno_ret, mpi_errno); + } + } + +fn_exit: + MPIR_CHKLMEM_FREEALL(); + return mpi_errno; +fn_fail: + goto fn_exit; + +} + + +#undef FUNCNAME +#define FUNCNAME MPIDI_Reduce_composition_beta +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_composition_beta(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container) +{ + int mpi_errno = MPI_SUCCESS; + void * reduce_container = MPIDI_coll_get_next_container(ch4_algo_parameters_container); + +#ifndef MPIDI_BUILD_CH4_SHM + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag, reduce_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); +#else + if(comm->is_single_node) { + mpi_errno = + MPIDI_SHM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag, reduce_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } else { + mpi_errno = + MPIDI_NM_mpi_reduce(sendbuf, recvbuf, count, datatype, op, root, + comm, errflag, reduce_container); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + } +#endif/*MPIDI_BUILD_CH4_SHM*/ + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#undef FUNCNAME +#define FUNCNAME MPIDI_Reduce_intercomm +#undef FCNAME +#define FCNAME MPL_QUOTE(FUNCNAME) +MPL_STATIC_INLINE_PREFIX int MPIDI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, + MPI_Datatype datatype, MPI_Op op, int root, + MPIR_Comm * comm, MPIR_Errflag_t * errflag, + MPIDI_coll_algo_container_t * ch4_algo_parameters_container ATTRIBUTE((unused))) +{ + int mpi_errno = MPI_SUCCESS; + + mpi_errno = MPIR_Reduce_inter(sendbuf, recvbuf, count, datatype, + op, root, comm, errflag); + if (mpi_errno) MPIR_ERR_POP(mpi_errno); + +fn_exit: + return mpi_errno; +fn_fail: + goto fn_exit; +} + +#endif /* CH4_COLL_IMPL_H_INCLUDED */ diff --git a/src/mpid/ch4/src/ch4_coll_select.h b/src/mpid/ch4/src/ch4_coll_select.h index 9c8295b3eea..7f42c86d9ae 100644 --- a/src/mpid/ch4/src/ch4_coll_select.h +++ b/src/mpid/ch4/src/ch4_coll_select.h @@ -13,108 +13,13 @@ #include "ch4_impl.h" #include "ch4r_proc.h" +#include "ch4_coll_impl.h" #ifdef MPIDI_BUILD_CH4_SHM #include "../shm/include/shm.h" #endif #include "../../mpi/coll/collutil.h" -#include "coll_algo_params.h" - -/* SHM_Barrier and NM_Barrier */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Barrier_composition_alpha(MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* SHM_Barrier or NM_Barrier */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Barrier_composition_beta(MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* Wrapper for intercommunicator */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Barrier_intercomm(MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* NM_Bcast and SHM_Bcast */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Bcast_composition_alpha(void *buffer, int count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; - -MPL_STATIC_INLINE_PREFIX -int MPIDI_Bcast_composition_beta(void *buffer, int count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* NM_Bcast or SHM_Bcast */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Bcast_composition_gamma(void *buffer, int count, - MPI_Datatype datatype, int root, - MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* Wrapper for intercommunicator */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Bcast_intercomm(void *buffer, int count, MPI_Datatype datatype, - int root, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* SHM_Reduce, NM_Allreduce and SHM_Bcast */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Allreduce_composition_alpha(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* NM_Reduce and NM_Bcast */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Allreduce_composition_beta(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* Wrapper for intercommunicator */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Allreduce_intercomm(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, MPI_Op op, - MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* SHM_Reduce and NM_Reduce */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Reduce_composition_alpha(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* NM_Reduce or SHM_Reduce */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Reduce_composition_beta(const void *sendbuf, void *recvbuf, - int count, MPI_Datatype datatype, - MPI_Op op, int root, MPIR_Comm * comm, - MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; -/* Wrapper for intercommunicator */ -MPL_STATIC_INLINE_PREFIX -int MPIDI_Reduce_intercomm(const void *sendbuf, void *recvbuf, int count, - MPI_Datatype datatype, MPI_Op op, int root, - MPIR_Comm * comm, MPIR_Errflag_t * errflag, - MPIDI_coll_algo_container_t * ch4_algo_parameters_container) -MPL_STATIC_INLINE_SUFFIX; MPL_STATIC_INLINE_PREFIX MPIDI_coll_algo_container_t *MPIDI_CH4_Barrier_select(MPIR_Comm * comm,