prov/sharp scaffolding

.travis.yml

@@ -104,6 +104,7 @@ install:
       --disable-udp
       --disable-usnic
       --disable-verbs
+      --disable-sharp
     - make -j2 $MAKE_FLAGS
     - make install
     - make test

Makefile.am

@@ -1,6 +1,6 @@
 #
 # Copyright (c) 2016 Cisco Systems, Inc.  All rights reserved.
-# Copyright (c) 2017-2018 Intel Corporation, Inc. All right reserved.
+# Copyright (c) 2017-2022 Intel Corporation, Inc. All right reserved.
 # Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All rights reserved.
 # (C) Copyright 2020 Hewlett Packard Enterprise Development LP
 #
@@ -454,6 +454,7 @@ include prov/rxd/Makefile.include
 include prov/bgq/Makefile.include
 include prov/opx/Makefile.include
 include prov/shm/Makefile.include
+include prov/sharp/Makefile.include
 include prov/tcp/Makefile.include
 include prov/net/Makefile.include
 include prov/rstream/Makefile.include

README.md

@@ -211,6 +211,15 @@ over MSG endpoints of a core provider.
 
 See [`fi_rxm`(7)](https://ofiwg.github.io/libfabric/main/man/fi_rxm.7.html) for more information.
 
+### sharp
+
+***
+
+The `off_sharp` provider is an utility provider that supports collective endpoints utilizing
+SHARP protocol for the barier and the allreduce operations.
+
+See [`fi_sharp`(7)](https://ofiwg.github.io/libfabric/main/man/fi_sharp.7.html) for more information.
+
 ### sockets
 
 ***

config/fi_provider.m4

@@ -85,10 +85,12 @@ dnl
 
 	# Check the --enable-<$1> value
 	$1_dl=0
+	$1_mocked=0
 	AS_CASE([$enable_$1],
 	[yes|no], [],
 	[dl],     [enable_$1=yes $1_dl=1],
-	[auto],   [],
+	[mocked], [enable_$1=yes $1_mocked=1],
+	[auto],   [$1_mocked=1],
 	[dl:*],   [FI_CHECK_PREFIX_DIR([${enable_$1:3}], [$1])
 		   enable_$1=yes $1_dl=1],
 	[FI_CHECK_PREFIX_DIR([$enable_$1], [$1])
@@ -130,13 +132,16 @@ dnl
 
 	AC_DEFINE_UNQUOTED([HAVE_]m4_translit([$1], [a-z], [A-Z]), $$1_happy, [$1 provider is built])
 	AC_DEFINE_UNQUOTED([HAVE_]m4_translit([$1], [a-z], [A-Z])[_DL], $$1_dl, [$1 provider is built as DSO])
+	AC_DEFINE_UNQUOTED([HAVE_]m4_translit([$1], [a-z], [A-Z])[_MOCKED], $$1_mocked, [$1 provider is mocked])
 
 	# Set AM conditionals for HAVE_<provider> and HAVE_<provider>_DL
 	# as well as AC defines
 	AM_CONDITIONAL([HAVE_]m4_translit([$1], [a-z], [A-Z]),
 		[test $$1_happy -eq 1])
 	AM_CONDITIONAL([HAVE_]m4_translit([$1], [a-z], [A-Z])[_DL],
 		[test $$1_dl -eq 1])
+	AM_CONDITIONAL([HAVE_]m4_translit([$1], [a-z], [A-Z])[_MOCKED],
+		[test $$1_mocked -eq 1])
 
 	# If this provider was specifically requested but we can't
 	# build it, error.

configure.ac

@@ -926,6 +926,7 @@ FI_PROVIDER_SETUP([hook_debug])
 FI_PROVIDER_SETUP([hook_hmem])
 FI_PROVIDER_SETUP([dmabuf_peer_mem])
 FI_PROVIDER_SETUP([opx])
+FI_PROVIDER_SETUP([sharp])
 FI_PROVIDER_FINI
 dnl Configure the .pc file
 FI_PROVIDER_SETUP_PC

include/ofi_prov.h

@@ -334,4 +334,18 @@ OPX_INI ;
 #define COLL_INIT fi_coll_ini()
 COLL_INI ;
 
+#ifndef HAVE_SHARP
+#define HAVE_SHARP 0
+#endif
+#if (HAVE_SHARP) && (HAVE_SHARP_DL)
+#  define SHARP_INI FI_EXT_INI
+#  define SHARP_INIT NULL
+#elif (HAVE_SHARP)
+#  define SHARP_INI INI_SIG(fi_sharp_ini)
+#  define SHARP_INIT fi_sharp_ini()
+SHARP_INI ;
+#else
+#  define SHARP_INIT NULL
+#endif
+
 #endif /* _OFI_PROV_H_ */

include/ofi_sharp.h

@@ -0,0 +1,70 @@
+/*
+ * Copyright (c) 2022 Intel Corporation. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _OFI_SHARP_H_
+#define _OFI_SHARP_H_
+
+#include "config.h"
+
+#include <stdint.h>
+#include <stddef.h>
+#include <sys/un.h>
+
+#include <ofi_atom.h>
+#include <ofi_proto.h>
+#include <ofi_mem.h>
+#include <ofi_rbuf.h>
+#include <ofi_tree.h>
+#include <ofi_hmem.h>
+
+#include <rdma/providers/fi_prov.h>
+
+#include "ofi_coll.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define SHARP_VERSION	1
+
+
+#if ENABLE_DEBUG
+#define SHARP_FLAG_DEBUG	(1 << 1)
+#else
+#define SHARP_FLAG_DEBUG	(0 << 1)
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _OFI_SHARP_H_ */

man/fi_sharp.7.md

@@ -0,0 +1,65 @@
+---
+layout: page
+title: fi_sharp(7)
+tagline: Libfabric Programmer's Manual
+---
+{% include JB/setup %}
+
+# NAME
+
+fi_sharp \- The SHARP Fabric Provider
+
+# OVERVIEW
+
+The SHARP provider is a collectives offload provider that can be used on Linux
+systems supporting the SHARP protocol.
+
+# SUPPORTED FEATURES
+
+This release contains an initial implementation of the SHARP provider that
+offers the following support:
+
+*Endpoint types*
+: The provider supports only the *FI_EP_RDM* endpoint type.
+
+*Endpoint capabilities*
+: Endpoint supports only the fi_barrier and the fi_allreduce operations.
+
+*Modes*
+: The provider does not require the use of any mode bits.
+
+*Progress*
+: The SHARP provider supports *FI_PROGRESS_MANUAL*.
+
+*Address Format*
+: TBD
+
+*Msg flags*
+  The provider does not support messaging.
+
+*MR registration mode*
+  The provider implements the FI_MR_VIRT_ADDR memory mode.
+
+*Atomic operations*
+  The provider does not support any atomic operation.
+
+# LIMITATIONS
+
+The SHARP provider has hard-coded maximums for supported queue sizes and data
+transfers. These values are reflected in the related fabric attribute
+structures.
+
+No support for counters.
+
+# RUNTIME PARAMETERS
+
+The SHARP provider checks for the following environment variables:
+
+*FI_SHARP_IB_PORT*
+: Default: 1
+
+# SEE ALSO
+
+[`fabric`(7)](fabric.7.html),
+[`fi_provider`(7)](fi_provider.7.html),
+[`fi_getinfo`(3)](fi_getinfo.3.html)

man/man7/fi_sharp.7

@@ -0,0 +1,68 @@
+.\" Automatically generated by Pandoc 2.0.6
+.\"
+.TH "fi_sharp" "7" "2022\-12\-14" "Libfabric Programmer's Manual" "#VERSION#"
+.hy
+.SH NAME
+.PP
+fi_sharp \- The SHARP Fabric Provider
+.SH OVERVIEW
+.PP
+The SHARP provider is a collectives offload provider that can be used on
+Linux systems supporting the SHARP protocol.
+.SH SUPPORTED FEATURES
+.PP
+This release contains an initial implementation of the SHARP provider
+that offers the following support:
+.TP
+.B \f[I]Endpoint types\f[]
+The provider supports only the \f[I]FI_EP_RDM\f[] endpoint type.
+.RS
+.RE
+.TP
+.B \f[I]Endpoint capabilities\f[]
+Endpoint supports only the fi_barrier and the fi_allreduce operations.
+.RS
+.RE
+.TP
+.B \f[I]Modes\f[]
+The provider does not require the use of any mode bits.
+.RS
+.RE
+.TP
+.B \f[I]Progress\f[]
+The SHARP provider supports \f[I]FI_PROGRESS_MANUAL\f[].
+.RS
+.RE
+.TP
+.B \f[I]Address Format\f[]
+TBD
+.RS
+.RE
+.PP
+\f[I]Msg flags\f[] The provider does not support messaging.
+.PP
+\f[I]MR registration mode\f[] The provider implements the
+FI_MR_VIRT_ADDR memory mode.
+.PP
+\f[I]Atomic operations\f[] The provider does not support any atomic
+operation.
+.SH LIMITATIONS
+.PP
+The SHARP provider has hard\-coded maximums for supported queue sizes
+and data transfers.
+These values are reflected in the related fabric attribute structures.
+.PP
+No support for counters.
+.SH RUNTIME PARAMETERS
+.PP
+The SHARP provider checks for the following environment variables:
+.TP
+.B \f[I]FI_SHARP_IB_PORT\f[]
+Default: 1
+.RS
+.RE
+.SH SEE ALSO
+.PP
+\f[C]fabric\f[](7), \f[C]fi_provider\f[](7), \f[C]fi_getinfo\f[](3)
+.SH AUTHORS
+OpenFabrics.

prov/coll/src/coll_coll.c

@@ -665,8 +665,7 @@ static struct fi_ops util_coll_fi_ops = {
 	.ops_open = fi_no_ops_open,
 };
 
-static int coll_find_local_rank(struct fid_ep *ep,
-				struct util_coll_mc *coll_mc)
+int coll_find_local_rank(struct fid_ep *ep, struct util_coll_mc *coll_mc)
 {
 	struct coll_av *av = container_of(coll_mc->av_set->av, struct coll_av,
 					  util_av.av_fid);

prov/rxm/src/rxm_domain.c

@@ -323,6 +323,9 @@ static int rxm_query_collective(struct fid_domain *domain,
 
 	if (!rxm_domain->util_coll_domain)
 		return -FI_ENOSYS;
+	if (rxm_domain->offload_coll_domain)
+		return fi_query_collective(rxm_domain->offload_coll_domain,
+					coll, attr, flags);
 
 	if (rxm_domain->offload_coll_domain)
 		ret = fi_query_collective(rxm_domain->offload_coll_domain,

prov/sharp/Makefile.include

@@ -0,0 +1,38 @@
+if HAVE_SHARP
+_sharp_files = \
+	include/ofi_sharp.h			\
+	prov/sharp/src/sharp.h			\
+	prov/sharp/src/sharp_attr.c		\
+	prov/sharp/src/sharp_init.c		\
+	prov/sharp/src/sharp_fabric.c		\
+	prov/sharp/src/sharp_domain.c		\
+	prov/sharp/src/sharp_eq.c		\
+	prov/sharp/src/sharp_ep.c		\
+	prov/sharp/src/sharp_cq.c		\
+	prov/sharp/src/sharp_coll.c		\
+	prov/sharp/src/sharp_progress.c
+
+if HAVE_SHARP_MOCKED
+_sharp_files += \
+	prov/sharp/src/mocks/api/sharp.h	\
+	prov/sharp/src/mocks/api/sharp_mocks.c
+else !HAVE_SHARP_MOCKED
+sharp_LIBS = -lsharp -lsharp_coll
+endif !HAVE_SHARP_MOCKED
+
+if HAVE_SHARP_DL
+pkglib_LTLIBRARIES += libsharp-fi.la
+libsharp_fi_la_SOURCES = $(_sharp_files) $(common_srcs)
+libsharp_fi_la_LIBADD = $(linkback) $(sharp_LIBS)
+libsharp_fi_la_LDFLAGS = -module -avoid-version -shared -export-dynamic
+libsharp_fi_la_DEPENDENCIES = $(linkback)
+else !HAVE_SHARP_DL
+src_libfabric_la_SOURCES += $(_sharp_files)
+src_libfabric_la_LIBADD += $(sharp_LIBS)
+endif !HAVE_SHARP_DL
+
+prov_install_man_pages += man/man7/fi_sharp.7
+
+endif HAVE_SHARP
+
+prov_dist_man_pages += man/man7/fi_sharp.7