Implemented zero copy between vale ports

LINUX/configure

@@ -7,6 +7,7 @@ DEBUG=1
 UTILS=1
 DRVERRFAIL=
 DMASYNC=1
+VALE_ZERO_COPY=0
 
 # setelem2n <set> <elem>
 setelem2n()
@@ -280,6 +281,7 @@ replace_vars()
 		-e "s|@DESTDIR@|$DESTDIR|g" \
 		-e "s|@DEBUG@|$DEBUG|g" \
 		-e "s|@UTILS@|$UTILS|g" \
+		-e "s|@VALE_ZERO_COPY@|$VALE_ZERO_COPY|g" \
 		$1
 }
 
@@ -329,6 +331,7 @@ Available options:
   --no-force-debug	       build the modules w/ or w/o debug symbols,
   --cache=		       dir for reusing/caching of netmap_linux_config.h
   --without-dmasync	       use this if you are on x86 with a recent NIC and no IOMMU
+  --vale-zero-copy	       enable zero copy between nic and host rings
 
   --cc=                        C compiler to be used for the apps and utils [$cc]
   --ld=                        linker to be used for the apps and utils [$ld]
@@ -650,6 +653,9 @@ for opt do
 	--without-dmasync)
 		DMASYNC=
 		;;
+	--vale-zero-copy)
+		VALE_ZERO_COPY=1
+		;;
 	*)
 		echo "Unrecognized option: $opt" | warning
 	;;

LINUX/netmap.mak.in

@@ -10,6 +10,7 @@ SUBSYS:=@SUBSYS@
 SRCDIR:=@SRCDIR@
 BUILDDIR:=@BUILDDIR@
 DEBUG:=@DEBUG@
+VALE_ZERO_COPY:=@VALE_ZERO_COPY@
 
 # The following commands are needed to build the modules as out-of-tree,
 # in fact the kernel sources path must be specified.
@@ -37,6 +38,10 @@ E_DRIVERS = @E_DRIVERS@
 DRVSUFFIX = @DRVSUFFIX@
 UTILS = @UTILS@
 
+ifeq (1,$(VALE_ZERO_COPY))
+EXTRA_CFLAGS += -DVALE_ZERO_COPY
+endif
+
 ifeq (,$(DRVSUFFIX))
 else
 EXTRA_CFLAGS += -DNETMAP_DRIVER_SUFFIX=$(DRVSUFFIX)

sys/dev/netmap/netmap_kern.h

@@ -2079,6 +2079,7 @@ void nm_os_mitigation_cleanup(struct nm_generic_mit *mit);
  */
 struct nm_bdg_fwd {	/* forwarding entry for a bridge */
 	void *ft_buf;		/* netmap or indirect buffer */
+	struct netmap_slot *ft_src_slot;	/* original slot - for zero copy */
 	uint8_t ft_frags;	/* how many fragments (only on 1st frag) */
 	uint16_t ft_offset;	/* dst port (unused) */
 	uint16_t ft_flags;	/* flags, e.g. indirect */

sys/dev/netmap/netmap_vale.c

@@ -643,6 +643,7 @@ nm_vale_preflush(struct netmap_kring *kring, u_int end)
 		ft[ft_i].ft_len = slot->len;
 		ft[ft_i].ft_flags = slot->flags;
 		ft[ft_i].ft_offset = 0;
+		ft[ft_i].ft_src_slot = slot;
 
 		ND("flags is 0x%x", slot->flags);
 		/* we do not use the buf changed flag, but we still need to reset it */
@@ -1080,6 +1081,7 @@ nm_vale_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,
 			struct netmap_slot *slot;
 			struct nm_bdg_fwd *ft_p, *ft_end;
 			u_int cnt;
+			bool is_bdcast = false;
 
 			/* find the queue from which we pick next packet.
 			 * NM_FT_NULL is always higher than valid indexes
@@ -1091,6 +1093,7 @@ nm_vale_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,
 				ft_p = ft + next;
 				next = ft_p->ft_next;
 			} else { /* insert broadcast */
+				is_bdcast = true;
 				ft_p = ft + brd_next;
 				brd_next = ft_p->ft_next;
 			}
@@ -1128,8 +1131,32 @@ nm_vale_flush(struct nm_bdg_fwd *ft, u_int n, struct netmap_vp_adapter *na,
 							dst_len = 0;
 						}
 					} else {
+
+#if VALE_ZERO_COPY
+						/* do zero copy only under the following conditions :
+						   1. not a broadacast packet
+						   2. if both src and dst slots are in the same memory region
+						 */
+						if(likely(!is_bdcast &&
+							 (na->up.nm_mem == dst_na->up.nm_mem))) {
+							struct netmap_slot *src_slot, *dst_slot;
+							struct netmap_slot tmp;
+							dst_slot = slot;
+							src_slot = ft_p->ft_src_slot;
+							tmp = *dst_slot;
+							*dst_slot = *src_slot;
+							*src_slot = tmp;
+							dst_slot->flags |= NS_BUF_CHANGED;
+							src_slot->flags |= NS_BUF_CHANGED;
+						} else {
+							//memcpy(dst, src, copy_len);
+							pkt_copy(src, dst, (int)copy_len);
+						}
+#else
 						//memcpy(dst, src, copy_len);
 						pkt_copy(src, dst, (int)copy_len);
+#endif
+
 					}
 					slot->len = dst_len;
 					slot->flags = (cnt << 8)| NS_MOREFRAG;