Support Cumulus VX virtual switches.

hsflowd.spec

@@ -1,6 +1,6 @@
 Summary: host sFlow daemon
 Name: hsflowd
-Version: 1.28.1
+Version: 1.28.2
 Release: 1
 License: http://host-sflow.sourceforge.net/license.html
 Group: Applications/Internet

src/Linux/Makefile

@@ -142,7 +142,7 @@ endif
 # /etc/redhat-release to get the version number as in xenserver-ddk/Makefile,  but then it
 # might not work properly for a XenSource build.  Seems safer to just check for the file:
 ifeq ($(XENSTORE_H),)
-	XENSTORE_H=$(shell if ls /usr/include/xenstore.h >/dev/null; then echo "yes"; else echo "no"; fi)
+	XENSTORE_H=$(shell if ls /usr/include/xenstore.h 2>&1 >/dev/null; then echo "yes"; else echo "no"; fi)
 endif
 ifeq ($(XENSTORE_H),yes)
 	CFLAGS += -DHSF_XENSTORE_H

src/Linux/hsflowconfig.c

@@ -311,6 +311,7 @@ extern int debug;
     st->pollingInterval = SFL_DEFAULT_POLLING_INTERVAL;
     st->headerBytes = SFL_DEFAULT_HEADER_SIZE;
     st->ulogGroup = HSP_DEFAULT_ULOG_GROUP;
+    st->nflogGroup = HSP_DEFAULT_NFLOG_GROUP;
     st->jsonPort = HSP_DEFAULT_JSON_PORT;
     st->jsonFIFO = NULL;
     st->xen_update_dominfo = 0;

src/Linux/hsflowd.c

@@ -1469,7 +1469,7 @@ extern "C" {
 
       static char *dockerPS[] = { HSF_DOCKER_CMD, "ps", "-q", NULL };
       char dockerLine[HSF_DOCKER_MAX_LINELEN];
-      if(myExec(sp, dockerPS, dockerContainerCB, dockerLine, HSF_DOCKER_MAX_LINELEN)) {
+      if(myExec(sp, dockerPS, dockerContainerCB, dockerLine, HSF_DOCKER_MAX_LINELEN, NULL)) {
 	// successful, now gather data for each one
 	UTStringArray *dockerInspect = strArrayNew();
 	strArrayAdd(dockerInspect, HSF_DOCKER_CMD);
@@ -1490,7 +1490,8 @@ extern "C" {
 			       strArray(dockerInspect),
 			       dockerInspectCB,
 			       dockerLine,
-			       HSF_DOCKER_MAX_LINELEN);
+			       HSF_DOCKER_MAX_LINELEN,
+			       NULL);
 	strArrayFree(dockerInspect);
 	char *ibuf = UTStrBuf_unwrap(inspectBuf);
 	if(inspectOK) {
@@ -2301,16 +2302,18 @@ extern "C" {
   /*_________________-------------------------------__________________
     _________________   setSwitchPortSamplingRates  __________________
     -----------------_______________________________------------------
+    return YES = hardware/kernel sampling configured OK
+    return NO  = hardware/kernel sampling not set - assume 1:1 on ULOG/NFLOG
   */
 
   static int execOutputLine(void *magic, char *line) {
-    char *prefix = (char *)magic;
-    if(debug) myLog(LOG_INFO, "%s: %s", prefix, line);
+    if(debug) myLog(LOG_INFO, "execOutputLine: %s", line);
     return YES;
   }
 
-  static void setSwitchPortSamplingRates(HSPSFlow *sf, HSPSFlowSettings *settings, uint32_t logGroup)
+  static int setSwitchPortSamplingRates(HSPSFlow *sf, HSPSFlowSettings *settings, uint32_t logGroup)
   {
+    int hw_sampling = YES;
     UTStringArray *cmdline = strArrayNew();
     strArrayAdd(cmdline, HSP_SWITCHPORT_CONFIG_PROG);
     // usage:  <prog> <interface> <ingress-rate> <egress-rate> <logGroup>
@@ -2339,8 +2342,21 @@ extern "C" {
 	    snprintf(srate, HSP_MAX_TOK_LEN, "%u", niostate->sampling_n);
 	    if(settings->samplingDirection & HSF_DIRN_IN) strArrayInsert(cmdline, 2, srate); // ingress
 	    if(settings->samplingDirection & HSF_DIRN_OUT) strArrayInsert(cmdline, 3, srate); // ingress
-	    if(myExec("Switchport config output", strArray(cmdline), execOutputLine, outputLine, HSP_MAX_EXEC_LINELEN)) {
-	      niostate->sampling_n_set = niostate->sampling_n;
+	    int status;
+	    if(myExec(NULL, strArray(cmdline), execOutputLine, outputLine, HSP_MAX_EXEC_LINELEN, &status)) {
+	      if(WEXITSTATUS(status) != 0) {
+
+		myLog(LOG_ERR, "myExec(%s) exitStatus=%d so assuming ULOG/NFLOG is 1:1",
+		      HSP_SWITCHPORT_CONFIG_PROG,
+		      WEXITSTATUS(status));
+
+		hw_sampling = NO;
+		break;
+	      }
+	      else {
+		// hardware or kernel sampling was successfully configured
+		niostate->sampling_n_set = niostate->sampling_n;
+	      }
 	    }
 	    else {
 	      myLog(LOG_ERR, "myExec() calling %s failed (adaptor=%s)",
@@ -2352,6 +2368,7 @@ extern "C" {
       }
     }
     strArrayFree(cmdline);
+    return hw_sampling;
   }
 #endif // HSP_SWITCHPORT_CONFIG
 
@@ -2363,21 +2380,26 @@ extern "C" {
 
   static void setPacketSamplingRates(HSPSFlow *sf, HSPSFlowSettings *settings)
   {
+    // set defaults assuming we will get 1:1 on ULOG or NFLOG and do our own sampling.
+    settings->ulogSubSamplingRate = settings->nflogSubSamplingRate = settings->samplingRate;
+    settings->ulogActualSamplingRate = settings->nflogActualSamplingRate = settings->samplingRate;
+
 #ifdef HSP_SWITCHPORT_CONFIG
     // We get to set the hardware sampling rate here, so do that and then force
     // the ulog settings to reflect it (so that the sub-sampling rate is 1:1)
-    setSwitchPortSamplingRates(sf, settings, sf->sFlowSettings_file->ulogGroup);
+    if(setSwitchPortSamplingRates(sf, settings, sf->sFlowSettings_file->ulogGroup)) {
+      // all sampling is done in the hardware
+      settings->ulogSubSamplingRate = settings->nflogSubSamplingRate = 1;
+      return;
+    }
+
 #endif // HSP_SWITCHPORT_CONFIG
 
     // calculate the ULOG sub-sampling rate to use.  We may get the local ULOG sampling-rate
     // from the probability setting in the config file and the desired sampling rate from DNS-SD,
     // so that's why we have to reconcile the two here.
     uint32_t ulogsr = sf->sFlowSettings_file->ulogSamplingRate;
-    if(ulogsr == 0) {
-      // assume we have to do all sampling in user-space
-      settings->ulogSubSamplingRate = settings->ulogActualSamplingRate = settings->samplingRate;
-    }
-    else {
+    if(ulogsr > 1) {
       // use an integer divide to get the sub-sampling rate, but make sure we round up
       settings->ulogSubSamplingRate = (settings->samplingRate + ulogsr - 1) / ulogsr;
       // and pre-calculate the actual sampling rate that we will end up applying
@@ -2386,11 +2408,7 @@ extern "C" {
 
     // repeat for nflog settings
     uint32_t nflogsr = sf->sFlowSettings_file->nflogSamplingRate;
-    if(nflogsr == 0) {
-      // assume we have to do all sampling in user-space
-      settings->nflogSubSamplingRate = settings->nflogActualSamplingRate = settings->samplingRate;
-    }
-    else {
+    if(nflogsr > 1) {
       // use an integer divide to get the sub-sampling rate, but make sure we round up
       settings->nflogSubSamplingRate = (settings->samplingRate + nflogsr - 1) / nflogsr;
       // and pre-calculate the actual sampling rate that we will end up applying

src/Linux/hsflowd.h

@@ -109,9 +109,11 @@ extern "C" {
 #define HSP_SWITCHPORT_CONFIG_PROG  "/usr/lib/cumulus/portsamp"
 #include "regex.h" // for switchport detection
 #define HSP_SWITCHPORT_REGEX "^swp[0-9s]+$"
-// uses ULOG (netlink) channel, so make sure that is enabled:
+  // uses ULOG (netlink) channel, so make sure that is enabled:
 #define HSF_ULOG 1
 #define HSP_DEFAULT_ULOG_GROUP 1
+  // starting in CL 2.5, uses NFLOG, also defaulting to group==1
+#define HSP_DEFAULT_NFLOG_GROUP 1
 #endif
 
 #if (HSF_ULOG || HSF_NFLOG)
@@ -125,19 +127,18 @@ extern "C" {
 #include <linux/netfilter_ipv4/ipt_ULOG.h>
 #define HSP_MAX_ULOG_MSG_BYTES 10000
 #define HSP_ULOG_RCV_BUF 8000000
-
-#ifndef HSP_DEFAULT_ULOG_GROUP
-#define HSP_DEFAULT_ULOG_GROUP 0
-#endif
-
 #endif /* HSF_ULOG */
 
 #ifdef HSF_NFLOG
-#define HSP_MAX_NFLOG_MSG_BYTES 10000
+/* Set this to 65K+ to make sure we handle the
+   case where virtual port TSOs coallesce packets
+   (ignoring MTU constraints). */
+#define HSP_MAX_NFLOG_MSG_BYTES 65536 + 128
 #define HSP_NFLOG_RCV_BUF 8000000
+
 #include <linux/netfilter/nfnetlink_log.h>
 #include <libnfnetlink.h>
-#endif
+#endif /* HSF_NFLOG */
 
 #ifdef HSF_JSON
 #include "cJSON.h"
@@ -183,6 +184,14 @@ extern "C" {
 #define HSP_DEFAULT_VMSTORE_FILE "/etc/hsflowd.data"
 #define HSP_DEFAULT_CRASH_FILE "/etc/hsflowd.crash"
 
+#ifndef HSP_DEFAULT_ULOG_GROUP
+#define HSP_DEFAULT_ULOG_GROUP 0
+#endif
+#ifndef HSP_DEFAULT_NFLOG_GROUP
+#define HSP_DEFAULT_NFLOG_GROUP 0
+#endif
+#define HSP_DEFAULT_JSON_PORT 0
+
 /* Numbering to avoid clash. See http://www.sflow.org/developers/dsindexnumbers.php */
 #define HSP_DEFAULT_PHYSICAL_DSINDEX 1
 #define HSP_DEFAULT_SUBAGENTID 100000
@@ -288,9 +297,6 @@ extern "C" {
 #define HSP_MAX_HEADER_BYTES 256
     HSPApplicationSettings *applicationSettings;
     uint32_t ulogGroup;
-#ifndef HSP_DEFAULT_ULOG_GROUP
-#define HSP_DEFAULT_ULOG_GROUP 0
-#endif
     double ulogProbability;
     uint32_t ulogSamplingRate;
     uint32_t ulogSubSamplingRate;
@@ -303,9 +309,6 @@ extern "C" {
     uint32_t nflogActualSamplingRate;
 
     uint32_t jsonPort;
-#ifndef HSP_DEFAULT_JSON_PORT
-#define HSP_DEFAULT_JSON_PORT 0
-#endif
     char *jsonFIFO;
     HSPCIDR *agentCIDRs;
   } HSPSFlowSettings;
@@ -438,9 +441,7 @@ extern "C" {
     // and those sending packet-samples will have a sampler.
     SFLSampler *sampler;
     uint32_t sampling_n;
-#ifdef HSP_SWITCHPORT_CONFIG
     uint32_t sampling_n_set;
-#endif
     uint32_t netlink_drops;
   } HSPAdaptorNIO;
 

src/Linux/readBroadcomCounters.c

@@ -7,6 +7,8 @@ extern "C" {
 #endif
 
 #include "hsflowd.h"
+
+  extern int debug;
 
 #ifdef HSF_CUMULUS
   /*_________________---------------------------__________________
@@ -37,6 +39,13 @@ extern "C" {
 
 #define HSF_BCM_FILES "/cumulus/switchd/run/"
 
+    struct stat statBuf;
+    if(stat(HSF_BCM_FILES, &statBuf) == -1) {
+      // don't include this structure at all if none of the data is there,
+      // which happens on "VX" virtual switches.
+      return NO;
+    }
+
     // hosts
     if(readOneIntFile(HSF_BCM_FILES "route_info/host/count", &scratch64)) bcm->bcm_host_entries = scratch64;
     if(readOneIntFile(HSF_BCM_FILES "route_info/host/max", &scratch64)) bcm->bcm_host_entries_max = scratch64;
@@ -45,7 +54,7 @@ extern "C" {
 
     // routing tables
     if(!readOneIntFile(HSF_BCM_FILES "route_info/route/mode", &mode)) {
-      myLog(LOG_ERR, "cannot read route-rable mode");
+      if(debug) myLog(LOG_INFO, "cannot read route-table mode");
     }
     if(mode == 1) {
       // (v4-v6, long-v6)

src/Linux/readPackets.c

@@ -198,7 +198,8 @@ extern "C" {
 #ifdef HSF_CUMULUS
     // On Cumulus Linux the sampling direction is indicated in the low
     // bit of the pkt->hook field: 0==ingress,1==egress
-    if((hook & 1) == 1) {
+    if(ad_out &&
+       (hook & 1) == 1) {
       sampler_dev = ad_out;
     }
 #else
@@ -281,10 +282,13 @@ extern "C" {
 	SFLADD_ELEMENT(&fs, &hdrElem);
 	// submit the actual sampling rate so it goes out with the sFlow feed
 	// otherwise the sampler object would fill in his own (sub-sampling) rate.
-	// If it's a switch port then samplerNIO->sampling_n will be set, so that
+	// If it's a switch port then samplerNIO->sampling_n may be set, so that
 	// takes precendence (allows different ports to have different sampling
 	// settings).
-	uint32_t actualSamplingRate = samplerNIO->sampling_n ?: sampling_n;
+	uint32_t actualSamplingRate = sampling_n;
+	if(samplerNIO->sampling_n_set && samplerNIO->sampling_n) {
+	  actualSamplingRate = samplerNIO->sampling_n;
+	}
 	fs.sampling_rate = actualSamplingRate;
 
 	// estimate the sample pool from the samples.  Could maybe do this
@@ -321,15 +325,10 @@ extern "C" {
       return 0;
     }
 
-#ifdef HSP_SWITCHPORT_CONFIG
-    // assume sampling is always-on and ignore the
-    // subSamplingRate field.
-#else
     if(sp->sFlow->sFlowSettings->ulogSubSamplingRate == 0) {
       // packet sampling was disabled by setting desired rate to 0
       return 0;
     }
-#endif
 
     if(sp->ulog_soc) {
       for( ; batch < HSP_READPACKET_BATCH; batch++) {
@@ -377,11 +376,8 @@ extern "C" {
 
 	      if(--MySkipCount == 0) {
 		/* reached zero. Set the next skip */
-#ifdef HSP_SWITCHPORT_CONFIG
-		MySkipCount = 1;
-#else
-		MySkipCount = sfl_random((2 * sp->sFlow->sFlowSettings->ulogSubSamplingRate) - 1);
-#endif
+		uint32_t sr = sp->sFlow->sFlowSettings->ulogSubSamplingRate;
+		MySkipCount = sr == 1 ? 1 : sfl_random((2 * sr) - 1);
 
 		/* and take a sample */
 
@@ -446,15 +442,10 @@ extern "C" {
     }
 
 
-#ifdef HSP_SWITCHPORT_CONFIG
-    // assume sampling is always-on and ignore the
-    // subSamplingRate field.
-#else
     if(sp->sFlow->sFlowSettings->nflogSubSamplingRate == 0) {
       // packet sampling was disabled by setting desired rate to 0
       return 0;
     }
-#endif
 
     if(sp->nflog_soc) {
       for( ; batch < HSP_READPACKET_BATCH; batch++) {
@@ -463,7 +454,14 @@ extern "C" {
 			    buf,
 			    HSP_MAX_NFLOG_MSG_BYTES);
 	if(len <= 0) break;
-	if(debug > 1) myLog(LOG_INFO, "got NFLOG msg: %u bytes", len);
+	if(debug > 1) {
+	  struct nlmsghdr *msg = (struct nlmsghdr *)buf;
+	  myLog(LOG_INFO, "got NFLOG msg: bytes_read=%u nlmsg_len=%u nlmsg_type=%u OK=%s",
+		len,
+		msg->nlmsg_len,
+		msg->nlmsg_type,
+		NLMSG_OK(msg, len) ? "true" : "false");
+	}
 	for(struct nlmsghdr *msg = (struct nlmsghdr *)buf; NLMSG_OK(msg, len); msg=NLMSG_NEXT(msg, len)) {
 	  if(debug > 1) {
 	    myLog(LOG_INFO, "netlink (%u bytes left) msg [len=%u type=%u flags=0x%x seq=%u pid=%u]",
@@ -506,7 +504,9 @@ extern "C" {
 	      while (NFA_OK(attr, attr_len)) {
 		if (NFA_TYPE(attr) <= NFULA_MAX) {
 		  tb[NFA_TYPE(attr)-1] = attr;
-		  if(debug > 2) myLog(LOG_INFO, "found attr %d\n", NFA_TYPE(attr));
+		  if(debug > 2) {
+		    myLog(LOG_INFO, "found attr %d attr_len=%d\n", NFA_TYPE(attr), attr_len);
+		  }
 		}
                 attr = NFA_NEXT(attr,attr_len);
 	      }
@@ -521,13 +521,14 @@ extern "C" {
 		continue;
 	      }
 
+	      if(debug > 2) {
+		myLog(LOG_INFO, "capture payload (cap_len)=%d\n", cap_len);
+	      }
+
 	      if(--MySkipCount == 0) {
 		/* reached zero. Set the next skip */
-#ifdef HSP_SWITCHPORT_CONFIG
-		MySkipCount = 1;
-#else
-		MySkipCount = sfl_random((2 * sp->sFlow->sFlowSettings->nflogSubSamplingRate) - 1);
-#endif
+		uint32_t sr = sp->sFlow->sFlowSettings->nflogSubSamplingRate;
+		MySkipCount = sr == 1 ? 1 : sfl_random((2 * sr) - 1);
 
 		/* and take a sample */
 		char *prefix = nfnl_get_pointer_to_data(tb, NFULA_PREFIX, char);

src/Linux/scripts/hsflowd.conf

@@ -100,7 +100,7 @@ sflow {
   #
   # Then the settings here should be:
   # nflogGroup = 5
-  # ulogProbability = 0.0025
+  # nflogProbability = 0.0025
   #
   # Or, if your OS is older and only suppports ULOG:
   #