mod_pcap: new config option to select devices by interface speed.

Instead of a systemd snowflake like: pcap { dev = ens33 } pcap { dev = en17736 } We can just write: pcap { speed = 1G-1T } and it will match all physical NICs with speeds in that range. The range can be partially specified: 1G-1T => speed >= 1G && speed <= 1T 1G- => speed >= 1G 1G => speed == 1G 1- => speed > 0 The aim is to make it easier to end up with a configuration file that will work for all hosts, such as: sflow { agent.cidr=10.0.0.0/8 DNS-SD {} pcap { speed = 1G- } tcp {} }
sflow · Oct 19, 2016 · 0b50b70 · 0b50b70
1 parent b7ac80f
commit 0b50b70
Show file tree

Hide file tree

Showing 8 changed files with 262 additions and 51 deletions.
diff --git a/hsflowd.spec b/hsflowd.spec
@@ -1,7 +1,7 @@
 Summary: host sFlow daemon
 Name: hsflowd
 Version: 2.0.5
-Release: 5
+Release: 6
 License: http://sflow.net/license.html
 Group: Applications/Internet
 URL: http://sflow.net

diff --git a/src/Linux/hsflowconfig.c b/src/Linux/hsflowconfig.c
@@ -135,7 +135,7 @@ extern "C" {
 
   // expectInteger32
 
-  static uint32_t getMultiplier(char *str)
+  static uint32_t getMultiplier32(char *str)
   {
     uint32_t mult = 1;
     uint32_t len = my_strlen(str);
@@ -159,7 +159,7 @@ extern "C" {
       return NULL;
     }
     char *str = my_strdup(t->str); // take a copy so we can modify it
-    uint32_t mult = getMultiplier(str);
+    uint32_t mult = getMultiplier32(str);
     *arg = (mult * strtol(str, NULL, 0));
     my_free(str);
     if(*arg < minVal || *arg > maxVal) {
@@ -169,6 +169,89 @@ extern "C" {
     return t;
   }
 
+
+  static uint64_t getMultiplier64(char *str)
+  {
+    uint64_t mult = 1;
+    uint32_t len = my_strlen(str);
+    char last = toupper(str[len - 1]);
+    if(last == 'K' || last == 'M' || last == 'G' || last == 'T' || last == 'P') {
+      // number of the form "100M" or "1G"
+      str[len - 1] = '\0'; // blat the K, M, G, T or P
+      if(last == 'K') mult = 1000LL;
+      if(last == 'M') mult = 1000000LL;
+      if(last == 'G') mult = 1000000000LL;
+      if(last == 'T') mult = 1000000000000LL;
+      if(last == 'P') mult = 1000000000000000LL;
+    }
+    return mult;
+  }
+
+#if 0  // expectInteger64 not needed yet
+
+  static HSPToken *expectInteger64(HSP *sp, HSPToken *tok, uint64_t *arg, uint64_t minVal, uint64_t maxVal)
+  {
+    HSPToken *t = tok;
+    t = t->nxt;
+    if(t == NULL || !isdigit(t->str[0])) {
+      parseError(sp, tok, "expected integer", "");
+      return NULL;
+    }
+    char *str = my_strdup(t->str); // take a copy so we can modify it
+    uint64_t mult = getMultiplier64(str);
+    *arg = (mult * strtoll(str, NULL, 0));
+    my_free(str);
+    if(*arg < minVal || *arg > maxVal) {
+      parseError(sp, tok, "range error", "");
+      return NULL;
+    }
+    return t;
+  }
+
+#endif // expectInteger64
+
+  static HSPToken *expectIntegerRange64(HSP *sp, HSPToken *tok, uint64_t *arg1, uint64_t *arg2, uint64_t minVal, uint64_t maxVal)
+  {
+    HSPToken *t = tok;
+    t = t->nxt;
+    if(t == NULL || !isdigit(t->str[0])) {
+      parseError(sp, tok, "expected integer", "");
+      return NULL;
+    }
+    char *str = my_strdup(t->str); // take a copy so we can modify it
+    int len = my_strlen(str);
+    int len1 = strcspn(str, "-");
+    str[len1] = '\0';
+    uint64_t mult1 = getMultiplier64(str);
+    *arg1 = (mult1 * strtoll(str, NULL, 0));
+    if(*arg1 < minVal || *arg1 > maxVal) {
+      parseError(sp, tok, "range error", "");
+      return NULL;
+    }
+    if(len > len1) {
+      // we have at least a trailing '-' such as "1G-"
+      char *str2 = str + len1 + 1;
+      if(my_strlen(str2) == 0) {
+	// trailing dash. Allow that to mean "<max>"
+	*arg2 = maxVal;
+      }
+      else {
+	uint64_t mult2 = getMultiplier64(str2);
+	*arg2 = (mult2 * strtoll(str2, NULL, 0));
+	if(*arg2 < minVal || *arg2 > maxVal) {
+	  parseError(sp, tok, "range error", "");
+	  return NULL;
+	}
+      }
+    }
+    else {
+      // no second number - indicate by setting arg2 to 0
+      *arg2 = 0;
+    }
+    my_free(str);
+    return t;
+  }
+
   // expectDouble
   static HSPToken *expectDouble(HSP *sp, HSPToken *tok, double *arg, double minVal, double maxVal)
   {
@@ -1199,6 +1282,10 @@ extern "C" {
 	      if((tok = expectONOFF(sp, tok, &pc->vport)) == NULL) return NO;
 	      pc->vport_set = YES;
 	      break;
+	    case HSPTOKEN_SPEED:
+	      if((tok = expectIntegerRange64(sp, tok, &pc->speed_min, &pc->speed_max, 0, LLONG_MAX)) == NULL) return NO;
+	      pc->speed_set = YES;
+	      break;
 	    default:
 	      unexpectedToken(sp, tok, level[depth]);
 	      return NO;

diff --git a/src/Linux/hsflowd.c b/src/Linux/hsflowd.c
@@ -16,6 +16,7 @@ extern "C" {
   FILE *f_crash = NULL;
 
   static void installSFlowSettings(HSP *sp, HSPSFlowSettings *settings);
+  static bool updatePollingInterval(HSP *sp);
 
   /*_________________---------------------------__________________
     _________________     agent callbacks       __________________
@@ -916,6 +917,33 @@ extern "C" {
     return UTStrBuf_unwrap(buf);
   }
 
+  /*_________________---------------------------__________________
+    _________________   pre_config_first        __________________
+    -----------------___________________________------------------
+  */
+
+  static void pre_config_first(HSP *sp) {
+    // make sure we are ready for someone to call getSampler/getPoller
+    updatePollingInterval(sp);
+
+    // before we do anything else,  read the interfaces again - this time with a full discovery
+    // so that modules can weigh in if required,  and, for example, sampling-rates can be set
+    // correctly.
+    readInterfaces(sp, YES, NULL, NULL, NULL, NULL, NULL);
+
+    // print some stats to help us size HSP_RLIMIT_MEMLOCK etc.
+    if(debug(1))
+      malloc_stats();
+
+    // add a <physicalEntity> poller to represent the whole physical host
+    SFLDataSource_instance dsi;
+    // ds_class = <physicalEntity>, ds_index = <my physical>, ds_instance = 0
+    SFL_DS_SET(dsi, SFL_DSCLASS_PHYSICAL_ENTITY, HSP_DEFAULT_PHYSICAL_DSINDEX, 0);
+    sp->poller = sfl_agent_addPoller(sp->agent, &dsi, sp, agentCB_getCounters_request);
+    sfl_poller_set_sFlowCpInterval(sp->poller, sp->actualPollingInterval);
+    sfl_poller_set_sFlowCpReceiver(sp->poller, HSP_SFLOW_RECEIVER_INDEX);
+  }
+
   /*_________________---------------------------__________________
     _________________   installSFlowSettings    __________________
     -----------------___________________________------------------
@@ -945,7 +973,14 @@ extern "C" {
       sp->sFlowSettings = settings;
 
       // announce the change
-      if(firstConfig) EVEventTxAll(sp->rootModule, HSPEVENT_CONFIG_FIRST, NULL, 0);
+      if(firstConfig) {
+	// make sure certain things are in place before we proceed. This
+	// could be done with an event such as CONFIG_PRE, but then
+	// we would have to handshake before raising CONFIG_FIRST
+	pre_config_first(sp);
+	// now offer it to the modules
+	EVEventTxAll(sp->rootModule, HSPEVENT_CONFIG_FIRST, NULL, 0);
+      }
       EVEventTxAll(sp->rootModule, HSPEVENT_CONFIG_CHANGED, NULL, 0);
       // delay the config-done event until every thread has processed the
       // config change.  This is especially important the first time because
@@ -1131,26 +1166,15 @@ extern "C" {
       if(!sp->DNSSD.DNSSD)
 	abort();
     }
+  }
 
-    // make sure we are ready for someone to call getSampler/getPoller
-    updatePollingInterval(sp);
-
-    // before we do anything else,  read the interfaces again - this time with a full discovery
-    // so that modules can weigh in if required,  and, for example, sampling-rates can be set
-    // correctly.
-    readInterfaces(sp, YES, NULL, NULL, NULL, NULL, NULL);
-
-    // print some stats to help us size HSP_RLIMIT_MEMLOCK etc.
-    if(debug(1))
-      malloc_stats();
+  /*_________________---------------------------__________________
+    _________________     evt_config_changed    __________________
+    -----------------___________________________------------------
+  */
 
-    // add a <physicalEntity> poller to represent the whole physical host
-    SFLDataSource_instance dsi;
-    // ds_class = <physicalEntity>, ds_index = <my physical>, ds_instance = 0
-    SFL_DS_SET(dsi, SFL_DSCLASS_PHYSICAL_ENTITY, HSP_DEFAULT_PHYSICAL_DSINDEX, 0);
-    sp->poller = sfl_agent_addPoller(sp->agent, &dsi, sp, agentCB_getCounters_request);
-    sfl_poller_set_sFlowCpInterval(sp->poller, sp->actualPollingInterval);
-    sfl_poller_set_sFlowCpReceiver(sp->poller, HSP_SFLOW_RECEIVER_INDEX);
+  static void evt_config_changed(EVMod *mod, EVEvent *evt, void *data, size_t dataLen) {
+    myDebug(1, "main: evt_config_changed()");
   }
 
   /*_________________---------------------------__________________
@@ -1550,6 +1574,33 @@ extern "C" {
     // initialize event bus
     sp->rootModule = EVInit(sp);
 
+    // convenience ptr to the poll-bus
+    sp->pollBus = EVGetBus(sp->rootModule, HSPBUS_POLL, YES);
+
+    // register for events that we are going to handle here in the main pollBus thread.  The
+    // events that form the config sequence are requested here before the modules are loaded
+    // so that these functions are called first for each event. For example, a module callback
+    // for HSPEVENT_CONFIG_FIRST will be called after evt_config_first() here,  but before
+    // evt_config_done().
+
+    // Events to feed lines of configuration in one line at a time
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_START), evt_config_start);
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_LINE), evt_config_line);
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_END), evt_config_end);
+
+    // An event that is called once,  after the config is settled and
+    // interfaces have been fully discovered, but before privileges are dropped
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_FIRST), evt_config_first);
+
+    // An event that is called for every config change.
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_CHANGED), evt_config_changed);
+
+    // A handshake event for sync across threads - to make sure CONFIG_FIRST and CONFIG_CHANGED
+    // have been processed to completion by all threads (all buses) before CONFIG_DONE is sent.
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_SHAKE), evt_config_shake);
+    // CONFIG_DONE is where privileges are dropped (the first time).
+    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_DONE), evt_config_done);
+
     // load modules (except DNSSD - loaded below).
     // The module init functions can assume that the
     // config is loaded,  but they can't assume anything
@@ -1581,17 +1632,6 @@ extern "C" {
     if(sp->os10.os10)
       EVLoadModule(sp->rootModule, "mod_os10", sp->modulesPath);
 
-    // convenience ptr to the poll-bus
-    sp->pollBus = EVGetBus(sp->rootModule, HSPBUS_POLL, YES);
-
-    // register for events that we are going to handle here in the main pollBus thread
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_START), evt_config_start);
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_LINE), evt_config_line);
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_END), evt_config_end);
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_FIRST), evt_config_first);
-    // EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_CHANGED), evt_config_changed);
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_SHAKE), evt_config_shake);
-    EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, HSPEVENT_CONFIG_DONE), evt_config_done);
     EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, EVEVENT_TICK), evt_poll_tick);
     EVEventRx(sp->rootModule, EVGetEvent(sp->pollBus, EVEVENT_TOCK), evt_poll_tock);
 

diff --git a/src/Linux/hsflowd.h b/src/Linux/hsflowd.h
@@ -32,7 +32,7 @@ extern "C" {
 #include <pwd.h> // for getpwnam()
 #include <grp.h>
 #include <sys/resource.h> // for setrlimit()
-#include <limits.h> // for UINT_MAX
+#include <limits.h> // for UINT_MAX, LLONG_MAX
 
 #if defined(__GLIBC__) || defined(__UCLIBC__)
 // for signal backtrace, if supported by libc
@@ -138,6 +138,9 @@ extern "C" {
     bool promisc;
     bool vport;
     bool vport_set;
+    uint64_t speed_min;
+    uint64_t speed_max;
+    bool speed_set;
   } HSPPcap;
 
   typedef struct _HSPCIDR {
@@ -535,6 +538,7 @@ extern "C" {
 
   // read functions
   int readInterfaces(HSP *sp, bool full_discovery, uint32_t *p_added, uint32_t *p_removed, uint32_t *p_cameup, uint32_t *p_wentdown, uint32_t *p_changed);
+  const char *devTypeName(EnumHSPDevType devType);
   int readCpuCounters(SFLHost_cpu_counters *cpu);
   int readMemoryCounters(SFLHost_mem_counters *mem);
   int readDiskCounters(HSP *sp, SFLHost_dsk_counters *dsk);

diff --git a/src/Linux/hsflowtokens.h b/src/Linux/hsflowtokens.h
@@ -39,6 +39,7 @@ HSPTOKEN_DATA( HSPTOKEN_SAMPLINGDIRECTION, "samplingDirection", HSPTOKENTYPE_ATT
 HSPTOKEN_DATA( HSPTOKEN_FORGET_VMS, "forgetVMs", HSPTOKENTYPE_ATTRIB, NULL)
 HSPTOKEN_DATA( HSPTOKEN_PCAP, "pcap", HSPTOKENTYPE_OBJ, NULL)
 HSPTOKEN_DATA( HSPTOKEN_DEV, "dev", HSPTOKENTYPE_ATTRIB, NULL)
+HSPTOKEN_DATA( HSPTOKEN_SPEED, "speed", HSPTOKENTYPE_ATTRIB, NULL)
 HSPTOKEN_DATA( HSPTOKEN_PROMISC, "promisc", HSPTOKENTYPE_ATTRIB, NULL)
 HSPTOKEN_DATA( HSPTOKEN_VPORT, "vport", HSPTOKENTYPE_ATTRIB, NULL)
 HSPTOKEN_DATA( HSPTOKEN_KVM, "kvm", HSPTOKENTYPE_OBJ, NULL)