Added new API calls for implementing Bloom-filter like data structures

ndpi_filter* ndpi_filter_alloc(uint32_t elements_number);
bool         ndpi_filter_add(ndpi_filter *f, uint64_t value);
bool         ndpi_filter_contains(ndpi_filter *f, uint64_t value);
void         ndpi_filter_free(ndpi_filter *f);

example/ndpiReader.c

@@ -5323,6 +5323,24 @@ void compressedBitmapUnitTest() {
 
 /* *********************************************** */
 
+void filterUnitTest() {
+  ndpi_filter* f = ndpi_filter_alloc(10000);
+  u_int32_t v, i;
+
+  assert(f);
+
+  srand(time(NULL));
+
+  for(i=0; i<1000; i++)
+    assert(ndpi_filter_add(f, v = rand()));
+
+  assert(ndpi_filter_contains(f, v));
+
+  ndpi_filter_free(f);
+}
+
+/* *********************************************** */
+
 void zscoreUnitTest() {
   u_int32_t values[] = { 1, 3, 3, 4, 5, 2, 6, 7, 30, 16 };
   u_int32_t i;
@@ -5455,6 +5473,7 @@ int main(int argc, char **argv) {
     dgaUnitTest();
     hllUnitTest();
     bitmapUnitTest();
+    filterUnitTest();
     automataUnitTest();
     analyzeUnitTest();
     ndpi_self_check_host_match(stderr);

src/include/ndpi_api.h

@@ -1965,6 +1965,8 @@ extern "C" {
 
   /* ******************************* */
 
+  /* Based on https://roaringbitmap.org */
+
   ndpi_bitmap* ndpi_bitmap_alloc(void);
   void ndpi_bitmap_free(ndpi_bitmap* b);
   ndpi_bitmap* ndpi_bitmap_copy(ndpi_bitmap* b);
@@ -1985,6 +1987,18 @@ extern "C" {
   void ndpi_bitmap_iterator_free(ndpi_bitmap* b);
   bool ndpi_bitmap_iterator_next(ndpi_bitmap_iterator* i, u_int32_t *value);
 
+  /* ******************************* */
+  /*
+    Bloom-filter on steroids
+    
+    Based on https://github.com/FastFilter/xor_singleheader
+  */
+
+  ndpi_filter* ndpi_filter_alloc(uint32_t elements_number);
+  bool         ndpi_filter_add(ndpi_filter *f, uint64_t value); /* returns true on success, false on failure */
+  bool         ndpi_filter_contains(ndpi_filter *f, uint64_t value); /* returns true on success, false on failure */
+  void         ndpi_filter_free(ndpi_filter *f);
+
   /* ******************************* */
 
   char* ndpi_get_flow_risk_info(struct ndpi_flow_struct *flow,

src/include/ndpi_typedefs.h

@@ -2019,6 +2019,7 @@ typedef int (*ndpi_custom_dga_predict_fctn)(const char* domain, int domain_lengt
 
 typedef void ndpi_bitmap;
 typedef void ndpi_bitmap_iterator;
+typedef void ndpi_filter;
 
 /* **************************************** */
 

src/lib/ndpi_bitmap.c

@@ -1,5 +1,5 @@
 /*
- * ndpi_utils.c
+ * ndpi_bitmap.c
  *
  * Copyright (C) 2011-23 - ntop.org and contributors
  *

src/lib/ndpi_filter.c

@@ -0,0 +1,88 @@
+/*
+ * ndpi_filter.c
+ *
+ * Copyright (C) 2011-23 - ntop.org and contributors
+ *
+ * This file is part of nDPI, an open source deep packet inspection
+ * library based on the OpenDPI and PACE technology by ipoque GmbH
+ *
+ * nDPI is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * nDPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with nDPI.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+#include <stdlib.h>
+#include <errno.h>
+#include <math.h>
+#include <sys/types.h>
+
+
+#define NDPI_CURRENT_PROTO NDPI_PROTOCOL_UNKNOWN
+
+#include "ndpi_config.h"
+#include "ndpi_api.h"
+#include "ndpi_includes.h"
+#include "ndpi_encryption.h"
+
+#include "third_party/include/binaryfusefilter.h"
+
+/* ******************************************* */
+
+ndpi_filter* ndpi_filter_alloc(uint32_t elements_number) {
+  binary_fuse8_t *filter = (binary_fuse8_t*)ndpi_malloc(sizeof(binary_fuse8_t));
+
+  if(filter == NULL) return(NULL);
+
+  if(!binary_fuse8_allocate(elements_number, filter)) {
+    ndpi_free(filter);
+    return(NULL);
+  } else
+    return((ndpi_filter*)filter);
+}
+
+/* ******************************************* */
+
+bool ndpi_filter_add(ndpi_filter *f, uint64_t value) {
+  if(!f)
+    return(false);
+  else {
+    binary_fuse8_t *filter = (binary_fuse8_t*)f;
+
+    return(binary_fuse8_populate(&value, 1, filter));
+  }
+}
+
+/* ******************************************* */
+
+bool ndpi_filter_contains(ndpi_filter *f, uint64_t value) {
+  if(!f)
+    return(false);
+  else {
+    binary_fuse8_t *filter = (binary_fuse8_t*)f;
+
+    return(binary_fuse8_contain(value, filter));
+  }
+}
+
+/* ******************************************* */
+
+void ndpi_filter_free(ndpi_filter *f) {
+  if(f != NULL) {
+    binary_fuse8_t *filter = (binary_fuse8_t*)f;
+
+    binary_fuse8_free(filter);
+    ndpi_free(filter);
+  }
+}
+