Skip to content

Commit

Permalink
Add infrastructure for explicit support of Fist Packet Classification
Browse files Browse the repository at this point in the history 
Let's start with some basic helpers and with FPC based on flow addresses.

See: #2322
  • Loading branch information
@IvanNardi
IvanNardi committed Jul 3, 2024
1 parent e566133 commit cb6817b
Show file tree
Hide file tree
Showing 561 changed files with 10,322 additions and 10,216 deletions.
18 changes: 18 additions & 0 deletions example/ndpiReader.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1747,6 +1747,24 @@ static void printFlow(u_int32_t id, struct ndpi_flow_info *flow, u_int16_t threa
flow->detected_protocol) ? "Encrypted" : "ClearText");

fprintf(out, "[Confidence: %s]", ndpi_confidence_get_name(flow->confidence));

if(flow->fpc.master_protocol == NDPI_PROTOCOL_UNKNOWN) {
fprintf(out, "[FPC: %u/%s, ",
flow->fpc.app_protocol,
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.app_protocol));
} else {
fprintf(out, "[FPC: %u.%u/%s.%s, ",
flow->fpc.master_protocol,
flow->fpc.app_protocol,
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.master_protocol),
ndpi_get_proto_name(ndpi_thread_info[thread_id].workflow->ndpi_struct,
flow->fpc.app_protocol));
}
fprintf(out, "Confidence: %s]",
ndpi_fpc_confidence_get_name(flow->fpc.confidence));

/* If someone wants to have the num_dissector_calls variable per flow, he can print it here.
Disabled by default to avoid too many diffs in the unit tests...
*/
Expand Down
3 changes: 3 additions & 0 deletions example/reader_util.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1055,6 +1055,9 @@ void process_ndpi_collected_info(struct ndpi_workflow * workflow, struct ndpi_fl
flow->risk_str = ndpi_strdup(s);

flow->confidence = flow->ndpi_flow->confidence;

flow->fpc = flow->ndpi_flow->fpc;

flow->num_dissector_calls = flow->ndpi_flow->num_dissector_calls;

ndpi_snprintf(flow->host_server_name, sizeof(flow->host_server_name), "%s",
Expand Down
1 change: 1 addition & 0 deletions example/reader_util.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,7 @@ typedef struct ndpi_flow_info {
// result only, not used for flow identification
ndpi_protocol detected_protocol;
ndpi_confidence_t confidence;
struct ndpi_fpc_info fpc;
u_int16_t num_dissector_calls;
u_int16_t dpi_packets;

Expand Down
1 change: 1 addition & 0 deletions fuzz/fuzz_config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
ndpi_risk2score(static_cast<ndpi_risk_enum>(fuzzed_data.ConsumeIntegral<u_int64_t>()), &unused1, &unused2);
ndpi_http_method2str(static_cast<ndpi_http_method>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_confidence_get_name(static_cast<ndpi_confidence_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_fpc_confidence_get_name(static_cast<ndpi_fpc_confidence_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_get_proto_breed_name(static_cast<ndpi_protocol_breed_t>(fuzzed_data.ConsumeIntegral<u_int8_t>()));
ndpi_get_l4_proto_name(static_cast<ndpi_l4_proto_info>(fuzzed_data.ConsumeIntegral<u_int8_t>()));

Expand Down
9 changes: 9 additions & 0 deletions src/include/ndpi_api.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -627,6 +627,15 @@ extern "C" {
*/
const char* ndpi_confidence_get_name(ndpi_confidence_t confidence);

/**
* Get FPC confidence as string
*
* @par confidence = the confidence value
* @return the string name of the confidence result
*
*/
const char* ndpi_fpc_confidence_get_name(ndpi_fpc_confidence_t fpc_confidence);

/**
* Set protocol category string
*
Expand Down
30 changes: 28 additions & 2 deletions src/include/ndpi_typedefs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,23 @@ typedef enum {
NDPI_CONFIDENCE_MAX,
} ndpi_confidence_t;

typedef enum {
NDPI_FPC_CONFIDENCE_UNKNOWN = 0, /* Unknown First Packet Classification */
NDPI_FPC_CONFIDENCE_IP, /* FPC based on IP address */
NDPI_FPC_CONFIDENCE_DNS, /* FPC based on DNS information */

/*
IMPORTANT
Please keep in sync with
ndpi_fpc_confidence_get_name()
in ndpi_main.c
*/

/* Last one */
NDPI_FPC_CONFIDENCE_MAX,
} ndpi_fpc_confidence_t;

typedef enum {
NDPI_PROTOCOL_SAFE = 0, /* Surely doesn't provide risks for the network. (e.g., a news site) */
NDPI_PROTOCOL_ACCEPTABLE, /* Probably doesn't provide risks, but could be malicious (e.g., Dropbox) */
Expand Down Expand Up @@ -1119,6 +1136,12 @@ typedef struct _ndpi_automa {

typedef void ndpi_str_hash;

struct ndpi_fpc_info {
u_int16_t master_protocol;
u_int16_t app_protocol;
ndpi_fpc_confidence_t confidence;
};

typedef struct ndpi_proto {
/*
Note
Expand Down Expand Up @@ -1195,6 +1218,9 @@ struct ndpi_flow_struct {
u_int16_t num_dissector_calls;
ndpi_confidence_t confidence; /* ndpi_confidence_t */

/* First Packet Classification info */
struct ndpi_fpc_info fpc;

/*
if ndpi_struct->direction_detect_disable == 1
tcp sequence number connection tracking
Expand Down Expand Up @@ -1510,8 +1536,8 @@ struct ndpi_flow_struct {
_Static_assert(sizeof(((struct ndpi_flow_struct *)0)->protos) <= 256,
"Size of the struct member protocols increased to more than 256 bytes, "
"please check if this change is necessary.");
_Static_assert(sizeof(struct ndpi_flow_struct) <= 1112,
"Size of the flow struct increased to more than 1112 bytes, "
_Static_assert(sizeof(struct ndpi_flow_struct) <= 1120,
"Size of the flow struct increased to more than 1120 bytes, "
"please check if this change is necessary.");
#endif
#endif
Expand Down
48 changes: 48 additions & 0 deletions src/lib/ndpi_main.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8287,6 +8287,33 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n

/* ********************************************************************************* */

static void fpc_update(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
u_int16_t fpc_master, u_int16_t fpc_app,
ndpi_fpc_confidence_t fpc_confidence)
{
NDPI_LOG_DBG(ndpi_str, "FPC %d.%d/%s -> %d.%d/%s\n",
flow->fpc.master_protocol, flow->fpc.app_protocol,
ndpi_fpc_confidence_get_name(flow->fpc.confidence),
fpc_master, fpc_app,
ndpi_fpc_confidence_get_name(fpc_confidence));
flow->fpc.master_protocol = fpc_master;
flow->fpc.app_protocol = fpc_app;
flow->fpc.confidence = fpc_confidence;
}

/* ********************************************************************************* */

static void fpc_check_ip(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow)
{
if(flow->guessed_protocol_id_by_ip != NDPI_PROTOCOL_UNKNOWN)
fpc_update(ndpi_str, flow, NDPI_PROTOCOL_UNKNOWN,
flow->guessed_protocol_id_by_ip, NDPI_FPC_CONFIDENCE_IP);
}

/* ********************************************************************************* */

static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detection_module_struct *ndpi_str,
struct ndpi_flow_struct *flow,
const unsigned char *packet_data,
Expand Down Expand Up @@ -8421,6 +8448,8 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio

if(ndpi_do_guess(ndpi_str, flow, &ret) == -1)
return(ret);

fpc_check_ip(ndpi_str, flow);
}

num_calls = ndpi_check_flow_func(ndpi_str, flow, &ndpi_selection_packet);
Expand Down Expand Up @@ -9383,6 +9412,25 @@ const char *ndpi_confidence_get_name(ndpi_confidence_t confidence)

/* ****************************************************** */

const char *ndpi_fpc_confidence_get_name(ndpi_fpc_confidence_t fpc_confidence)
{
switch(fpc_confidence) {
case NDPI_FPC_CONFIDENCE_UNKNOWN:
return "Unknown";

case NDPI_FPC_CONFIDENCE_IP:
return "IP address";

case NDPI_FPC_CONFIDENCE_DNS:
return "DNS";

default:
return "Invalid"; /* Out of sync with ndpi_fpc_confidence_t definition */
}
}

/* ****************************************************** */

const char *ndpi_category_get_name(struct ndpi_detection_module_struct *ndpi_str,
ndpi_protocol_category_t category) {
if((!ndpi_str) || (category >= NDPI_PROTOCOL_NUM_CATEGORIES)) {
Expand Down
12 changes: 6 additions & 6 deletions tests/cfgs/caches_cfg/result/ookla.pcap.out
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ JA3 Host Stats:
1 192.168.1.128 2


1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Risk Info: Expected on port 443][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA4: t13d1714h2_5b57614c22b0_8f66f9ee9c6c][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA4: t13d1715h2_5b57614c22b0_3d5424432f57][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][Hostname/SNI: massarosa-1.speedtest.welcomeitalia.it][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][URL: massarosa-1.speedtest.welcomeitalia.it/crossdomain.xml][StatusCode: 200][Content-Type: application/xml][Server: Apache/2.2.22 (Ubuntu)][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8][Risk: ** HTTP Obsolete Server **][Risk Score: 50][Risk Info: Obsolete Apache server 2.2.22][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][DPI packets: 10][cat: Web/5][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][Risk: ** Fully encrypted flow **][Risk Score: 50][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
1 TCP 192.168.1.128:35830 <-> 89.96.108.170:8080 [proto: 91/TLS][IP: 0/Unknown][Encrypted][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Web/5][21 pkts/21216 bytes <-> 8 pkts/1950 bytes][Goodput ratio: 93/72][0.32 sec][Hostname/SNI: spd-pub-mi-01-01.fastwebnet.it][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: 0.832 (Upload)][IAT c2s/s2c min/avg/max/stddev: 0/0 17/61 274/280 62/109][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 1010/244 1514/387 612/138][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Risk Info: Expected on port 443][TLSv1.3][JA3C: c279b0189edb9269da7bc43dea5e0c36][JA4: t13d1714h2_5b57614c22b0_8f66f9ee9c6c][JA3S: fcb2d4d0991292272fcb1e464eedfd43][Firefox][Cipher: TLS_AES_128_GCM_SHA256][Plen Bins: 0,0,4,0,0,0,0,4,9,0,9,0,0,0,0,0,4,0,0,13,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0]
2 TCP 192.168.1.128:48854 <-> 104.16.209.12:443 [proto: 91.191/TLS.Ookla][IP: 220/Cloudflare][Encrypted][Confidence: DPI][FPC: 220/Cloudflare, Confidence: IP address][DPI packets: 6][cat: Network/14][8 pkts/1620 bytes <-> 6 pkts/3818 bytes][Goodput ratio: 67/89][0.06 sec][Hostname/SNI: www.speedtest.net][(Advertised) ALPNs: h2;http/1.1][TLS Supported Versions: TLSv1.3;TLSv1.2][bytes ratio: -0.404 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/5 18/15 7/6][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 202/636 583/1514 181/646][TLSv1.3][JA3C: 579ccef312d18482fc42e2b822ca2430][JA4: t13d1715h2_5b57614c22b0_3d5424432f57][JA3S: eb1d94daa7e0344597e756a1fb6e7054][Firefox][Cipher: TLS_AES_128_GCM_SHA256][PLAIN TEXT (@oTAgOeedtest.net)][Plen Bins: 0,0,14,0,0,14,0,0,0,0,14,0,0,0,0,0,28,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,28,0,0]
3 TCP 192.168.1.7:51207 <-> 46.44.253.187:80 [proto: 7.191/HTTP.Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][12 pkts/2238 bytes <-> 8 pkts/2082 bytes][Goodput ratio: 64/74][5.33 sec][Hostname/SNI: massarosa-1.speedtest.welcomeitalia.it][bytes ratio: 0.036 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/4 528/47 5005/84 1493/28][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 186/260 430/523 168/194][URL: massarosa-1.speedtest.welcomeitalia.it/crossdomain.xml][StatusCode: 200][Content-Type: application/xml][Server: Apache/2.2.22 (Ubuntu)][User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/602.4.8 (KHTML, like Gecko) Version/10.0.3 Safari/602.4.8][Risk: ** HTTP Obsolete Server **][Risk Score: 50][Risk Info: Obsolete Apache server 2.2.22][PLAIN TEXT (GET /crossdomain.xml HTTP/1.1)][Plen Bins: 0,0,0,0,0,0,0,0,0,0,12,75,0,0,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
4 TCP 192.168.1.192:51156 <-> 89.96.108.170:8080 [proto: 131/HTTP_Proxy][IP: 0/Unknown][ClearText][Confidence: Match by port][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 10][cat: Web/5][6 pkts/591 bytes <-> 4 pkts/1784 bytes][Goodput ratio: 32/85][0.05 sec][bytes ratio: -0.502 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 9/10 15/20 6/8][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 98/446 143/1514 31/617][Risk: ** Fully encrypted flow **][Risk Score: 50][PLAIN TEXT (gKRZvA)][Plen Bins: 0,40,40,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,0,0]
5 TCP 192.168.1.7:51215 <-> 46.44.253.187:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][19 pkts/1421 bytes <-> 11 pkts/920 bytes][Goodput ratio: 11/20][0.80 sec][bytes ratio: 0.214 (Upload)][IAT c2s/s2c min/avg/max/stddev: 26/0 44/75 103/137 23/41][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 75/84 85/100 9/8][PLAIN TEXT ( 6HELLO 2.4 2016)][Plen Bins: 94,5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
6 TCP 192.168.1.192:37790 <-> 185.157.229.246:8080 [proto: 191/Ookla][IP: 0/Unknown][ClearText][Confidence: DPI][FPC: 0/Unknown, Confidence: Unknown][DPI packets: 6][cat: Network/14][6 pkts/454 bytes <-> 4 pkts/317 bytes][Goodput ratio: 11/14][0.06 sec][bytes ratio: 0.178 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 12/5 46/9 17/4][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 76/79 106/108 14/17][PLAIN TEXT (HELLO 2.9 )][Plen Bins: 50,50,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
Loading

0 comments on commit cb6817b

Please sign in to comment.