From 694cd2bc17f25bc95e2b61939636eeee1a6fff31 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" <yuri.lipnesh@datadoghq.com> Date: Tue, 14 Jan 2025 14:42:44 -0500 Subject: [PATCH 1/5] [usm] add continuation_frames counter to ebpf.http2_telemetry_t --- pkg/network/ebpf/c/protocols/http2/decoding-defs.h | 3 ++- pkg/network/ebpf/c/protocols/http2/decoding.h | 8 ++++++-- pkg/network/protocols/http2/model_linux.go | 7 ++++--- pkg/network/protocols/http2/telemetry.go | 5 +++++ pkg/network/protocols/http2/types_linux.go | 1 + 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/network/ebpf/c/protocols/http2/decoding-defs.h b/pkg/network/ebpf/c/protocols/http2/decoding-defs.h index 5b55f8e0d216f..2a861b9829f70 100644 --- a/pkg/network/ebpf/c/protocols/http2/decoding-defs.h +++ b/pkg/network/ebpf/c/protocols/http2/decoding-defs.h @@ -226,8 +226,8 @@ typedef struct { // literal_value_exceeds_frame Count of times we couldn't retrieve the literal value due to reaching the end of the frame. // exceeding_max_interesting_frames Count of times we reached the max number of frames per iteration. // exceeding_max_frames_to_filter Count of times we have left with more frames to filter than the max number of frames to filter. +// continuation_frames Count of occurrences where a frame of type CONTINUATION was found. // path_size_bucket Count of path sizes and divided into buckets. -// frames_split_count Count of times we tried to read more data than the end of the data end. typedef struct { __u64 request_seen; __u64 response_seen; @@ -236,6 +236,7 @@ typedef struct { __u64 literal_value_exceeds_frame; __u64 exceeding_max_interesting_frames; __u64 exceeding_max_frames_to_filter; + __u64 continuation_frames; __u64 path_size_bucket[HTTP2_TELEMETRY_PATH_BUCKETS+1]; } http2_telemetry_t; diff --git a/pkg/network/ebpf/c/protocols/http2/decoding.h b/pkg/network/ebpf/c/protocols/http2/decoding.h index 2582951c20ea5..78dd5bb1f6981 100644 --- a/pkg/network/ebpf/c/protocols/http2/decoding.h +++ b/pkg/network/ebpf/c/protocols/http2/decoding.h @@ -896,12 +896,12 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup } bpf_memset(headers_to_process, 0, HTTP2_MAX_HEADERS_COUNT_FOR_PROCESSING * sizeof(http2_header_t)); - __u8 interesting_headers = 0; - __u64 *global_dynamic_counter = get_dynamic_counter(tup); if (global_dynamic_counter == NULL) { goto delete_iteration; } + __u8 interesting_headers = 0; + __u8 continuation_frames = 0; #pragma unroll(HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL) for (__u16 index = 0; index < HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL; index++) { @@ -915,6 +915,9 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup current_frame = frames_array[tail_call_state->iteration]; tail_call_state->iteration += 1; + if (current_frame.frame.type == kContinuationFrame) { + continuation_frames++; + } if (current_frame.frame.type != kHeadersFrame) { continue; } @@ -930,6 +933,7 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup interesting_headers = pktbuf_filter_relevant_headers(pkt, global_dynamic_counter, &http2_ctx->dynamic_index, headers_to_process, current_frame.frame.length, http2_tel); pktbuf_process_headers(pkt, &http2_ctx->dynamic_index, current_stream, headers_to_process, interesting_headers, http2_tel); } + __sync_fetch_and_add(&http2_tel->continuation_frames, continuation_frames); if (tail_call_state->iteration < HTTP2_MAX_FRAMES_ITERATIONS && tail_call_state->iteration < tail_call_state->frames_count && diff --git a/pkg/network/protocols/http2/model_linux.go b/pkg/network/protocols/http2/model_linux.go index f5675cd8372eb..0d6dfdc441c51 100644 --- a/pkg/network/protocols/http2/model_linux.go +++ b/pkg/network/protocols/http2/model_linux.go @@ -445,6 +445,7 @@ HTTP2Telemetry{ "literal values exceed message count": %d, "messages with more frames than we can filter": %d, "messages with more interesting frames than we can process": %d, + "continuation frames" : %d, "path headers length distribution": { "in range [0, 120)": %d, "in range [120, 130)": %d, @@ -456,7 +457,7 @@ HTTP2Telemetry{ "in range [180, infinity)": %d } }`, t.Request_seen, t.Response_seen, t.End_of_stream, t.End_of_stream_rst, t.Literal_value_exceeds_frame, - t.Exceeding_max_frames_to_filter, t.Exceeding_max_interesting_frames, t.Path_size_bucket[0], t.Path_size_bucket[1], - t.Path_size_bucket[2], t.Path_size_bucket[3], t.Path_size_bucket[4], t.Path_size_bucket[5], t.Path_size_bucket[6], - t.Path_size_bucket[7]) + t.Exceeding_max_frames_to_filter, t.Exceeding_max_interesting_frames, t.Continuation_frames, + t.Path_size_bucket[0], t.Path_size_bucket[1], t.Path_size_bucket[2], t.Path_size_bucket[3], + t.Path_size_bucket[4], t.Path_size_bucket[5], t.Path_size_bucket[6], t.Path_size_bucket[7]) } diff --git a/pkg/network/protocols/http2/telemetry.go b/pkg/network/protocols/http2/telemetry.go index 6e74ad3e7ba07..72af545009d7f 100644 --- a/pkg/network/protocols/http2/telemetry.go +++ b/pkg/network/protocols/http2/telemetry.go @@ -34,6 +34,8 @@ type kernelTelemetry struct { exceedingMaxInterestingFrames *libtelemetry.TLSAwareCounter // exceedingMaxFramesToFilter Count of times we have left with more frames to filter than the max number of frames to filter. exceedingMaxFramesToFilter *libtelemetry.TLSAwareCounter + // continuationFramesCount Count of occurrences where a frame of type CONTINUATION was found. + continuationFramesCount *libtelemetry.TLSAwareCounter // fragmentedFrameCountRST Count of times we have seen a fragmented RST frame. fragmentedFrameCountRST *libtelemetry.TLSAwareCounter // fragmentedHeadersFrameEOSCount Count of times we have seen a fragmented headers frame with EOS. @@ -58,6 +60,7 @@ func newHTTP2KernelTelemetry() *kernelTelemetry { literalValueExceedsFrame: libtelemetry.NewTLSAwareCounter(metricGroup, "literal_value_exceeds_frame"), exceedingMaxInterestingFrames: libtelemetry.NewTLSAwareCounter(metricGroup, "exceeding_max_interesting_frames"), exceedingMaxFramesToFilter: libtelemetry.NewTLSAwareCounter(metricGroup, "exceeding_max_frames_to_filter"), + continuationFramesCount: libtelemetry.NewTLSAwareCounter(metricGroup, "continuation_frames"), fragmentedDataFrameEOSCount: libtelemetry.NewTLSAwareCounter(metricGroup, "exceeding_data_end_data_eos"), fragmentedHeadersFrameCount: libtelemetry.NewTLSAwareCounter(metricGroup, "exceeding_data_end_headers"), fragmentedHeadersFrameEOSCount: libtelemetry.NewTLSAwareCounter(metricGroup, "exceeding_data_end_headers_eos"), @@ -81,6 +84,7 @@ func (t *kernelTelemetry) update(tel *HTTP2Telemetry, isTLS bool) { t.literalValueExceedsFrame.Add(int64(telemetryDelta.Literal_value_exceeds_frame), isTLS) t.exceedingMaxInterestingFrames.Add(int64(telemetryDelta.Exceeding_max_interesting_frames), isTLS) t.exceedingMaxFramesToFilter.Add(int64(telemetryDelta.Exceeding_max_frames_to_filter), isTLS) + t.continuationFramesCount.Add(int64(telemetryDelta.Continuation_frames), isTLS) for bucketIndex := range t.pathSizeBucket { t.pathSizeBucket[bucketIndex].Add(int64(telemetryDelta.Path_size_bucket[bucketIndex]), isTLS) } @@ -104,6 +108,7 @@ func (t *HTTP2Telemetry) Sub(other HTTP2Telemetry) *HTTP2Telemetry { Literal_value_exceeds_frame: t.Literal_value_exceeds_frame - other.Literal_value_exceeds_frame, Exceeding_max_interesting_frames: t.Exceeding_max_interesting_frames - other.Exceeding_max_interesting_frames, Exceeding_max_frames_to_filter: t.Exceeding_max_frames_to_filter - other.Exceeding_max_frames_to_filter, + Continuation_frames: t.Continuation_frames - other.Continuation_frames, Path_size_bucket: computePathSizeBucketDifferences(t.Path_size_bucket, other.Path_size_bucket), } } diff --git a/pkg/network/protocols/http2/types_linux.go b/pkg/network/protocols/http2/types_linux.go index fdc8aba36c321..5bda5aeea347c 100644 --- a/pkg/network/protocols/http2/types_linux.go +++ b/pkg/network/protocols/http2/types_linux.go @@ -83,6 +83,7 @@ type HTTP2Telemetry struct { Literal_value_exceeds_frame uint64 Exceeding_max_interesting_frames uint64 Exceeding_max_frames_to_filter uint64 + Continuation_frames uint64 Path_size_bucket [8]uint64 } type HTTP2IncompleteFrameEntry struct { From 06fe2759ca355102184a69da8078679d3a982eda Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" <yuri.lipnesh@datadoghq.com> Date: Wed, 15 Jan 2025 08:41:22 -0500 Subject: [PATCH 2/5] [usm] continuation frames, reduce processed instructions in usm/socket__http2_headers_parser --- pkg/network/ebpf/c/protocols/http2/decoding.h | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/pkg/network/ebpf/c/protocols/http2/decoding.h b/pkg/network/ebpf/c/protocols/http2/decoding.h index 78dd5bb1f6981..c616037fa6ec4 100644 --- a/pkg/network/ebpf/c/protocols/http2/decoding.h +++ b/pkg/network/ebpf/c/protocols/http2/decoding.h @@ -901,7 +901,6 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup goto delete_iteration; } __u8 interesting_headers = 0; - __u8 continuation_frames = 0; #pragma unroll(HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL) for (__u16 index = 0; index < HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL; index++) { @@ -915,10 +914,10 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup current_frame = frames_array[tail_call_state->iteration]; tail_call_state->iteration += 1; - if (current_frame.frame.type == kContinuationFrame) { - continuation_frames++; - } if (current_frame.frame.type != kHeadersFrame) { + if (current_frame.frame.type == kContinuationFrame) { + __sync_fetch_and_add(&http2_tel->continuation_frames, 1); + } continue; } @@ -933,7 +932,6 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup interesting_headers = pktbuf_filter_relevant_headers(pkt, global_dynamic_counter, &http2_ctx->dynamic_index, headers_to_process, current_frame.frame.length, http2_tel); pktbuf_process_headers(pkt, &http2_ctx->dynamic_index, current_stream, headers_to_process, interesting_headers, http2_tel); } - __sync_fetch_and_add(&http2_tel->continuation_frames, continuation_frames); if (tail_call_state->iteration < HTTP2_MAX_FRAMES_ITERATIONS && tail_call_state->iteration < tail_call_state->frames_count && From 7690b2cb61129ecf8b557cca8b916ba2278035f0 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" <yuri.lipnesh@datadoghq.com> Date: Wed, 15 Jan 2025 11:43:17 -0500 Subject: [PATCH 3/5] [usm] continuation frames, move http2_tel->continuation_frames++ to pktbuf_find_relevant_frames() --- pkg/network/ebpf/c/protocols/http2/decoding.h | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/network/ebpf/c/protocols/http2/decoding.h b/pkg/network/ebpf/c/protocols/http2/decoding.h index c616037fa6ec4..453280f56e1be 100644 --- a/pkg/network/ebpf/c/protocols/http2/decoding.h +++ b/pkg/network/ebpf/c/protocols/http2/decoding.h @@ -575,10 +575,14 @@ static __always_inline bool pktbuf_find_relevant_frames(pktbuf_t pkt, http2_tail // https://datatracker.ietf.org/doc/html/rfc7540#section-6.2 for headers frame. is_headers_or_rst_frame = current_frame.type == kHeadersFrame || current_frame.type == kRSTStreamFrame; is_data_end_of_stream = ((current_frame.flags & HTTP2_END_OF_STREAM) == HTTP2_END_OF_STREAM) && (current_frame.type == kDataFrame); - if (iteration_value->frames_count < HTTP2_MAX_FRAMES_ITERATIONS && (is_headers_or_rst_frame || is_data_end_of_stream)) { - iteration_value->frames_array[iteration_value->frames_count].frame = current_frame; - iteration_value->frames_array[iteration_value->frames_count].offset = pktbuf_data_offset(pkt); - iteration_value->frames_count++; + if (iteration_value->frames_count < HTTP2_MAX_FRAMES_ITERATIONS) { + if (is_headers_or_rst_frame || is_data_end_of_stream) { + iteration_value->frames_array[iteration_value->frames_count].frame = current_frame; + iteration_value->frames_array[iteration_value->frames_count].offset = pktbuf_data_offset(pkt); + iteration_value->frames_count++; + } else if (current_frame.type == kContinuationFrame) { + __sync_fetch_and_add(&http2_tel->continuation_frames, 1); + } } pktbuf_advance(pkt, current_frame.length); @@ -896,11 +900,12 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup } bpf_memset(headers_to_process, 0, HTTP2_MAX_HEADERS_COUNT_FOR_PROCESSING * sizeof(http2_header_t)); + __u8 interesting_headers = 0; + __u64 *global_dynamic_counter = get_dynamic_counter(tup); if (global_dynamic_counter == NULL) { goto delete_iteration; } - __u8 interesting_headers = 0; #pragma unroll(HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL) for (__u16 index = 0; index < HTTP2_MAX_FRAMES_FOR_HEADERS_PARSER_PER_TAIL_CALL; index++) { @@ -915,9 +920,6 @@ static __always_inline void headers_parser(pktbuf_t pkt, void *map_key, conn_tup tail_call_state->iteration += 1; if (current_frame.frame.type != kHeadersFrame) { - if (current_frame.frame.type == kContinuationFrame) { - __sync_fetch_and_add(&http2_tel->continuation_frames, 1); - } continue; } From 92bf39f4f37e979289e371e2321defb2d79aeaab Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" <yuri.lipnesh@datadoghq.com> Date: Wed, 15 Jan 2025 14:11:11 -0500 Subject: [PATCH 4/5] [usm] continuation frames, add UTs including usmHTTP2Suite.TestContinuationFrame() --- pkg/network/protocols/http2/telemetry_test.go | 4 + pkg/network/usm/usm_http2_monitor_test.go | 77 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/pkg/network/protocols/http2/telemetry_test.go b/pkg/network/protocols/http2/telemetry_test.go index fdcb054bbdb3d..c8174d442fcbb 100644 --- a/pkg/network/protocols/http2/telemetry_test.go +++ b/pkg/network/protocols/http2/telemetry_test.go @@ -46,6 +46,7 @@ func testKernelTelemetryUpdate(t *testing.T, isTLS bool) { Literal_value_exceeds_frame: 20, Exceeding_max_interesting_frames: 30, Exceeding_max_frames_to_filter: 40, + Continuation_frames: 5, Path_size_bucket: [8]uint64{1, 2, 3, 4, 5, 6, 7, 8}, } kernelTelemetryGroup.update(http2Telemetry, isTLS) @@ -61,6 +62,7 @@ func testKernelTelemetryUpdate(t *testing.T, isTLS bool) { http2Telemetry.Literal_value_exceeds_frame = 26 http2Telemetry.Exceeding_max_interesting_frames = 32 http2Telemetry.Exceeding_max_frames_to_filter = 42 + http2Telemetry.Continuation_frames = 6 http2Telemetry.Path_size_bucket = [8]uint64{2, 3, 4, 5, 6, 7, 8, 9} kernelTelemetryGroup.update(http2Telemetry, isTLS) assertTelemetryEquality(t, http2Telemetry, kernelTelemetryGroup, isTLS) @@ -74,6 +76,8 @@ func assertTelemetryEquality(t *testing.T, http2Telemetry *HTTP2Telemetry, kerne assert.Equal(t, http2Telemetry.Literal_value_exceeds_frame, uint64(kernelTelemetryGroup.literalValueExceedsFrame.Get(isTLS))) assert.Equal(t, http2Telemetry.Exceeding_max_interesting_frames, uint64(kernelTelemetryGroup.exceedingMaxInterestingFrames.Get(isTLS))) assert.Equal(t, http2Telemetry.Exceeding_max_frames_to_filter, uint64(kernelTelemetryGroup.exceedingMaxFramesToFilter.Get(isTLS))) + assert.Equal(t, http2Telemetry.Continuation_frames, uint64(kernelTelemetryGroup.continuationFramesCount.Get(isTLS))) + for i, bucket := range kernelTelemetryGroup.pathSizeBucket { assert.Equal(t, http2Telemetry.Path_size_bucket[i], uint64(bucket.Get(isTLS))) } diff --git a/pkg/network/usm/usm_http2_monitor_test.go b/pkg/network/usm/usm_http2_monitor_test.go index dea8d20edc345..a904e7ca2bf95 100644 --- a/pkg/network/usm/usm_http2_monitor_test.go +++ b/pkg/network/usm/usm_http2_monitor_test.go @@ -1422,6 +1422,83 @@ func (s *usmHTTP2Suite) TestIncompleteFrameTable() { } } +// TestContinuationFrame tests CONTINUATION frame is captured by kernel telemetry. +func (s *usmHTTP2Suite) TestContinuationFrame() { + t := s.T() + cfg := s.getCfg() + + // Start local server and register its cleanup. + t.Cleanup(startH2CServer(t, authority, s.isTLS)) + + // Start the proxy server. + proxyProcess, cancel := proxy.NewExternalUnixTransparentProxyServer(t, unixPath, authority, s.isTLS) + t.Cleanup(cancel) + require.NoError(t, proxy.WaitForConnectionReady(unixPath)) + + tests := []struct { + name string + messageBuilder func() [][]byte + expectedTelemetry *usmhttp2.HTTP2Telemetry + }{ + { + name: "CONTINUATION frame", + messageBuilder: func() [][]byte { + const headersFrameEndHeaders = false + fullHeaders := generateTestHeaderFields(headersGenerationOptions{}) + prefixHeadersFrame, err := usmhttp2.NewHeadersFrameMessage(usmhttp2.HeadersFrameOptions{ + Headers: fullHeaders[:2], + }) + require.NoError(t, err, "could not create prefix headers frame") + + suffixHeadersFrame, err := usmhttp2.NewHeadersFrameMessage(usmhttp2.HeadersFrameOptions{ + Headers: fullHeaders[2:], + }) + require.NoError(t, err, "could not create suffix headers frame") + + framer := newFramer() + framer. + writeRawHeaders(t, 1, headersFrameEndHeaders, prefixHeadersFrame). + writeRawContinuation(t, 1, endHeaders, suffixHeadersFrame) + return [][]byte{framer.bytes()} + }, + expectedTelemetry: &usmhttp2.HTTP2Telemetry{ + Continuation_frames: 1, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + monitor := setupUSMTLSMonitor(t, cfg, useExistingConsumer) + if s.isTLS { + utils.WaitForProgramsToBeTraced(t, consts.USMModuleName, GoTLSAttacherName, proxyProcess.Process.Pid, utils.ManualTracingFallbackEnabled) + } + + c := dialHTTP2Server(t) + + // Composing CONTINUATION frame. + require.NoError(t, writeInput(c, 500*time.Millisecond, tt.messageBuilder()...)) + + var telemetry *usmhttp2.HTTP2Telemetry + var err error + assert.Eventually(t, func() bool { + + telemetry, err = getHTTP2KernelTelemetry(monitor, s.isTLS) + require.NoError(t, err) + + if telemetry.Continuation_frames != tt.expectedTelemetry.Continuation_frames { + return false + } + return true + }, time.Second*5, time.Millisecond*100) + if t.Failed() { + t.Logf("CONTINUATION frames count: %d != %d", tt.expectedTelemetry.Continuation_frames, telemetry.Continuation_frames) + ebpftest.DumpMapsTestHelper(t, monitor.DumpMaps, usmhttp2.InFlightMap, "http2_dynamic_table") + dumpTelemetry(t, monitor, s.isTLS) + } + }) + } +} + func (s *usmHTTP2Suite) TestRawHuffmanEncoding() { t := s.T() cfg := s.getCfg() From 2b2368cfa930df6d6bbaeb18abfbd451130d4946 Mon Sep 17 00:00:00 2001 From: "yuri.lipnesh" <yuri.lipnesh@datadoghq.com> Date: Wed, 15 Jan 2025 15:24:46 -0500 Subject: [PATCH 5/5] [usm] continuation frames, fix linter error in usmHTTP2Suite.TestContinuationFrame() --- pkg/network/usm/usm_http2_monitor_test.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pkg/network/usm/usm_http2_monitor_test.go b/pkg/network/usm/usm_http2_monitor_test.go index a904e7ca2bf95..1d37ff3e14ba4 100644 --- a/pkg/network/usm/usm_http2_monitor_test.go +++ b/pkg/network/usm/usm_http2_monitor_test.go @@ -1485,10 +1485,7 @@ func (s *usmHTTP2Suite) TestContinuationFrame() { telemetry, err = getHTTP2KernelTelemetry(monitor, s.isTLS) require.NoError(t, err) - if telemetry.Continuation_frames != tt.expectedTelemetry.Continuation_frames { - return false - } - return true + return telemetry.Continuation_frames == tt.expectedTelemetry.Continuation_frames }, time.Second*5, time.Millisecond*100) if t.Failed() { t.Logf("CONTINUATION frames count: %d != %d", tt.expectedTelemetry.Continuation_frames, telemetry.Continuation_frames)