From f4eb4840a02c880c95cf9e70b3a8957d853adaea Mon Sep 17 00:00:00 2001 From: Tyler Finethy Date: Tue, 28 Jan 2025 14:21:47 -0500 Subject: [PATCH 1/2] add: tests for probe budgets fix: generalize line mapping and add java test fix: filter non-spring-boot weblogs fix: add session_id tag to get correct budgets add: link to feature-parity dashboard --- .../probe_snapshot_log_line_budgets.json | 20 ++++++ ...obe_snapshot_log_line_trigger_budgets.json | 22 +++++++ .../test_debugger_expression_language.py | 32 +++------ .../debugger/test_debugger_probe_snapshot.py | 66 ++++++++++++++++++- tests/debugger/utils.py | 13 ++++ utils/_features.py | 9 +++ .../debugger/DebuggerController.java | 8 +++ .../python/flask/debugger_controller.py | 7 ++ 8 files changed, 154 insertions(+), 23 deletions(-) create mode 100644 tests/debugger/probes/probe_snapshot_log_line_budgets.json create mode 100644 tests/debugger/probes/probe_snapshot_log_line_trigger_budgets.json diff --git a/tests/debugger/probes/probe_snapshot_log_line_budgets.json b/tests/debugger/probes/probe_snapshot_log_line_budgets.json new file mode 100644 index 0000000000..a31df0bc29 --- /dev/null +++ b/tests/debugger/probes/probe_snapshot_log_line_budgets.json @@ -0,0 +1,20 @@ +[ + { + "language": "", + "type": "", + "id": "log170aa-acda-4453-9111-1478a697line", + "version": 0, + "where": { + "typeName": null, + "sourceFile": "ACTUAL_SOURCE_FILE", + "lines": [ + "141" + ] + }, + "captureSnapshot": true, + "capture": { + "maxFieldCount": 200 + }, + "tags": [] + } +] diff --git a/tests/debugger/probes/probe_snapshot_log_line_trigger_budgets.json b/tests/debugger/probes/probe_snapshot_log_line_trigger_budgets.json new file mode 100644 index 0000000000..5477ac2297 --- /dev/null +++ b/tests/debugger/probes/probe_snapshot_log_line_trigger_budgets.json @@ -0,0 +1,22 @@ +[ + { + "language": "", + "type": "", + "id": "log170aa-acda-4453-9111-1478a697line", + "version": 0, + "where": { + "typeName": null, + "sourceFile": "ACTUAL_SOURCE_FILE", + "lines": [ + "141" + ] + }, + "captureSnapshot": true, + "capture": { + "maxFieldCount": 200 + }, + "tags": [ + "session_id: r3z0v" + ] + } +] diff --git a/tests/debugger/test_debugger_expression_language.py b/tests/debugger/test_debugger_expression_language.py index abb81e6bf6..3daae1d9c8 100644 --- a/tests/debugger/test_debugger_expression_language.py +++ b/tests/debugger/test_debugger_expression_language.py @@ -92,7 +92,7 @@ def setup_expression_language_access_variables(self): Dsl("index", [Dsl("getmember", [Dsl("ref", "testStruct"), "Dictionary"]), "two"]), ], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -124,7 +124,7 @@ def setup_expression_language_access_exception(self): message_map, probes = self._create_expression_probes( methodName=method, expressions=[["Accessing exception", ".*Hello from exception", Dsl("ref", "@exception")]], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -187,7 +187,7 @@ def setup_expression_language_comparison_operators(self): ["strValue le a", False, Dsl("le", [Dsl("ref", "strValue"), "a"])], ["strValue ge z", False, Dsl("ge", [Dsl("ref", "strValue"), "z"])], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -240,7 +240,7 @@ def setup_expression_language_instance_of(self): ], ["pii instanceof string", False, Dsl("instanceof", [Dsl("ref", "pii"), self._get_type("string")])], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -279,7 +279,7 @@ def setup_expression_language_logical_operators(self): ], ["not intValue eq 10", False, Dsl("not", Dsl("eq", [Dsl("ref", "intValue"), 5]))], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -327,7 +327,7 @@ def setup_expression_language_string_operations(self): ["emptyString matches empty", True, Dsl("matches", [Dsl("ref", "emptyString"), ""])], ["emptyString matches some", False, Dsl("matches", [Dsl("ref", "emptyString"), "foo"])], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -403,7 +403,7 @@ def setup_expression_language_collection_operations(self): Dsl("len", Dsl("filter", [Dsl("ref", "l5"), Dsl("lt", [Dsl("ref", "@it"), 2])])), ], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -548,7 +548,7 @@ def setup_expression_language_hash_operations(self): ), ], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -569,7 +569,7 @@ def setup_expression_language_nulls_true(self): ["strValue eq null", True, Dsl("eq", [Dsl("ref", "strValue"), None])], ["pii eq null", True, Dsl("eq", [Dsl("ref", "pii"), None])], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -588,7 +588,7 @@ def setup_expression_language_nulls_false(self): ["strValue eq null", False, Dsl("eq", [Dsl("ref", "strValue"), None])], ["pii eq null", False, Dsl("eq", [Dsl("ref", "pii"), None])], ], - lines=self._method_and_language_to_line_number(method, language), + lines=self.method_and_language_to_line_number(method, language), ) self.message_map = message_map @@ -650,18 +650,6 @@ def _get_hash_value_property_name(self): else: return "value" - def _method_and_language_to_line_number(self, method, language): - """_method_and_language_to_line_number returns the respective line number given the method and language""" - return { - "Expression": {"java": [71], "dotnet": [74], "python": [72]}, - # The `@exception` variable is not available in the context of line probes. - "ExpressionException": {}, - "ExpressionOperators": {"java": [82], "dotnet": [90], "python": [87]}, - "StringOperations": {"java": [87], "dotnet": [97], "python": [96]}, - "CollectionOperations": {"java": [114], "dotnet": [114], "python": [123]}, - "Nulls": {"java": [130], "dotnet": [127], "python": [136]}, - }.get(method, {}).get(language, []) - def _create_expression_probes(self, methodName, expressions, lines=()): probes = [] expected_message_map = {} diff --git a/tests/debugger/test_debugger_probe_snapshot.py b/tests/debugger/test_debugger_probe_snapshot.py index 83d53487e8..caf18d894f 100644 --- a/tests/debugger/test_debugger_probe_snapshot.py +++ b/tests/debugger/test_debugger_probe_snapshot.py @@ -11,11 +11,19 @@ @scenarios.debugger_probes_snapshot class Test_Debugger_Probe_Snaphots(debugger._Base_Debugger_Test): ############ setup ############ - def _setup(self, probes_name: str, request_path: str): + def _setup(self, probes_name: str, request_path: str, lines=None): self.initialize_weblog_remote_config() ### prepare probes probes = debugger.read_probes(probes_name) + if lines is not None: + for probe in probes: + if "methodName" in probe["where"]: + del probe["where"]["methodName"] + probe["where"]["lines"] = lines + probe["where"]["sourceFile"] = "ACTUAL_SOURCE_FILE" + probe["where"]["typeName"] = None + self.set_probes(probes) ### send requests @@ -132,3 +140,59 @@ def test_code_origin_entry_present(self): code_origins_entry_found = code_origin_type == "entry" assert code_origins_entry_found + + def setup_log_line_probe_snaphots_budgets(self): + self._setup( + "probe_snapshot_log_line_budgets", + "/debugger/budgets/150", + lines=self.method_and_language_to_line_number("Budgets", self.get_tracer()["language"]), + ) + + @features.debugger_probe_budgets + @missing_feature(context.library == "dotnet", reason="Probe snapshot budgets are not yet implemented") + @missing_feature(context.library == "nodejs", reason="Probe snapshot budgets are not yet implemented") + @missing_feature(context.library == "ruby", reason="Probe snapshot budgets are not yet implemented") + def test_log_line_probe_snaphots_budgets(self): + self._assert() + self._validate_snapshots() + + snapshots = 0 + for _id in self.probe_ids: + for span in self.probe_snapshots[_id]: + snapshot = span.get("debugger", {}).get("snapshot", None) + if snapshot is None: + continue + + snapshots += 1 + + # Snapshot budgets are implemented via a PID-like mechanism, so on startup we expect to see a higher number + # of events than the budget. Eventually it will stabilize to the 1/s rate. + assert 1 <= snapshots <= 20, f"Expected 1-20 snapshots, got {snapshots}" + + def setup_log_line_trigger_probe_snaphots_budgets(self): + self._setup( + "probe_snapshot_log_line_trigger_budgets", + "/debugger/budgets/150", + lines=self.method_and_language_to_line_number("Budgets", self.get_tracer()["language"]), + ) + + @features.debugger_probe_budgets + @missing_feature(context.library == "java", reason="DEBUG-3456 probe stops emitting") + @missing_feature(context.library == "python", reason="Trigger probe snapshot waiting for next release") + @missing_feature(context.library == "dotnet", reason="Trigger probe snapshot budgets are not yet implemented") + @missing_feature(context.library == "nodejs", reason="Trigger probe snapshot budgets are not yet implemented") + @missing_feature(context.library == "ruby", reason="Trigger probe snapshot budgets are not yet implemented") + def test_log_line_trigger_probe_snaphots_budgets(self): + self._assert() + self._validate_snapshots() + + snapshots = 0 + for _id in self.probe_ids: + for span in self.probe_snapshots[_id]: + snapshot = span.get("debugger", {}).get("snapshot", None) + if snapshot is None: + continue + + snapshots += 1 + + assert snapshots == 10, f"Expected 10 snapshots, got {snapshots}" diff --git a/tests/debugger/utils.py b/tests/debugger/utils.py index a116a2ac64..054c5d2759 100644 --- a/tests/debugger/utils.py +++ b/tests/debugger/utils.py @@ -87,6 +87,19 @@ def initialize_weblog_remote_config(self): f"Failed to get /debugger/init: expected status code: 200, actual status code: {response.status_code}" ) + def method_and_language_to_line_number(self, method, language): + """method_and_language_to_line_number returns the respective line number given the method and language""" + return { + "Budgets": {"python": [142], "java": [138]}, + "Expression": {"java": [71], "dotnet": [74], "python": [72]}, + # The `@exception` variable is not available in the context of line probes. + "ExpressionException": {}, + "ExpressionOperators": {"java": [82], "dotnet": [90], "python": [87]}, + "StringOperations": {"java": [87], "dotnet": [97], "python": [96]}, + "CollectionOperations": {"java": [114], "dotnet": [114], "python": [123]}, + "Nulls": {"java": [130], "dotnet": [127], "python": [136]}, + }.get(method, {}).get(language, []) + ###### set ##### def set_probes(self, probes): def _enrich_probes(probes): diff --git a/utils/_features.py b/utils/_features.py index 17f42f55a7..8e48e5b440 100644 --- a/utils/_features.py +++ b/utils/_features.py @@ -2390,6 +2390,15 @@ def debugger_code_origins(test_object): pytest.mark.features(feature_id=360)(test_object) return test_object + @staticmethod + def debugger_probe_budgets(test_object): + """Probe Budgets + + https://feature-parity.us1.prod.dog/#/?feature=368 + """ + pytest.mark.features(feature_id=368)(test_object) + return test_object + @staticmethod def otel_propagators_api(test_object): """OpenTelemetry Propagators API diff --git a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/debugger/DebuggerController.java b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/debugger/DebuggerController.java index af61f16c5f..ec63f7e5b9 100644 --- a/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/debugger/DebuggerController.java +++ b/utils/build/docker/java/spring-boot/src/main/java/com/datadoghq/system_tests/springboot/debugger/DebuggerController.java @@ -131,4 +131,12 @@ public String nulls( ". intValue is null: " + (intValue == null) + ". strValue is null: " + (strValue == null) + "."; } + + @GetMapping("/budgets/{loops}") + public String budgets(@PathVariable int loops) { + for (int i = 0; i < loops; i++) { + int noOp = 0; // Line probe is instrumented here. + } + return "Budgets"; + } } diff --git a/utils/build/docker/python/flask/debugger_controller.py b/utils/build/docker/python/flask/debugger_controller.py index b6160daf7f..cb158d97f3 100644 --- a/utils/build/docker/python/flask/debugger_controller.py +++ b/utils/build/docker/python/flask/debugger_controller.py @@ -134,3 +134,10 @@ def nulls(): pii = Pii() return f"Pii is null {pii is None}. intValue is null {intValue is None}. strValue is null {strValue is None}." + + +@debugger_blueprint.route("/budgets/", methods=["GET"]) +def budgets(loops): + for _ in range(loops): + pass + return "Budgets", 200 From 1d524b001e5ef2bac164203f5663fa02e0da9bad Mon Sep 17 00:00:00 2001 From: Tyler Finethy Date: Tue, 11 Feb 2025 16:07:45 -0500 Subject: [PATCH 2/2] fix: check captures instead of presence of snapshot only --- .../probes/probe_log_method_budgets.json | 19 +++++++++ .../debugger/test_debugger_probe_snapshot.py | 41 +++++++++++++------ 2 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 tests/debugger/probes/probe_log_method_budgets.json diff --git a/tests/debugger/probes/probe_log_method_budgets.json b/tests/debugger/probes/probe_log_method_budgets.json new file mode 100644 index 0000000000..1c04be614a --- /dev/null +++ b/tests/debugger/probes/probe_log_method_budgets.json @@ -0,0 +1,19 @@ +[ + { + "language": "", + "type": "", + "id": "loga0cf2-meth-45cf-9f39-592method", + "version": 0, + "where": { + "typeName": "ACTUAL_TYPE_NAME", + "methodName": "Budgets", + "sourceFile": null + }, + "segments": [ + { + "str": "log message received" + } + ], + "evaluateAt": "EXIT" + } +] diff --git a/tests/debugger/test_debugger_probe_snapshot.py b/tests/debugger/test_debugger_probe_snapshot.py index caf18d894f..a69d74e86d 100644 --- a/tests/debugger/test_debugger_probe_snapshot.py +++ b/tests/debugger/test_debugger_probe_snapshot.py @@ -2,8 +2,8 @@ # This product includes software developed at Datadog (https://www.datadoghq.com/). # Copyright 2021 Datadog, Inc. +import time import tests.debugger.utils as debugger - from utils import scenarios, features, missing_feature, context, rfc @@ -29,7 +29,13 @@ def _setup(self, probes_name: str, request_path: str, lines=None): ### send requests self.send_rc_probes() self.wait_for_all_probes_installed() + + start_time = time.time() self.send_weblog_request(request_path) + end_time = time.time() + # Store the total request time for later use in debugging tests where budgets are limited by time. + self.total_request_time = end_time - start_time + self.wait_for_all_probes_emitting() ########### assert ############ @@ -156,18 +162,25 @@ def test_log_line_probe_snaphots_budgets(self): self._assert() self._validate_snapshots() - snapshots = 0 + snapshots_with_captures = 0 for _id in self.probe_ids: for span in self.probe_snapshots[_id]: - snapshot = span.get("debugger", {}).get("snapshot", None) - if snapshot is None: + snapshot_with_captures = span.get("debugger", {}).get("snapshot", {}).get("captures", None) + if snapshot_with_captures is None: continue - snapshots += 1 + snapshots_with_captures += 1 - # Snapshot budgets are implemented via a PID-like mechanism, so on startup we expect to see a higher number - # of events than the budget. Eventually it will stabilize to the 1/s rate. - assert 1 <= snapshots <= 20, f"Expected 1-20 snapshots, got {snapshots}" + # In the java tracer, we implemented this using a PID-like model, so it inititally returns above the probe + # budget until it stabilizes. + if self.get_tracer()["language"] == "java": + assert ( + snapshots_with_captures == 16 + ), f"Expected 16 snapshot with captures, got {snapshots_with_captures} in {self.total_request_time} seconds" + else: + assert ( + snapshots_with_captures == 1 + ), f"Expected 1 snapshot with captures, got {snapshots_with_captures} in {self.total_request_time} seconds" def setup_log_line_trigger_probe_snaphots_budgets(self): self._setup( @@ -186,13 +199,15 @@ def test_log_line_trigger_probe_snaphots_budgets(self): self._assert() self._validate_snapshots() - snapshots = 0 + snapshots_with_captures = 0 for _id in self.probe_ids: for span in self.probe_snapshots[_id]: - snapshot = span.get("debugger", {}).get("snapshot", None) - if snapshot is None: + snapshot_with_captures = span.get("debugger", {}).get("snapshot", {}).get("captures", None) + if snapshot_with_captures is None: continue - snapshots += 1 + snapshots_with_captures += 1 - assert snapshots == 10, f"Expected 10 snapshots, got {snapshots}" + assert ( + snapshots_with_captures == 10 + ), f"Expected 10 snapshot with captures in session, got {snapshots_with_captures}"