Merge branch 'main' into brettlangdon/docker_ssi.nodejs

.github/workflows/run-end-to-end.yml

@@ -106,6 +106,9 @@ jobs:
     - name: Run IAST_STANDALONE scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"IAST_STANDALONE"')
       run: ./run.sh IAST_STANDALONE
+    - name: Run SCA_STANDALONE scenario
+      if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"SCA_STANDALONE"')
+      run: ./run.sh SCA_STANDALONE
     - name: Run IAST_DEDUPLICATION scenario
       if: always() && steps.build.outcome == 'success' && contains(inputs.scenarios, '"IAST_DEDUPLICATION"')
       run: ./run.sh IAST_DEDUPLICATION

manifests/dotnet.yml

@@ -242,6 +242,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: v2.55.0
       Test_IastStandalone_UpstreamPropagation: v2.55.0
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v2.53.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v2.53.0 but will be replaced by V2)

manifests/golang.yml

@@ -297,6 +297,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: missing_feature
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: missing_feature
       Test_Login_Events_Extended: missing_feature

manifests/java.yml

@@ -907,6 +907,7 @@ tests/:
       Test_IastStandalone_UpstreamPropagation:
         '*': v1.36.0
         spring-boot-3-native: missing_feature (GraalVM. Tracing support only)
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v1.36.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v1.36.0 but will be replaced by V2)

manifests/nodejs.yml

@@ -39,6 +39,7 @@ refs:
   - &ref_5_24_0 '>=5.24.0 || ^4.48.0'
   - &ref_5_25_0 '>=5.25.0 || ^4.49.0'
   - &ref_5_26_0 '>=5.26.0 || ^4.50.0'
+  - &ref_5_27_0 '>=5.27.0 || ^4.51.0'
 
 tests/:
   apm_tracing_e2e/:
@@ -427,6 +428,9 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: *ref_5_18_0
       Test_IastStandalone_UpstreamPropagation: missing_feature # was supposed to be released in 5.18.0
+      Test_SCAStandalone_Telemetry:
+        '*': *ref_5_18_0
+        nextjs: missing_feature
     test_automated_login_events.py:
       Test_Login_Events:
         '*': *ref_4_4_0
@@ -644,7 +648,8 @@ tests/:
       Test_Config_TraceEnabled: *ref_4_3_0
       Test_Config_TraceLogDirectory: missing_feature
       Test_Config_UnifiedServiceTagging: *ref_5_25_0
-    test_crashtracking.py: missing_feature
+    test_crashtracking.py:
+      Test_Crashtracking: *ref_5_27_0
     test_dynamic_configuration.py:
       TestDynamicConfigSamplingRules: *ref_5_16_0
       TestDynamicConfigTracingEnabled: *ref_5_4_0

manifests/php.yml

@@ -201,6 +201,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: missing_feature
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v0.89.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v0.89.0 but will be replaced by V2)

manifests/python.yml

@@ -424,6 +424,7 @@ tests/:
         '*': v2.12.3
         uwsgi-poc: flaky (APPSEC-55222)
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events: irrelevant (was v2.10.0 but will be replaced by V2)
       Test_Login_Events_Extended: irrelevant (was v2.10.0 but will be replaced by V2)

manifests/ruby.yml

@@ -189,6 +189,7 @@ tests/:
     test_asm_standalone.py:
       Test_AppSecStandalone_UpstreamPropagation: v2.4.1-dev
       Test_IastStandalone_UpstreamPropagation: missing_feature
+      Test_SCAStandalone_Telemetry: missing_feature
     test_automated_login_events.py:
       Test_Login_Events:
         '*': v1.13.0

tests/appsec/test_asm_standalone.py

@@ -2,7 +2,8 @@
 
 from requests.structures import CaseInsensitiveDict
 
-from utils import weblog, interfaces, scenarios, features, rfc, bug, flaky
+from utils.telemetry_utils import TelemetryUtils
+from utils import context, weblog, interfaces, scenarios, features, rfc, bug, flaky
 
 
 class AsmStandalone_UpstreamPropagation_Base:
@@ -667,3 +668,65 @@ def test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_1(self)
     @bug(library="java", weblog_variant="play", reason="APPSEC-55552")
     def test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_2(self):
         super().test_no_appsec_upstream__no_asm_event__is_kept_with_priority_1__from_2()
+
+
+@rfc("https://docs.google.com/document/d/12NBx-nD-IoQEMiCRnJXneq4Be7cbtSc6pJLOFUWTpNE/edit")
+@features.sca_standalone
+@scenarios.sca_standalone
+class Test_SCAStandalone_Telemetry:
+    """Tracer correctly propagates SCA telemetry in distributing tracing."""
+
+    def assert_standalone_is_enabled(self, request):
+        # test standalone is enabled and dropping traces
+        for data, _trace, span in interfaces.library.get_spans(request):
+            assert span["metrics"]["_sampling_priority_v1"] <= 0
+            assert span["metrics"]["_dd.apm.enabled"] == 0
+
+    def setup_telemetry_sca_enabled_propagated(self):
+        self.r = weblog.get("/")
+
+    def test_telemetry_sca_enabled_propagated(self):
+        self.assert_standalone_is_enabled(self.r)
+
+        for data in interfaces.library.get_telemetry_data():
+            content = data["request"]["content"]
+            if content.get("request_type") != "app-started":
+                continue
+            configuration = content["payload"]["configuration"]
+
+            configuration_by_name = {item["name"]: item for item in configuration}
+
+        assert configuration_by_name
+
+        DD_APPSEC_SCA_ENABLED = TelemetryUtils.get_dd_appsec_sca_enabled_str(context.library)
+
+        cfg_appsec_enabled = configuration_by_name.get(DD_APPSEC_SCA_ENABLED)
+        assert cfg_appsec_enabled is not None, "Missing telemetry config item for '{}'".format(DD_APPSEC_SCA_ENABLED)
+
+        outcome_value = True
+        if context.library == "java":
+            outcome_value = str(outcome_value).lower()
+        assert cfg_appsec_enabled.get("value") == outcome_value
+
+    def setup_app_dependencies_loaded(self):
+        self.r = weblog.get("/load_dependency")
+
+    def test_app_dependencies_loaded(self):
+        self.assert_standalone_is_enabled(self.r)
+
+        seen_loaded_dependencies = TelemetryUtils.get_loaded_dependency(context.library.library)
+
+        for data in interfaces.library.get_telemetry_data():
+            content = data["request"]["content"]
+            if content.get("request_type") != "app-dependencies-loaded":
+                continue
+
+            for dependency in content["payload"]["dependencies"]:
+                dependency_id = dependency["name"]  # +dependency["version"]
+
+                if dependency_id in seen_loaded_dependencies:
+                    seen_loaded_dependencies[dependency_id] = True
+
+        for dependency, seen in seen_loaded_dependencies.items():
+            if not seen:
+                raise Exception(dependency + " not received in app-dependencies-loaded message")

tests/debugger/probes/pii_line.json

@@ -0,0 +1,18 @@
+[
+    {
+        "language": "",
+        "pii": "",
+        "id": "log170aa-acda-4453-9111-1478a600line",
+        "where": {
+           "typeName": null,
+           "sourceFile": "ACTUAL_SOURCE_FILE",
+            "lines": [
+                "33"
+            ]
+        },
+        "captureSnapshot": true,
+        "capture": {
+            "maxFieldCount": 200
+        }
+    }
+]

tests/debugger/test_debugger_pii.py

@@ -121,35 +121,38 @@ def filter(keys_to_filter):
 @features.debugger_pii_redaction
 @scenarios.debugger_pii_redaction
 class Test_Debugger_PII_Redaction(base._Base_Debugger_Test):
-    def _setup(self):
-        probes = base.read_probes("pii")
+    def _setup(self, probes_file):
+        probes = base.read_probes(probes_file)
         self.expected_probe_ids = base.extract_probe_ids(probes)
         self.rc_state = rc.send_debugger_command(probes, version=1)
 
         interfaces.agent.wait_for(self.wait_for_all_probes_installed, timeout=30)
 
         self.weblog_responses = [weblog.get("/debugger/pii")]
 
-    def _test(self, redacted_keys, redacted_types):
+    def _test(self, redacted_keys, redacted_types, line_probe=False):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
         self.assert_all_weblog_responses_ok()
 
-        self._validate_pii_keyword_redaction(redacted_keys)
-        self._validate_pii_type_redaction(redacted_types)
+        self._validate_pii_keyword_redaction(redacted_keys, line_probe=line_probe)
+        self._validate_pii_type_redaction(redacted_types, line_probe=line_probe)
 
     def setup_pii_redaction_full(self):
-        self._setup()
+        self._setup("pii")
 
     @missing_feature(context.library < "[email protected]", reason="keywords are not fully redacted")
     @missing_feature(context.library < "[email protected]", reason="keywords are not fully redacted")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    # Ruby requires @irrelevant rather than @missing_feature to skip setup
+    # for this test (which will interfere with the line probe test).
+    @irrelevant(context.library == "ruby", reason="Local variable capture not implemented for method probes")
     def test_pii_redaction_full(self):
         self._test(REDACTED_KEYS, REDACTED_TYPES)
 
     def setup_pii_redaction_java_1_33(self):
-        self._setup()
+        self._setup("pii")
 
     @irrelevant(context.library != "[email protected]", reason="not relevant for other version")
     def test_pii_redaction_java_1_33(self):
@@ -170,7 +173,7 @@ def test_pii_redaction_java_1_33(self):
         )
 
     def setup_pii_redaction_dotnet_2_50(self):
-        self._setup()
+        self._setup("pii")
 
     @irrelevant(context.library != "[email protected]", reason="not relevant for other version")
     @bug(
@@ -179,7 +182,14 @@ def setup_pii_redaction_dotnet_2_50(self):
     def test_pii_redaction_dotnet_2_50(self):
         self._test(filter(["applicationkey", "connectionstring"]), REDACTED_TYPES)
 
-    def _validate_pii_keyword_redaction(self, should_redact_field_names):
+    def setup_pii_redaction_line(self):
+        self._setup("pii_line")
+
+    @irrelevant(context.library != "ruby", reason="Ruby needs to use line probes to capture variables")
+    def test_pii_redaction_line(self):
+        self._test(REDACTED_KEYS, REDACTED_TYPES, True)
+
+    def _validate_pii_keyword_redaction(self, should_redact_field_names, line_probe=False):
         agent_logs_endpoint_requests = list(interfaces.agent.get_data(path_filters="/api/v2/logs"))
         not_redacted = []
         not_found = list(set(should_redact_field_names))
@@ -193,12 +203,21 @@ def _validate_pii_keyword_redaction(self, should_redact_field_names):
 
                     if snapshot:
                         for field_name in should_redact_field_names:
-                            fields = snapshot["captures"]["return"]["locals"]["pii"]["fields"]
-
-                            if field_name in fields:
+                            if line_probe:
+                                fields = snapshot["captures"]["lines"]["33"]["locals"]["pii"]["fields"]
+                            else:
+                                fields = snapshot["captures"]["return"]["locals"]["pii"]["fields"]
+
+                            # Ruby prefixes instance variable names with @
+                            if context.library == "ruby":
+                                check_field_name = "@" + field_name
+                            else:
+                                check_field_name = field_name
+
+                            if check_field_name in fields:
                                 not_found.remove(field_name)
 
-                                if "value" in fields[field_name]:
+                                if "value" in fields[check_field_name]:
                                     not_redacted.append(field_name)
         error_message = ""
         if not_redacted:
@@ -212,7 +231,7 @@ def _validate_pii_keyword_redaction(self, should_redact_field_names):
         if error_message != "":
             raise ValueError(error_message)
 
-    def _validate_pii_type_redaction(self, should_redact_types):
+    def _validate_pii_type_redaction(self, should_redact_types, line_probe=False):
         agent_logs_endpoint_requests = list(interfaces.agent.get_data(path_filters="/api/v2/logs"))
         not_redacted = []
 
@@ -225,7 +244,10 @@ def _validate_pii_type_redaction(self, should_redact_types):
 
                     if snapshot:
                         for type_name in should_redact_types:
-                            type_info = snapshot["captures"]["return"]["locals"][type_name]
+                            if line_probe:
+                                type_info = snapshot["captures"]["lines"]["33"]["locals"][type_name]
+                            else:
+                                type_info = snapshot["captures"]["return"]["locals"][type_name]
 
                             if "fields" in type_info:
                                 not_redacted.append(type_name)

tests/debugger/test_debugger_probe_snapshot.py

@@ -4,7 +4,7 @@
 
 import tests.debugger.utils as base
 
-from utils import scenarios, interfaces, weblog, features, remote_config as rc, bug
+from utils import scenarios, interfaces, weblog, features, remote_config as rc, bug, missing_feature, context
 
 
 @features.debugger
@@ -41,6 +41,7 @@ def setup_span_method_probe_snaphots(self):
         ]
 
     @bug(library="python", reason="DEBUG-2708, DEBUG-2709")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_method_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
@@ -61,6 +62,7 @@ def setup_span_decoration_method_probe_snaphots(self):
         ]
 
     @bug(library="python", reason="DEBUG-2708, DEBUG-2709")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_decoration_method_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()
@@ -105,6 +107,7 @@ def setup_span_decoration_line_probe_snaphots(self):
             weblog.get("/debugger/span-decoration/asd/1"),
         ]
 
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_span_decoration_line_probe_snaphots(self):
         self.assert_all_states_not_error()
         self.assert_all_probes_are_installed()

tests/debugger/test_debugger_probe_status.py

@@ -4,7 +4,7 @@
 
 import tests.debugger.utils as base
 
-from utils import scenarios, features, remote_config as rc, bug, context
+from utils import weblog, scenarios, features, remote_config as rc, bug, context, missing_feature
 
 
 @features.debugger
@@ -40,6 +40,7 @@ def setup_probe_status_metric(self):
 
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_metric(self):
         self._assert()
 
@@ -49,6 +50,7 @@ def setup_probe_status_span(self):
 
         self._setup(probes)
 
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_span(self):
         self._assert()
 
@@ -60,6 +62,7 @@ def setup_probe_status_spandecoration(self):
 
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
     @bug(context.library == "[email protected]", reason="DEBUG-3127")
+    @missing_feature(context.library == "ruby", reason="Not yet implemented")
     def test_probe_status_spandecoration(self):
         self._assert()