rename test_inconsistency to incomplete_test

DataDog · Nov 25, 2024 · 62502de · 62502de
1 parent cc9258d
commit 62502de
Show file tree

Hide file tree

Showing 11 changed files with 32 additions and 32 deletions.
diff --git a/conftest.py b/conftest.py
@@ -170,7 +170,7 @@ def _collect_item_metadata(item):
             result["testDeclaration"] = "flaky"
         elif result["details"].startswith("bug"):
             result["testDeclaration"] = "bug"
-        elif result["details"].startswith("test_inconsistency"):
+        elif result["details"].startswith("incomplete_test"):
             result["testDeclaration"] = "testIncomplete"
         elif result["details"].startswith("missing_feature"):
             result["testDeclaration"] = "notImplemented"

diff --git a/docs/edit/skip-tests.md b/docs/edit/skip-tests.md
@@ -4,7 +4,7 @@ Three decorators allow you to skip test functions or classes for a library:
 * `@bug`: The lib does not implement the feature correctly/up to spec
 * `@flaky` (subclass of `bug`): The feature sometimes fails, sometimes passes. It's not reliable, so don't run it.
 * `@missing_feature`: The tested feature/behavior does not exist in the library
-* `@test_inconsistency`: There is a deficit in the weblog/parametric apps or testing interface that prevents us from validating a feature across different applications.
+* `@incomplete_test`: There is a deficit in the weblog/parametric apps or testing interface that prevents us from validating a feature across different applications.
 
 To skip specific test functions within a test class, use them as in-line decorators (Example below).
 To skip test classes or test files, use the decorator in the library's [manifest file](./manifest.md).
@@ -18,7 +18,7 @@ The decorators take several arguments:
 
 
 ```python
-from utils import irrelevant, test_inconsistency, bug, missing_feature
+from utils import irrelevant, incomplete_test, bug, missing_feature
 
 
 @irrelevant(library="nodejs")
@@ -37,7 +37,7 @@ class Test_AwesomeFeature:
     def test_full(self)
         assert 42
 
-    @test_inconsistency(library="python", "trace/span/start endpoint does not exist")
+    @incomplete_test(library="python", "trace/span/start endpoint does not exist")
     def test_span_creation(self):
         assert 68
 ```
diff --git a/docs/execute/test-outcomes.md b/docs/execute/test-outcomes.md
@@ -5,8 +5,8 @@ Those declaration are interpreted by system-tests and impacts the test execution
 | Declaration                                      | Test is executed  | Test actual outcome | System test output  | Comment
 | -                                                | -                 | -                   | -                   | -
 | \<no_declaration>                                | Yes               | ✅ Pass             | 🟢 Success          | All good :sunglasses:
-| Missing feature or bug or test inconsistency     | Yes               | ❌ Fail             | 🟢 Success          | Expected failure
-| Missing feature or bug or test inconsistency     | Yes               | ✅ Pass             | 🟠 Success          | XPASS: The feature has been implemented, bug has been fixed -> easy win
+| Missing feature or bug or incomplete test        | Yes               | ❌ Fail             | 🟢 Success          | Expected failure
+| Missing feature or bug or incomplete test        | Yes               | ✅ Pass             | 🟠 Success          | XPASS: The feature has been implemented, bug has been fixed -> easy win
 | Flaky                                            | No                | N.A.                | N.A.                | A flaky test doesn't provide any usefull information, and thus, is not executed.
 | Irrelevant                                       | No                | N.A.                | N.A                 | There is no purpose of running such a test
 | \<no_declaration>                                | Yes               | ❌ Fail             | 🔴 Fail             | Only use case where system test fails : the test should have been ok, and is not
diff --git a/manifests/cpp.yml b/manifests/cpp.yml
@@ -175,13 +175,13 @@ tests/:
       Test_Parametric_DDSpan_Add_Link: missing_feature (add_link is not supported)
       Test_Parametric_DDSpan_Set_Error: bug (APMAPI-778)  # The expected error status is not set
       Test_Parametric_DDSpan_Set_Metric: missing_feature (Tracer does not provide a public method for directly setting a span metric)
-      Test_Parametric_DDSpan_Set_Resource: test_inconsistency (set_resource endpoint is not implemented)
+      Test_Parametric_DDSpan_Set_Resource: incomplete_test (set_resource endpoint is not implemented)
       Test_Parametric_DDSpan_Start: bug (APMAPI-778)  # Cpp parametric app does not support creating a child span from a finished span
       Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported)
-      Test_Parametric_DDTrace_Config: test_inconsistency (config endpoint is not implemented)
-      Test_Parametric_DDTrace_Crash: test_inconsistency (crash endpoint is not implemented)
-      Test_Parametric_DDTrace_Current_Span: test_inconsistency (current_span endpoint is not implemented)
-      Test_Parametric_DDTrace_Extract_Headers: test_inconsistency (extract_headers endpoint is not implemented)
+      Test_Parametric_DDTrace_Config: incomplete_test (config endpoint is not implemented)
+      Test_Parametric_DDTrace_Crash: incomplete_test (crash endpoint is not implemented)
+      Test_Parametric_DDTrace_Current_Span: incomplete_test (current_span endpoint is not implemented)
+      Test_Parametric_DDTrace_Extract_Headers: incomplete_test (extract_headers endpoint is not implemented)
       # cpp tracer does not support the OpenTelemetry API, otel parametric endpoints are not implemented
       Test_Parametric_OtelSpan_End: missing_feature (otel api is not supported)
       Test_Parametric_OtelSpan_Events: missing_feature (otel api is not supported)

diff --git a/manifests/dotnet.yml b/manifests/dotnet.yml
@@ -393,15 +393,15 @@ tests/:
     test_otel_tracer.py:
       Test_Otel_Tracer: v2.8.0
     test_parametric_endpoints.py:
-      Test_Parametric_DDSpan_Add_Link: test_inconsistency (add_link parametric endpoint is not implemented)
+      Test_Parametric_DDSpan_Add_Link: incomplete_test (add_link parametric endpoint is not implemented)
       Test_Parametric_DDSpan_Set_Error: bug (APMAPI-778) # Set error endpoint does not set the error.type tag on spans
-      Test_Parametric_DDSpan_Set_Resource: test_inconsistency (set_resource parametric endpoint is not implemented)
-      Test_Parametric_DDTrace_Baggage: test_inconsistency (baggage endpoints are not implemented)
-      Test_Parametric_DDTrace_Current_Span: test_inconsistency (current span endpoint is not implemented)
+      Test_Parametric_DDSpan_Set_Resource: incomplete_test (set_resource parametric endpoint is not implemented)
+      Test_Parametric_DDTrace_Baggage: incomplete_test (baggage endpoints are not implemented)
+      Test_Parametric_DDTrace_Current_Span: incomplete_test (current span endpoint is not implemented)
       Test_Parametric_OtelSpan_Is_Recording: bug (APMAPI-778) # parametric endpoint attempts to retrieve spans by the `id` instead of `span_id`
       Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # updates the operation name of the span not the resource name
-      Test_Parametric_Otel_Baggage: test_inconsistency (otel baggage endpoints are not implemented)
-      Test_Parametric_Otel_Current_Span: test_inconsistency (otel current span endpoint are not implemented)
+      Test_Parametric_Otel_Baggage: incomplete_test (otel baggage endpoints are not implemented)
+      Test_Parametric_Otel_Current_Span: incomplete_test (otel current span endpoint are not implemented)
     test_span_links.py: missing_feature
     test_telemetry.py:
       Test_Consistent_Configs: missing_feature

diff --git a/manifests/java.yml b/manifests/java.yml
@@ -1463,11 +1463,11 @@ tests/:
     test_otel_span_with_baggage.py:
       Test_Otel_Span_With_Baggage: missing_feature
     test_parametric_endpoints.py:
-      Test_Parametric_DDSpan_Add_Link: missing_feature (add_link endpoint is not implemented)
-      Test_Parametric_DDTrace_Baggage: missing_feature (baggage endpoints are not implemented)
+      Test_Parametric_DDSpan_Add_Link: incomplete_test (add_link endpoint is not implemented)
+      Test_Parametric_DDTrace_Baggage: incomplete_test (baggage endpoints are not implemented)
       Test_Parametric_DDTrace_Crash: bug (APMAPI-778) # The crash endpoint does not kill the application
       Test_Parametric_DDTrace_Current_Span: bug (APMAPI-778) # Fails to retreive the current span after a span has finished
-      Test_Parametric_Otel_Baggage: missing_feature (baggage is not supported)
+      Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported)
       Test_Parametric_Otel_Current_Span: bug (APMAPI-778) # Current span endpoint does not return DataDog spans created by the otel api
     test_span_links.py: missing_feature
     test_telemetry.py:

diff --git a/manifests/nodejs.yml b/manifests/nodejs.yml
@@ -664,15 +664,15 @@ tests/:
     test_otel_span_with_baggage.py:
       Test_Otel_Span_With_Baggage: missing_feature
     test_parametric_endpoints.py:
-      Test_Parametric_DDSpan_Set_Resource: test_inconsistency (set_resource endpoint is not implemented)
+      Test_Parametric_DDSpan_Set_Resource: incomplete_test (set_resource endpoint is not implemented)
       Test_Parametric_DDSpan_Start: bug (APMAPI-778) # The resource name of the child span is overidden by the parent span.
-      Test_Parametric_DDTrace_Baggage: test_inconsistency (baggage endpoints are not implemented)
-      Test_Parametric_DDTrace_Crash: test_inconsistency (crash endpoint is not implemented)
-      Test_Parametric_DDTrace_Current_Span: test_inconsistency (current_span endpoint is not supported)
+      Test_Parametric_DDTrace_Baggage: incomplete_test (baggage endpoints are not implemented)
+      Test_Parametric_DDTrace_Crash: incomplete_test (crash endpoint is not implemented)
+      Test_Parametric_DDTrace_Current_Span: incomplete_test (current_span endpoint is not supported)
       Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # set_name endpoint should set the resource name on a span (not the operation name)
       Test_Parametric_OtelSpan_Start: bug (APMAPI-778)  # The expected span.kind tag is not set
       Test_Parametric_Otel_Baggage: missing_feature (baggage is not supported)
-      Test_Parametric_Otel_Current_Span: test_inconsistency (otel current_span endpoint is not supported)
+      Test_Parametric_Otel_Current_Span: incomplete_test (otel current_span endpoint is not supported)
     test_partial_flushing.py:
       Test_Partial_Flushing: bug (APMLP-270)
     test_span_links.py:

diff --git a/manifests/ruby.yml b/manifests/ruby.yml
@@ -397,12 +397,12 @@ tests/:
     test_otel_span_with_baggage.py:
       Test_Otel_Span_With_Baggage: missing_feature
     test_parametric_endpoints.py:
-      Test_Parametric_DDSpan_Set_Resource: test_inconsistency (set_resource endpoint is not supported)
+      Test_Parametric_DDSpan_Set_Resource: incomplete_test (set_resource endpoint is not supported)
       Test_Parametric_DDTrace_Baggage: missing_feature (baggage is not supported)
-      Test_Parametric_DDTrace_Current_Span: test_inconsistency (current span endpoint is not supported)
+      Test_Parametric_DDTrace_Current_Span: incomplete_test (current span endpoint is not supported)
       Test_Parametric_OtelSpan_Set_Name: bug (APMAPI-778) # set_name endpoint should set the resource name on a span (not the operation name)
       Test_Parametric_Otel_Baggage: missing_feature (otel baggage is not supported)
-      Test_Parametric_Otel_Current_Span: test_inconsistency (otel current span endpoint is not supported)
+      Test_Parametric_Otel_Current_Span: incomplete_test (otel current span endpoint is not supported)
     test_otel_api_interoperability.py: missing_feature
     test_partial_flushing.py: # Not configurable in a standard way
       Test_Partial_Flushing: missing_feature

diff --git a/tests/test_the_test/test_manifest.py b/tests/test_the_test/test_manifest.py
@@ -57,7 +57,7 @@ def assert_in(elements, module, nodeid):
     def assert_valid_declaration(declaration):
         assert isinstance(declaration, str)
 
-        if re.match(r"^(bug|flaky|irrelevant|missing_feature|test_inconsistency)( \(.+\))?$", declaration):
+        if re.match(r"^(bug|flaky|irrelevant|missing_feature|incomplete_test)( \(.+\))?$", declaration):
             return
 
         # must be a version declaration or semver spec

diff --git a/utils/__init__.py b/utils/__init__.py
@@ -6,7 +6,7 @@
 from utils._weblog import weblog
 from utils._context.core import context
 from utils._context._scenarios import scenarios
-from utils._decorators import bug, irrelevant, missing_feature, rfc, flaky, test_inconsistency
+from utils._decorators import bug, irrelevant, missing_feature, rfc, flaky, incomplete_test
 from utils import interfaces, _remote_config as remote_config
 from utils.interfaces._core import ValidationError
 from utils._features import features
diff --git a/utils/_decorators.py b/utils/_decorators.py
@@ -133,11 +133,11 @@ def missing_feature(condition=None, library=None, weblog_variant=None, reason=No
     return partial(decorator, skip, condition, "missing_feature", reason, False, _get_expected_failure_item)
 
 
-def test_inconsistency(condition=None, library=None, weblog_variant=None, reason=None):
+def incomplete_test(condition=None, library=None, weblog_variant=None, reason=None):
     """Decorator, allow to mark a test function/class as not compatible with the tested application"""
 
     skip = _should_skip(library=library, weblog_variant=weblog_variant, condition=condition)
-    return partial(decorator, skip, condition, "test_inconsistency", reason, False, _get_expected_failure_item)
+    return partial(decorator, skip, condition, "incomplete_test", reason, False, _get_expected_failure_item)
 
 
 def irrelevant(condition=None, library=None, weblog_variant=None, reason=None):