Small fix of system tests (apache#44815)

Co-authored-by: Ulada Zakharava <[email protected]>
astronomer · Dec 11, 2024 · 42dfa7e · 42dfa7e
1 parent cb74a41
commit 42dfa7e
Show file tree

Hide file tree

Showing 7 changed files with 15 additions and 8 deletions.
diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_batch_persistent.py
@@ -42,7 +42,7 @@
 DAG_ID = "dataproc_batch_ps"
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT") or DEFAULT_GCP_SYSTEM_TEST_PROJECT_ID
 BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}".replace("-", "_")
-REGION = "europe-north1"
+REGION = "us-east4"
 CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-")
 CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-")
 CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL

diff --git a/.../system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py b/.../system/google/cloud/dataproc/example_dataproc_cluster_create_existing_stopped_cluster.py
@@ -46,7 +46,7 @@
 CLUSTER_NAME_BASE = f"{DAG_ID}".replace("_", "-")
 CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-")
 CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL
-REGION = "europe-north1"
+REGION = "us-east4"
 
 # Cluster definition
 CLUSTER_CONFIG = {

diff --git a/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py b/providers/tests/system/google/cloud/dataproc/example_dataproc_cluster_deferrable.py
@@ -43,7 +43,7 @@
 CLUSTER_NAME_BASE = f"cluster-{DAG_ID}".replace("_", "-")
 CLUSTER_NAME_FULL = CLUSTER_NAME_BASE + f"-{ENV_ID}".replace("_", "-")
 CLUSTER_NAME = CLUSTER_NAME_BASE if len(CLUSTER_NAME_FULL) >= 33 else CLUSTER_NAME_FULL
-REGION = "europe-north1"
+REGION = "us-east4"
 
 
 # Cluster definition

diff --git a/providers/tests/system/google/cloud/gcs/example_firestore.py b/providers/tests/system/google/cloud/gcs/example_firestore.py
@@ -142,7 +142,7 @@
         environment={
             "tempLocation": f"gs://{BUCKET_NAME}/tmp",
         },
-        location="us-central1",
+        location="us-east4",
         append_job_name=False,
         trigger_rule=TriggerRule.ALL_DONE,
     )

diff --git a/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py b/providers/tests/system/google/cloud/vertex_ai/example_vertex_ai_auto_ml_video_training.py
@@ -59,7 +59,7 @@
 VIDEO_DATA_CONFIG = [
     {
         "import_schema_uri": schema.dataset.ioformat.video.classification,
-        "gcs_source": {"uris": [f"gs://{RESOURCE_DATA_BUCKET}/vertex-ai/datasets/video-dataset.csv"]},
+        "gcs_source": {"uris": [f"gs://{RESOURCE_DATA_BUCKET}/automl/datasets/video/classification.csv"]},
     },
 ]
 

diff --git a/providers/tests/system/google/marketing_platform/example_campaign_manager.py b/providers/tests/system/google/marketing_platform/example_campaign_manager.py
@@ -326,7 +326,13 @@ def delete_connection(connection_id: str) -> None:
 
     # This test needs watcher in order to properly mark success/failure
     # when "tearDown" task with trigger rule is part of the DAG
-    list(dag.tasks) >> watcher()
+
+    # Excluding sensor because we expect it to fail due to cancelled operation
+    [
+        task
+        for task in dag.tasks
+        if task.task_id not in ["insert_conversion", "update_conversion", "delete_connection"]
+    ] >> watcher()
 
 from tests_common.test_utils.system_tests import get_test_run  # noqa: E402
 

diff --git a/scripts/ci/pre_commit/check_system_tests.py b/scripts/ci/pre_commit/check_system_tests.py
@@ -35,6 +35,7 @@
 errors: list[str] = []
 
 WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()"
+WATCHER_APPEND_INSTRUCTION_SHORT = " >> watcher()"
 
 PYTEST_FUNCTION = """
 from tests_common.test_utils.system_tests import get_test_run  # noqa: E402
@@ -52,13 +53,13 @@
 def _check_file(file: Path):
     content = file.read_text()
     if "from tests_common.test_utils.watcher import watcher" in content:
-        index = content.find(WATCHER_APPEND_INSTRUCTION)
+        index = content.find(WATCHER_APPEND_INSTRUCTION_SHORT)
         if index == -1:
             errors.append(
                 f"[red]The example {file} imports tests_common.test_utils.watcher "
                 f"but does not use it properly![/]\n\n"
                 "[yellow]Make sure you have:[/]\n\n"
-                f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
+                f"        {WATCHER_APPEND_INSTRUCTION_SHORT}\n\n"
                 "[yellow]as the last instruction in your example DAG.[/]\n"
             )
         else: