Update sandbox dag with retool

stellar · Dec 19, 2024 · b76195e · b76195e
1 parent afc5bb1
commit b76195e
Showing 1 changed file with 6 additions and 2 deletions.
diff --git a/dags/sandbox_update_dag.py b/dags/sandbox_update_dag.py
@@ -45,6 +45,7 @@
     BQ_DATASET = Variable.get("public_dataset")
     SANDBOX_PROJECT = Variable.get("bq_project")
     SANDBOX_DATASET = Variable.get("sandbox_dataset")
+    INTERNAL_DATASET = Variable.get("bq_dataset")
 
     batch_run_date = "{{ batch_run_date_as_datetime_string(dag, data_interval_start) }}"
 
@@ -59,8 +60,6 @@
     )
 
     for table_id in TABLES_ID:
-        if table_id == "diagnostic_events":
-            continue
         query_path = get_query_filepath("update_table")
         query = file_to_string(query_path)
         sql_params = {
@@ -71,6 +70,11 @@
             "target_dataset": SANDBOX_DATASET,
             "batch_run_date": batch_run_date,
         }
+
+        if table_id == "retool_entity_data":
+            sql_params["project_id"] = SANDBOX_PROJECT
+            sql_params["dataset_id"] = INTERNAL_DATASET
+
         query = query.format(**sql_params)
         tables_update_task = BigQueryInsertJobOperator(
             task_id=f"update_{table_id}",