From b2c1b80105fead09e7c7e4724fef6097c18f4e48 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Wed, 18 Dec 2024 15:33:08 -0500 Subject: [PATCH 1/2] Add shredder_mitigation to KPI and search related tables --- .../event_aggregates_spons_tiles_v1/metadata.yaml | 2 ++ .../event_aggregates_suggest_v1/metadata.yaml | 2 ++ .../event_aggregates_v1/metadata.yaml | 2 ++ .../request_payload_suggest_v2/metadata.yaml | 2 ++ .../request_payload_tiles_v2/metadata.yaml | 1 + .../search_derived/search_metric_contribution_v1/metadata.yaml | 1 + .../search_derived/search_revenue_levers_daily_v1/metadata.yaml | 1 + .../active_users_aggregates_attribution_v1/metadata.yaml | 2 ++ .../active_users_aggregates_device_v1/metadata.yaml | 2 ++ .../active_users_aggregates_v4/templates/metadata.yaml | 1 + .../templates/metadata_deletion_request.yaml | 2 ++ 11 files changed, 18 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_spons_tiles_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_spons_tiles_v1/metadata.yaml index 8731aca2a2a..65dd8386ccc 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_spons_tiles_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_spons_tiles_v1/metadata.yaml @@ -8,6 +8,8 @@ labels: schedule: daily dag: bqetl_ctxsvc_derived owner1: rburwei + shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_ctxsvc_derived bigquery: diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/metadata.yaml index c9398a2ba0a..5a83160c543 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_suggest_v1/metadata.yaml @@ -7,6 +7,8 @@ labels: incremental: true schedule: daily dag: bqetl_ctxsvc_derived + shredder_mitigation: true + table_type: aggregate owner1: rburwei scheduling: dag_name: bqetl_ctxsvc_derived diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/metadata.yaml index 354e7a04ff3..ee3326a0d92 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/event_aggregates_v1/metadata.yaml @@ -6,6 +6,8 @@ owners: labels: incremental: true schedule: daily + shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_ctxsvc_derived arguments: ['--schema_update_option=ALLOW_FIELD_ADDITION'] diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/metadata.yaml index e9f44966e10..ae1898951fc 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_suggest_v2/metadata.yaml @@ -12,6 +12,8 @@ labels: incremental: true schedule: daily change_controlled: true + shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_ctxsvc_derived bigquery: diff --git a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/metadata.yaml b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/metadata.yaml index 9c4c3dd4c92..f7cc7a9c5d0 100644 --- a/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/contextual_services_derived/request_payload_tiles_v2/metadata.yaml @@ -12,6 +12,7 @@ labels: incremental: true schedule: daily change_controlled: true + table_type: aggregate scheduling: dag_name: bqetl_ctxsvc_derived bigquery: diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_metric_contribution_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_metric_contribution_v1/metadata.yaml index 99cf829729b..523a3732ce2 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_metric_contribution_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_metric_contribution_v1/metadata.yaml @@ -10,6 +10,7 @@ owners: - jklukas@mozilla.com labels: schedule: daily + table_type: aggregate scheduling: dag_name: bqetl_search bigquery: diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml index b3c13252957..02a25a48693 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_revenue_levers_daily_v1/metadata.yaml @@ -12,6 +12,7 @@ labels: change_controlled: true dag: bqetl_search_dashboard shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_search_dashboard bigquery: diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_attribution_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_attribution_v1/metadata.yaml index d1b0e0ddab8..4f740715eeb 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_attribution_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_attribution_v1/metadata.yaml @@ -22,6 +22,8 @@ owners: - lvargas@mozilla.com labels: incremental: true + shredder_mitigation: true + table_type: aggregate bigquery: time_partitioning: type: day diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_device_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_device_v1/metadata.yaml index 99715ef01dc..31df469a4e6 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_device_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/active_users_aggregates_device_v1/metadata.yaml @@ -12,6 +12,8 @@ owners: - lvargas@mozilla.com labels: incremental: true + shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_analytics_aggregations task_name: active_users_aggregates_device_v1 diff --git a/sql_generators/active_users_aggregates_v4/templates/metadata.yaml b/sql_generators/active_users_aggregates_v4/templates/metadata.yaml index 2c2860b21b0..00e5b0f3f92 100644 --- a/sql_generators/active_users_aggregates_v4/templates/metadata.yaml +++ b/sql_generators/active_users_aggregates_v4/templates/metadata.yaml @@ -27,6 +27,7 @@ labels: incremental: true change_controlled: true shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_analytics_aggregations task_name: {{ app_name }}_{{ table_name }} diff --git a/sql_generators/active_users_deletion_requests/templates/metadata_deletion_request.yaml b/sql_generators/active_users_deletion_requests/templates/metadata_deletion_request.yaml index 82eae690844..80d93ab54ab 100644 --- a/sql_generators/active_users_deletion_requests/templates/metadata_deletion_request.yaml +++ b/sql_generators/active_users_deletion_requests/templates/metadata_deletion_request.yaml @@ -10,6 +10,8 @@ owners: - lvargas@mozilla.com labels: incremental: true + table_type: aggregate + shredder_mitigation: true scheduling: dag_name: bqetl_kpis_shredder task_name: {{ app_name }}_active_users_aggregates_for_deletion_requests From be2eaa5336668ce15c40d225d1c6fba4161a5265 Mon Sep 17 00:00:00 2001 From: alekhyamoz Date: Fri, 20 Dec 2024 10:22:12 -0500 Subject: [PATCH 2/2] add more search files --- .../desktop_mobile_search_clients_monthly_v1/metadata.yaml | 2 ++ .../search_derived/search_aggregates_v8/metadata.yaml | 3 ++- .../search_derived/search_dau_aggregates_v1/metadata.yaml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/search_derived/desktop_mobile_search_clients_monthly_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/desktop_mobile_search_clients_monthly_v1/metadata.yaml index 476ee986776..78ac0a7a2e1 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/desktop_mobile_search_clients_monthly_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/desktop_mobile_search_clients_monthly_v1/metadata.yaml @@ -7,6 +7,8 @@ owners: labels: incremental: true schedule: monthly + shredder_mitigation: true + table_type: aggregate scheduling: dag_name: bqetl_desktop_mobile_search_monthly date_partition_parameter: null diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/metadata.yaml index 7fa24cd45ef..449f2d8db94 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/metadata.yaml @@ -3,7 +3,8 @@ description: |- Daily search clients, aggregated across unique sets of dimensions and partitioned by day. owners: -- akommasani@mozilla.com +- akomar@mozilla.com +- anicholson@mozilla.com - cmorales@mozilla.com labels: schedule: daily diff --git a/sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/metadata.yaml index 6ead916fc69..a19e1ca7484 100644 --- a/sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/search_derived/search_dau_aggregates_v1/metadata.yaml @@ -10,8 +10,8 @@ labels: incremental: true schedule: daily change_controlled: true - shredder_mitigation: true table_type: aggregate + shredder_mitigation: true scheduling: dag_name: bqetl_search_dashboard bigquery: