DENG-6880 - Daily aggregations for NewTab

sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/metadata.yaml

@@ -0,0 +1,5 @@
+friendly_name: Newtab Conditional Daily Aggregates
+description: |-
+  Conditional computations for daily aggregation for newtab.
+owners:
+- [email protected]

sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.newtab_conditional_daily_aggregates`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.newtab_conditional_daily_aggregates_v1`

sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/metadata.yaml

@@ -0,0 +1,5 @@
+friendly_name: Newtab Interactions Daily Aggregates
+description: |-
+  A daily aggregation of newtab interactions, partitioned by day.
+owners:
+- [email protected]

sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.newtab_daily_interactions_aggregates`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.newtab_daily_interactions_aggregates_v1`

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/query.sql

@@ -49,6 +49,20 @@ SELECT
   SUM(pocket_saves) AS pocket_saves,
   SUM(sponsored_pocket_saves) AS sponsored_pocket_saves,
   SUM(organic_pocket_saves) AS organic_pocket_saves,
+  SUM(pocket_thumb_voting_events) AS pocket_thumb_voting_events,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_updates) AS topic_selection_updates,
+  SUM(weather_widget_clicks) AS weather_widget_clicks,
+  SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed,
+  SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple,
+  SUM(wallpaper_clicks) AS wallpaper_clicks,
+  SUM(
+    IF(topic_selection_selected_topics_first_time, 1, 0)
+  ) AS topic_selection_selected_topics_first_time_count,
+  SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals,
+  SUM(organic_pocket_dismissals) AS organic_pocket_dismissals,
+  SUM(topic_selection_dismissals) AS topic_selection_dismissals,
+  SUM(visits_with_default_ui) AS visits_with_default_ui,
 FROM
   `moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1`
 WHERE

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/schema.yaml

@@ -149,3 +149,39 @@ fields:
 - name: organic_pocket_saves
   type: INTEGER
   mode: NULLABLE
+- name: pocket_thumb_voting_events
+  type: INTEGER
+  mode: NULLABLE
+- name: topic_selection_opened
+  type: INTEGER
+  mode: NULLABLE
+- name: topic_selection_updates
+  type: INTEGER
+  mode: NULLABLE
+- name: weather_widget_clicks
+  type: INTEGER
+  mode: NULLABLE
+- name: weather_widget_change_display_to_detailed
+  type: INTEGER
+  mode: NULLABLE
+- name: weather_widget_change_display_to_simple
+  type: INTEGER
+  mode: NULLABLE
+- name: wallpaper_clicks
+  type: INTEGER
+  mode: NULLABLE
+- name: topic_selection_selected_topics_first_time_count
+  type: INTEGER
+  mode: NULLABLE
+- name: sponsored_pocket_dismissals
+  type: INTEGER
+  mode: NULLABLE
+- name: organic_pocket_dismissals
+  type: INTEGER
+  mode: NULLABLE
+- name: topic_selection_dismissals
+  type: INTEGER
+  mode: NULLABLE
+- name: visits_with_default_ui
+  type: INTEGER
+  mode: NULLABLE

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Newtab Conditional Daily Aggregates
+description: |-
+  Conditional computations for daily aggregation for newtab.
+owners:
+  - [email protected]
+labels:
+  application: firefox
+  incremental: true
+  schedule: daily
+  dag: bqetl_newtab
+  owner1: gkatre
+scheduling:
+  dag_name: bqetl_newtab
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  clustering:
+    fields:
+      - country_code
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql

@@ -0,0 +1,171 @@
+SELECT
+  submission_date,
+  country_code,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled,
+  COUNT(DISTINCT client_id) AS total_client_count,
+  COUNT(DISTINCT newtab_visit_id) AS total_visit_count,
+  -- conditional
+  COUNT(DISTINCT IF(channel = 'release', client_id, NULL)) AS client_count,
+  COUNT(DISTINCT IF(channel = 'release', newtab_visit_id, NULL)) AS visit_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:home'
+      AND newtab_homepage_category = 'enabled',
+      client_id,
+      NULL
+    )
+  ) AS home_default_client_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:home'
+      AND newtab_homepage_category = 'enabled',
+      newtab_visit_id,
+      NULL
+    )
+  ) AS home_default_visit_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:newtab'
+      AND newtab_newtab_category = 'enabled',
+      client_id,
+      NULL
+    )
+  ) AS newtab_default_client_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:newtab'
+      AND newtab_newtab_category = 'enabled',
+      newtab_visit_id,
+      NULL
+    )
+  ) AS newtab_default_visit_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:home'
+      AND newtab_homepage_category != 'enabled',
+      client_id,
+      NULL
+    )
+  ) AS home_nondefault_client_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:home'
+      AND newtab_homepage_category != 'enabled',
+      newtab_visit_id,
+      NULL
+    )
+  ) AS home_nondefault_visit_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:newtab'
+      AND newtab_newtab_category != 'enabled',
+      client_id,
+      NULL
+    )
+  ) AS newtab_nondefault_client_count,
+  COUNT(
+    DISTINCT IF(
+      channel = 'release'
+      AND newtab_open_source = 'about:newtab'
+      AND newtab_newtab_category != 'enabled',
+      newtab_visit_id,
+      NULL
+    )
+  ) AS newtab_nondefault_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND newtab_open_source = 'about:welcome', client_id, NULL)
+  ) AS welcome_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND newtab_open_source = 'about:welcome', newtab_visit_id, NULL)
+  ) AS welcome_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND searches > 0, client_id, NULL)
+  ) AS searched_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND searches > 0, newtab_visit_id, NULL)
+  ) AS searched_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_topsite_impressions > 0, client_id, NULL)
+  ) AS sponsored_topsite_impressed_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_topsite_impressions > 0, newtab_visit_id, NULL)
+  ) AS sponsored_topsite_impressed_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_topsite_clicks > 0, client_id, NULL)
+  ) AS sponsored_topsite_clicked_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_topsite_clicks > 0, newtab_visit_id, NULL)
+  ) AS sponsored_topsite_clicked_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND topsite_impressions > 0, client_id, NULL)
+  ) AS topsite_impressed_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND topsite_impressions > 0, newtab_visit_id, NULL)
+  ) AS topsite_impressed_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND topsite_clicks > 0, client_id, NULL)
+  ) AS topsite_clicked_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND topsite_clicks > 0, newtab_visit_id, NULL)
+  ) AS topsite_clicked_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_pocket_impressions > 0, client_id, NULL)
+  ) AS sponsored_pocket_impressed_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_pocket_impressions > 0, newtab_visit_id, NULL)
+  ) AS sponsored_pocket_impressed_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_pocket_clicks > 0, client_id, NULL)
+  ) AS sponsored_pocket_clicked_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND sponsored_pocket_clicks > 0, newtab_visit_id, NULL)
+  ) AS sponsored_pocket_clicked_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND organic_pocket_impressions > 0, client_id, NULL)
+  ) AS organic_pocket_impressed_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND organic_pocket_impressions > 0, newtab_visit_id, NULL)
+  ) AS organic_pocket_impressed_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND organic_pocket_clicks > 0, client_id, NULL)
+  ) AS organic_pocket_clicked_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND organic_pocket_clicks > 0, newtab_visit_id, NULL)
+  ) AS organic_pocket_clicked_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_impressions > 0, client_id, NULL)
+  ) AS pocket_impressed_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_impressions > 0, newtab_visit_id, NULL)
+  ) AS pocket_impressed_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_clicks > 0, client_id, NULL)
+  ) AS pocket_clicked_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_clicks > 0, newtab_visit_id, NULL)
+  ) AS pocket_clicked_visit_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_is_signed_in, client_id, NULL)
+  ) AS pocket_signed_in_client_count,
+  COUNT(
+    DISTINCT IF(channel = 'release' AND pocket_is_signed_in, newtab_visit_id, NULL)
+  ) AS pocket_signed_in_visit_count,
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1`
+WHERE
+  submission_date = @submission_date
+GROUP BY
+  submission_date,
+  country_code,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled