Skip to content

Commit

Permalink
adding interactions aggregations
Browse files Browse the repository at this point in the history
  • Loading branch information
gkatre committed Jan 21, 2025
1 parent 4330322 commit 5593c06
Showing 1 changed file with 84 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
select
submission_date,
search_engine,
search_access_point,
pocket_story_position,
default_private_search_engine,
default_search_engine,
os,
os_version,
country_code,
channel,
browser_version,
browser_name,
newtab_open_source,
pocket_is_signed_in,
pocket_enabled,
pocket_sponsored_stories_enabled,
topsites_enabled,
newtab_homepage_category,
newtab_newtab_category,
newtab_search_enabled,
topsites_rows,
is_new_profile,
activity_segment,

COUNT(DISTINCT client_id) AS client_count,
COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count,
COUNT(DISTINCT newtab_visit_id) AS newtab_interactions_visits,
-- Topsite
sum(topsite_clicks) AS topsite_clicks,
sum(sponsored_topsite_clicks) AS sponsored_topsite_clicks,
sum(organic_topsite_clicks) AS organic_topsite_clicks,
sum(topsite_impressions) AS topsite_impressions,
sum(sponsored_topsite_impressions) AS sponsored_topsite_impressions,
sum(organic_topsite_impressions) AS organic_topsite_impressions,
sum(topsite_dismissals) AS topsite_dismissals,
sum(sponsored_topsite_dismissals) AS sponsored_topsite_dismissals,
sum(organic_topsite_dismissals) AS organic_topsite_dismissals,
-- Search
sum(searches) AS searches,
sum(tagged_search_ad_clicks) AS tagged_search_ad_clicks,
sum(tagged_search_ad_impressions) AS tagged_search_ad_impressions,
sum(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks,
sum(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions,
sum(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks,
sum(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions,
-- Pocket
sum(pocket_impressions) AS pocket_impressions,
sum(sponsored_pocket_impressions) AS sponsored_pocket_impressions,
sum(organic_pocket_impressions) AS organic_pocket_impressions,
sum(pocket_clicks) AS pocket_clicks,
sum(sponsored_pocket_clicks) AS sponsored_pocket_clicks,
sum(organic_pocket_clicks) AS organic_pocket_clicks,
sum(pocket_saves) AS pocket_saves,
sum(sponsored_pocket_saves) AS sponsored_pocket_saves,
sum(organic_pocket_saves) AS organic_pocket_saves,
from
`moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1`
WHERE
submission_date = @submission_date
GROUP BY
submission_date,
search_engine,
search_access_point,
pocket_story_position,
default_private_search_engine,
default_search_engine,
os,
os_version,
country_code,
channel,
browser_version,
browser_name,
newtab_open_source,
pocket_is_signed_in,
pocket_enabled,
pocket_sponsored_stories_enabled,
topsites_enabled,
newtab_homepage_category,
newtab_newtab_category,
newtab_search_enabled,
topsites_rows,
is_new_profile,
activity_segment

1 comment on commit 5593c06

@dataops-ci-bot
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Integration report for "adding interactions aggregations"

sql.diff

Click to expand!
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived: newtab_clients_daily_aggregates_v2
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived: newtab_clients_daily_v2
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived: newtab_daily_interactions_aggregates_v1
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived: newtab_visits_v2
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/metadata.yaml	2025-01-21 18:52:35.000000000 +0000
@@ -0,0 +1,13 @@
+friendly_name: Newtab Clients Daily Aggregates V2
+description: |-
+  Please provide a description for the query
+owners: []
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql	2025-01-21 18:53:28.000000000 +0000
@@ -0,0 +1,98 @@
+SELECT
+  submission_date,
+  normalized_os,
+  normalized_os_version,
+  country_code,
+  locale,
+  channel,
+  browser_version,
+  browser_name,
+  default_search_engine,
+  default_private_search_engine,
+  pocket_is_signed_in,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled,
+  topsites_sponsored_enabled,
+  newtab_homepage_category,
+  newtab_newtab_category,
+  topsites_rows,
+  is_new_profile,
+  activity_segment,
+  COUNT(DISTINCT client_id) AS client_count,
+  COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count,
+  SUM(newtab_visit_count) AS newtab_visit_count,
+  SUM(visits_with_non_impression_engagement) AS visits_with_non_impression_engagement,
+  SUM(visits_with_non_search_engagement) AS visits_with_non_search_engagement,
+  SUM(searches) AS searches,
+  SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks,
+  SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions,
+  SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks,
+  SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions,
+  SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks,
+  SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions,
+  SUM(topsite_tile_clicks) AS topsite_tile_clicks,
+  SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks,
+  SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks,
+  SUM(topsite_tile_impressions) AS topsite_tile_impressions,
+  SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions,
+  SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions,
+  SUM(topsite_tile_dismissals) AS topsite_tile_dismissals,
+  SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals,
+  SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals,
+  SUM(pocket_impressions) AS pocket_impressions,
+  SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions,
+  SUM(organic_pocket_impressions) AS organic_pocket_impressions,
+  SUM(pocket_clicks) AS pocket_clicks,
+  SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks,
+  SUM(organic_pocket_clicks) AS organic_pocket_clicks,
+  SUM(pocket_saves) AS pocket_saves,
+  SUM(sponsored_pocket_saves) AS sponsored_pocket_saves,
+  SUM(organic_pocket_saves) AS organic_pocket_saves,
+-- New additions
+  SUM(pocket_thumb_voting_events) AS pocket_thumb_voting_events,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_updates) AS topic_selection_updates,
+  SUM(weather_widget_clicks) AS weather_widget_clicks,
+  SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed,
+  SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple,
+  SUM(wallpaper_clicks) AS wallpaper_clicks,
+  SUM(topic_selection_selected_topics_first_time) AS topic_selection_selected_topics_first_time,
+  SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals,
+  SUM(organic_pocket_dismissals) AS organic_pocket_dismissals,
+  SUM(topic_selection_dismissals) AS topic_selection_dismissals,
+  SUM(visits_with_default_ui) AS visits_with_default_ui,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+  SUM(topic_selection_opened) AS topic_selection_opened,
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1`
+WHERE
+  {% if is_init() %}
+    submission_date >= "2022-11-01"
+  {% else %}
+    submission_date = @submission_date
+  {% endif %}
+GROUP BY
+  submission_date,
+  normalized_os,
+  normalized_os_version,
+  country_code,
+  locale,
+  channel,
+  browser_version,
+  browser_name,
+  default_search_engine,
+  default_private_search_engine,
+  pocket_is_signed_in,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled,
+  topsites_sponsored_enabled,
+  newtab_homepage_category,
+  newtab_newtab_category,
+  topsites_rows,
+  is_new_profile,
+  activity_segment
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/metadata.yaml	2025-01-21 18:52:35.000000000 +0000
@@ -0,0 +1,13 @@
+friendly_name: Newtab Clients Daily V2
+description: |-
+  Please provide a description for the query
+owners: []
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.telemetry_derived.newtab_visits_v1
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql	2025-01-21 18:53:27.000000000 +0000
@@ -0,0 +1,364 @@
+WITH visits_data_base AS (
+  SELECT
+    client_id,
+    submission_date,
+    legacy_telemetry_client_id,
+    newtab_visit_id,
+    normalized_os,
+    normalized_os_version,
+    country_code,
+    locale,
+    channel,
+    browser_version,
+    browser_name,
+    default_search_engine,
+    default_private_search_engine,
+    pocket_is_signed_in,
+    pocket_enabled,
+    pocket_sponsored_stories_enabled,
+    topsites_enabled,
+    topsites_sponsored_enabled,
+    newtab_weather_widget_enabled,
+    newtab_homepage_category,
+    newtab_newtab_category,
+    topsites_rows,
+    experiments,
+    had_non_impression_engagement,
+    had_non_search_engagement,
+    is_new_profile,
+    activity_segment,
+    search_interactions,
+    topsite_tile_interactions,
+    pocket_interactions,
+    wallpaper_interactions,
+    weather_interactions,
+    topic_selection_interactions,
+    default_ui,
+    newtab_selected_topics,
+    profile_group_id,
+    topsites_sponsored_tiles_configured,
+    newtab_open_source
+  FROM
+    `moz-fx-data-shared-prod.telemetry_derived.newtab_visits_v1`
+  WHERE
+    submission_date = @submission_date
+),
+visits_data AS (
+  SELECT
+    client_id,
+    submission_date,
+    ANY_VALUE(profile_group_id) AS profile_group_id,
+    ANY_VALUE(legacy_telemetry_client_id) AS legacy_telemetry_client_id,
+    COUNT(newtab_visit_id) AS newtab_visit_count,
+    ANY_VALUE(normalized_os) AS normalized_os,
+    ANY_VALUE(normalized_os_version) AS normalized_os_version,
+    ANY_VALUE(country_code) AS country_code,
+    ANY_VALUE(locale) AS locale,
+    ANY_VALUE(channel) AS channel,
+    ANY_VALUE(browser_version) AS browser_version,
+    ANY_VALUE(browser_name) AS browser_name,
+    ANY_VALUE(default_search_engine) AS default_search_engine,
+    ANY_VALUE(default_private_search_engine) AS default_private_search_engine,
+    LOGICAL_OR(pocket_is_signed_in) AS pocket_is_signed_in,
+    LOGICAL_AND(pocket_enabled) AS pocket_enabled,
+    LOGICAL_AND(pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled,
+    LOGICAL_AND(topsites_enabled) AS topsites_enabled,
+    LOGICAL_AND(topsites_sponsored_enabled) AS topsites_sponsored_enabled,
+    LOGICAL_AND(newtab_weather_widget_enabled) AS newtab_weather_widget_enabled,
+    ANY_VALUE(newtab_homepage_category) AS newtab_homepage_category,
+    ANY_VALUE(newtab_newtab_category) AS newtab_newtab_category,
+    ANY_VALUE(topsites_rows) AS topsites_rows,
+    ANY_VALUE(experiments) AS experiments,
+    COUNTIF(had_non_impression_engagement) AS visits_with_non_impression_engagement,
+    COUNTIF(had_non_search_engagement) AS visits_with_non_search_engagement,
+    COUNTIF(default_ui = "default") AS visits_with_default_ui,
+    COUNTIF(
+      default_ui = "default"
+      AND had_non_impression_engagement
+    ) AS visits_with_default_ui_with_non_impression_engagement,
+    COUNTIF(
+      default_ui = "default"
+      AND had_non_search_engagement
+    ) AS visits_with_default_ui_with_non_search_engagement,
+    COUNTIF(
+      default_ui = "home - non-default"
+      OR default_ui = "newtab - non-default"
+    ) AS visits_with_non_default_ui,
+    LOGICAL_OR(is_new_profile) AS is_new_profile,
+    ANY_VALUE(activity_segment) AS activity_segment,
+    LOGICAL_OR(ARRAY_LENGTH(newtab_selected_topics) > 0) AS topic_preferences_set,
+    ANY_VALUE(topsites_sponsored_tiles_configured) AS topsites_sponsored_tiles_configured
+  FROM
+    visits_data_base
+  GROUP BY
+    client_id,
+    submission_date
+),
+search_data AS (
+  SELECT
+    client_id,
+    SUM(searches) AS searches,
+    SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks,
+    SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions,
+    SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks,
+    SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions,
+    SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks,
+    SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions,
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(search_interactions)
+  GROUP BY
+    client_id
+),
+tiles_data AS (
+  SELECT
+    client_id,
+    SUM(topsite_tile_clicks) AS topsite_tile_clicks,
+    SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks,
+    SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks,
+    SUM(topsite_tile_impressions) AS topsite_tile_impressions,
+    SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions,
+    SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions,
+    SUM(topsite_tile_dismissals) AS topsite_tile_dismissals,
+    SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals,
+    SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals,
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(topsite_tile_interactions)
+  GROUP BY
+    client_id
+),
+pocket_data AS (
+  SELECT
+    client_id,
+    SUM(pocket_impressions) AS pocket_impressions,
+    SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions,
+    SUM(organic_pocket_impressions) AS organic_pocket_impressions,
+    SUM(pocket_clicks) AS pocket_clicks,
+    SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks,
+    SUM(organic_pocket_clicks) AS organic_pocket_clicks,
+    SUM(pocket_saves) AS pocket_saves,
+    SUM(sponsored_pocket_saves) AS sponsored_pocket_saves,
+    SUM(organic_pocket_saves) AS organic_pocket_saves,
+    SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals,
+    SUM(organic_pocket_dismissals) AS organic_pocket_dismissals,
+    SUM(pocket_thumbs_up) AS pocket_thumbs_up,
+    SUM(pocket_thumbs_down) AS pocket_thumbs_down,
+    SUM(pocket_thumbs_down) + SUM(pocket_thumbs_up) AS pocket_thumb_voting_events,
+    SUM(list_card_clicks) AS list_card_clicks,
+    SUM(organic_list_card_clicks) AS organic_list_card_clicks,
+    SUM(sponsored_list_card_clicks) AS sponsored_list_card_clicks,
+    SUM(list_card_impressions) AS list_card_impressions,
+    SUM(organic_list_card_impressions) AS organic_list_card_impressions,
+    SUM(sponsored_list_card_impressions) AS sponsored_list_card_impressions,
+    SUM(list_card_saves) AS list_card_saves,
+    SUM(organic_list_card_saves) AS organic_list_card_saves,
+    SUM(sponsored_list_card_saves) AS sponsored_list_card_saves,
+    SUM(list_card_dismissals) AS list_card_dismissals,
+    SUM(organic_list_card_dismissals) AS organic_list_card_dismissals,
+    SUM(sponsored_list_card_dismissals) AS sponsored_list_card_dismissals,
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(pocket_interactions)
+  GROUP BY
+    client_id
+),
+wallpaper_data AS (
+  SELECT
+    client_id,
+    SUM(wallpaper_clicks) AS wallpaper_clicks,
+    SUM(wallpaper_clicks_had_previous_wallpaper) AS wallpaper_clicks_had_previous_wallpaper,
+    SUM(wallpaper_clicks_first_selected_wallpaper) AS wallpaper_clicks_first_selected_wallpaper,
+    SUM(wallpaper_category_clicks) AS wallpaper_category_clicks,
+    SUM(wallpaper_highlight_dismissals) AS wallpaper_highlight_dismissals,
+    SUM(wallpaper_highlight_cta_clicks) AS wallpaper_highlight_cta_clicks
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(wallpaper_interactions)
+  GROUP BY
+    client_id
+),
+weather_data AS (
+  SELECT
+    client_id,
+    SUM(weather_widget_impressions) AS weather_widget_impressions,
+    SUM(weather_widget_clicks) AS weather_widget_clicks,
+    SUM(weather_widget_load_errors) AS weather_widget_load_errors,
+    SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed,
+    SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple,
+    SUM(weather_widget_location_selected) AS weather_widget_location_selected
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(weather_interactions)
+  GROUP BY
+    client_id
+),
+topic_selection_data AS (
+  SELECT
+    client_id,
+    LOGICAL_OR(
+      topic_selection_topics_first_saved > 0
+    ) AS topic_selection_selected_topics_first_time,
+    SUM(topic_selection_topics_updated) AS topic_selection_updates,
+    SUM(topic_selection_open) AS topic_selection_opened,
+    SUM(topic_selection_dismiss) AS topic_selection_dismissals
+  FROM
+    visits_data_base
+  CROSS JOIN
+    UNNEST(topic_selection_interactions)
+  GROUP BY
+    client_id
+),
+joined AS (
+  SELECT
+    visits_data.client_id,
+    visits_data.submission_date,
+    visits_data.legacy_telemetry_client_id,
+    visits_data.newtab_visit_count,
+    visits_data.normalized_os,
+    visits_data.normalized_os_version,
+    visits_data.country_code,
+    visits_data.locale,
+    visits_data.channel,
+    visits_data.browser_version,
+    visits_data.browser_name,
+    visits_data.default_search_engine,
+    visits_data.default_private_search_engine,
+    visits_data.pocket_is_signed_in,
+    visits_data.pocket_enabled,
+    visits_data.pocket_sponsored_stories_enabled,
+    visits_data.topsites_enabled,
+    visits_data.topsites_sponsored_enabled,
+    visits_data.newtab_weather_widget_enabled,
+    visits_data.newtab_homepage_category,
+    visits_data.newtab_newtab_category,
+    visits_data.topsites_rows,
+    visits_data.experiments,
+    visits_data.visits_with_non_impression_engagement,
+    visits_data.visits_with_non_search_engagement,
+    visits_data.visits_with_non_default_ui,
+    visits_data.is_new_profile,
+    visits_data.activity_segment,
+    -- COALESCE calls for visits where no interactions with a surface were performed and are all Null
+    COALESCE(search_data.searches, 0) AS searches,
+    COALESCE(search_data.tagged_search_ad_clicks, 0) AS tagged_search_ad_clicks,
+    COALESCE(search_data.tagged_search_ad_impressions, 0) AS tagged_search_ad_impressions,
+    COALESCE(search_data.follow_on_search_ad_clicks, 0) AS follow_on_search_ad_clicks,
+    COALESCE(search_data.follow_on_search_ad_impressions, 0) AS follow_on_search_ad_impressions,
+    COALESCE(search_data.tagged_follow_on_search_ad_clicks, 0) AS tagged_follow_on_search_ad_clicks,
+    COALESCE(
+      search_data.tagged_follow_on_search_ad_impressions,
+      0
+    ) AS tagged_follow_on_search_ad_impressions,
+    COALESCE(tiles_data.topsite_tile_clicks, 0) AS topsite_tile_clicks,
+    COALESCE(tiles_data.sponsored_topsite_tile_clicks, 0) AS sponsored_topsite_tile_clicks,
+    COALESCE(tiles_data.organic_topsite_tile_clicks, 0) AS organic_topsite_tile_clicks,
+    COALESCE(tiles_data.topsite_tile_impressions, 0) AS topsite_tile_impressions,
+    COALESCE(
+      tiles_data.sponsored_topsite_tile_impressions,
+      0
+    ) AS sponsored_topsite_tile_impressions,
+    COALESCE(tiles_data.organic_topsite_tile_impressions, 0) AS organic_topsite_tile_impressions,
+    COALESCE(tiles_data.topsite_tile_dismissals, 0) AS topsite_tile_dismissals,
+    COALESCE(tiles_data.sponsored_topsite_tile_dismissals, 0) AS sponsored_topsite_tile_dismissals,
+    COALESCE(tiles_data.organic_topsite_tile_dismissals, 0) AS organic_topsite_tile_dismissals,
+    COALESCE(pocket_data.pocket_impressions, 0) AS pocket_impressions,
+    COALESCE(pocket_data.sponsored_pocket_impressions, 0) AS sponsored_pocket_impressions,
+    COALESCE(pocket_data.organic_pocket_impressions, 0) AS organic_pocket_impressions,
+    COALESCE(pocket_data.pocket_clicks, 0) AS pocket_clicks,
+    COALESCE(pocket_data.sponsored_pocket_clicks, 0) AS sponsored_pocket_clicks,
+    COALESCE(pocket_data.organic_pocket_clicks, 0) AS organic_pocket_clicks,
+    COALESCE(pocket_data.pocket_saves, 0) AS pocket_saves,
+    COALESCE(pocket_data.sponsored_pocket_saves, 0) AS sponsored_pocket_saves,
+    COALESCE(pocket_data.organic_pocket_saves, 0) AS organic_pocket_saves,
+    COALESCE(wallpaper_clicks, 0) AS wallpaper_clicks,
+    COALESCE(wallpaper_clicks_had_previous_wallpaper, 0) AS wallpaper_clicks_had_previous_wallpaper,
+    COALESCE(
+      wallpaper_clicks_first_selected_wallpaper,
+      0
+    ) AS wallpaper_clicks_first_selected_wallpaper,
+    COALESCE(wallpaper_data.wallpaper_category_clicks, 0) AS wallpaper_category_clicks,
+    COALESCE(wallpaper_data.wallpaper_highlight_dismissals, 0) AS wallpaper_highlight_dismissals,
+    COALESCE(wallpaper_data.wallpaper_highlight_cta_clicks, 0) AS wallpaper_highlight_cta_clicks,
+    COALESCE(weather_data.weather_widget_impressions, 0) AS weather_widget_impressions,
+    COALESCE(weather_data.weather_widget_clicks, 0) AS weather_widget_clicks,
+    COALESCE(weather_data.weather_widget_load_errors, 0) AS weather_widget_load_errors,
+    COALESCE(
+      weather_data.weather_widget_change_display_to_detailed,
+      0
+    ) AS weather_widget_change_display_to_detailed,
+    COALESCE(
+      weather_data.weather_widget_change_display_to_simple,
+      0
+    ) AS weather_widget_change_display_to_simple,
+    COALESCE(weather_data.weather_widget_location_selected, 0) AS weather_widget_location_selected,
+    COALESCE(visits_data.visits_with_default_ui, 0) AS visits_with_default_ui,
+    COALESCE(
+      visits_data.visits_with_default_ui_with_non_impression_engagement,
+      0
+    ) AS visits_with_default_ui_with_non_impression_engagement,
+    COALESCE(
+      visits_data.visits_with_default_ui_with_non_search_engagement,
+      0
+    ) AS visits_with_default_ui_with_non_search_engagement,
+    COALESCE(visits_data.topic_preferences_set, FALSE) AS topic_preferences_set,
+    COALESCE(pocket_data.sponsored_pocket_dismissals, 0) AS sponsored_pocket_dismissals,
+    COALESCE(pocket_data.organic_pocket_dismissals, 0) AS organic_pocket_dismissals,
+    COALESCE(pocket_data.pocket_thumbs_up, 0) AS pocket_thumbs_up,
+    COALESCE(pocket_data.pocket_thumbs_down, 0) AS pocket_thumbs_down,
+    COALESCE(pocket_data.pocket_thumb_voting_events, 0) AS pocket_thumb_voting_events,
+    COALESCE(
+      topic_selection_data.topic_selection_selected_topics_first_time,
+      FALSE
+    ) AS topic_selection_selected_topics_first_time,
+    COALESCE(topic_selection_data.topic_selection_updates, 0) AS topic_selection_updates,
+    COALESCE(topic_selection_data.topic_selection_opened, 0) AS topic_selection_opened,
+    COALESCE(topic_selection_data.topic_selection_dismissals, 0) AS topic_selection_dismissals,
+    visits_data.profile_group_id AS profile_group_id,
+    visits_data.topsites_sponsored_tiles_configured AS topsites_sponsored_tiles_configured,
+    COALESCE(pocket_data.list_card_clicks, 0) AS list_card_clicks,
+    COALESCE(pocket_data.organic_list_card_clicks, 0) AS organic_list_card_clicks,
+    COALESCE(pocket_data.sponsored_list_card_clicks, 0) AS sponsored_list_card_clicks,
+    COALESCE(pocket_data.list_card_impressions, 0) AS list_card_impressions,
+    COALESCE(pocket_data.organic_list_card_impressions, 0) AS organic_list_card_impressions,
+    COALESCE(pocket_data.sponsored_list_card_impressions, 0) AS sponsored_list_card_impressions,
+    COALESCE(pocket_data.list_card_saves, 0) AS list_card_saves,
+    COALESCE(pocket_data.organic_list_card_saves, 0) AS organic_list_card_saves,
+    COALESCE(pocket_data.sponsored_list_card_saves, 0) AS sponsored_list_card_saves,
+    COALESCE(pocket_data.list_card_dismissals, 0) AS list_card_dismissals,
+    COALESCE(pocket_data.organic_list_card_dismissals, 0) AS organic_list_card_dismissals,
+    COALESCE(pocket_data.sponsored_list_card_dismissals, 0) AS sponsored_list_card_dismissals,
+  FROM
+    visits_data
+  LEFT JOIN
+    search_data
+    USING (client_id)
+  LEFT JOIN
+    tiles_data
+    USING (client_id)
+  LEFT JOIN
+    pocket_data
+    USING (client_id)
+  LEFT JOIN
+    weather_data
+    USING (client_id)
+  LEFT JOIN
+    wallpaper_data
+    USING (client_id)
+  LEFT JOIN
+    topic_selection_data
+    USING (client_id)
+)
+SELECT
+  *,
+  topsite_tile_clicks + pocket_clicks + pocket_saves + wallpaper_clicks + pocket_thumb_voting_events + topic_selection_updates + topic_selection_opened + CAST(
+    topic_selection_selected_topics_first_time AS INT
+  ) + weather_widget_clicks + weather_widget_change_display_to_detailed + weather_widget_change_display_to_simple + topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS non_search_engagement_count,
+  topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS newtab_dismissal_count
+FROM
+  joined
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml	2025-01-21 18:52:35.000000000 +0000
@@ -0,0 +1,13 @@
+friendly_name: Newtab Daily Interactions Aggregates V1
+description: |-
+  Please provide a description for the query
+owners: []
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql	2025-01-21 18:53:27.000000000 +0000
@@ -0,0 +1,83 @@
+SELECT
+  submission_date,
+  search_engine,
+  search_access_point,
+  pocket_story_position,
+  default_private_search_engine,
+  default_search_engine,
+  os,
+  os_version,
+  country_code,
+  channel,
+  browser_version,
+  browser_name,
+  newtab_open_source,
+  pocket_is_signed_in,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled,
+  newtab_homepage_category,
+  newtab_newtab_category,
+  newtab_search_enabled,
+  topsites_rows,
+  is_new_profile,
+  activity_segment,
+  COUNT(DISTINCT client_id) AS client_count,
+  COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count,
+  COUNT(DISTINCT newtab_visit_id) AS newtab_interactions_visits,
+        -- Topsite
+  SUM(topsite_clicks) AS topsite_clicks,
+  SUM(sponsored_topsite_clicks) AS sponsored_topsite_clicks,
+  SUM(organic_topsite_clicks) AS organic_topsite_clicks,
+  SUM(topsite_impressions) AS topsite_impressions,
+  SUM(sponsored_topsite_impressions) AS sponsored_topsite_impressions,
+  SUM(organic_topsite_impressions) AS organic_topsite_impressions,
+  SUM(topsite_dismissals) AS topsite_dismissals,
+  SUM(sponsored_topsite_dismissals) AS sponsored_topsite_dismissals,
+  SUM(organic_topsite_dismissals) AS organic_topsite_dismissals,
+        -- Search
+  SUM(searches) AS searches,
+  SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks,
+  SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions,
+  SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks,
+  SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions,
+  SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks,
+  SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions,
+        -- Pocket
+  SUM(pocket_impressions) AS pocket_impressions,
+  SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions,
+  SUM(organic_pocket_impressions) AS organic_pocket_impressions,
+  SUM(pocket_clicks) AS pocket_clicks,
+  SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks,
+  SUM(organic_pocket_clicks) AS organic_pocket_clicks,
+  SUM(pocket_saves) AS pocket_saves,
+  SUM(sponsored_pocket_saves) AS sponsored_pocket_saves,
+  SUM(organic_pocket_saves) AS organic_pocket_saves,
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1`
+WHERE
+  submission_date = @submission_date
+GROUP BY
+  submission_date,
+  search_engine,
+  search_access_point,
+  pocket_story_position,
+  default_private_search_engine,
+  default_search_engine,
+  os,
+  os_version,
+  country_code,
+  channel,
+  browser_version,
+  browser_name,
+  newtab_open_source,
+  pocket_is_signed_in,
+  pocket_enabled,
+  pocket_sponsored_stories_enabled,
+  topsites_enabled,
+  newtab_homepage_category,
+  newtab_newtab_category,
+  newtab_search_enabled,
+  topsites_rows,
+  is_new_profile,
+  activity_segment
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/metadata.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/metadata.yaml	2025-01-21 18:52:35.000000000 +0000
@@ -0,0 +1,14 @@
+friendly_name: Newtab Visits V2
+description: |-
+  Please provide a description for the query
+owners: []
+labels: {}
+bigquery: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+references:
+  query.sql:
+  - moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1
+  - moz-fx-data-shared-prod.telemetry_derived.unified_metrics_v1
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql	2025-01-21 18:53:28.000000000 +0000
@@ -0,0 +1,630 @@
+WITH events_unnested AS (
+  SELECT
+    DATE(submission_timestamp) AS submission_date,
+    category AS event_category,
+    name AS event_name,
+    timestamp AS event_timestamp,
+    client_info,
+    METADATA,
+    normalized_os,
+    normalized_os_version,
+    normalized_country_code,
+    normalized_channel,
+    ping_info,
+    extra AS event_details,
+    metrics
+  FROM
+    -- https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/pings/newtab
+    `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1`,
+    UNNEST(events)
+  WHERE
+    DATE(submission_timestamp) = @submission_date
+    AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket')
+),
+visit_metadata AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    submission_date,
+    ANY_VALUE(client_info.client_id) AS client_id,
+    ANY_VALUE(metrics.uuid.legacy_telemetry_client_id) AS legacy_telemetry_client_id,
+    ANY_VALUE(metrics.uuid.legacy_telemetry_profile_group_id) AS profile_group_id,
+    ANY_VALUE(normalized_os) AS normalized_os,
+    ANY_VALUE(normalized_os_version) AS normalized_os_version,
+    ANY_VALUE(normalized_country_code) AS country_code,
+    ANY_VALUE(normalized_channel) AS channel,
+    ANY_VALUE(client_info.locale) AS locale,
+    ANY_VALUE(client_info.app_display_version) AS browser_version,
+    "Firefox Desktop" AS browser_name,
+    ANY_VALUE(metrics.string.search_engine_default_engine_id) AS default_search_engine,
+    ANY_VALUE(metrics.string.search_engine_private_engine_id) AS default_private_search_engine,
+    ANY_VALUE(metrics.boolean.pocket_is_signed_in) AS pocket_is_signed_in,
+    ANY_VALUE(metrics.boolean.pocket_enabled) AS pocket_enabled,
+    ANY_VALUE(metrics.boolean.pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled,
+    ANY_VALUE(metrics.boolean.topsites_enabled) AS topsites_enabled,
+    ANY_VALUE(metrics.boolean.topsites_sponsored_enabled) AS topsites_sponsored_enabled,
+    ANY_VALUE(
+      metrics.quantity.topsites_sponsored_tiles_configured
+    ) AS topsites_sponsored_tiles_configured,
+    ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category,
+    ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category,
+    ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled,
+    ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled,
+    ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows,
+    ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors,
+    ANY_VALUE(ping_info.experiments) AS experiments,
+    MIN(IF(event_name = "opened", event_timestamp, NULL)) AS newtab_visit_started_at,
+    MIN(IF(event_name = "closed", event_timestamp, NULL)) AS newtab_visit_ended_at,
+    ANY_VALUE(
+      IF(event_name = "opened", mozfun.map.get_key(event_details, "source"), NULL)
+    ) AS newtab_open_source,
+    LOGICAL_OR(event_name IN ("click", "issued", "save")) AS had_non_impression_engagement,
+    LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement,
+    ANY_VALUE(metrics.string_list.newtab_selected_topics) AS newtab_selected_topics,
+    ANY_VALUE(
+      IF(
+        event_name = "opened",
+        SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_height") AS INT),
+        NULL
+      )
+    ) AS newtab_window_inner_height,
+    ANY_VALUE(
+      IF(
+        event_name = "opened",
+        SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_width") AS INT),
+        NULL
+      )
+    ) AS newtab_window_inner_width,
+  FROM
+    events_unnested
+  GROUP BY
+    newtab_visit_id,
+    submission_date
+  HAVING
+    newtab_visit_id IS NOT NULL
+),
+search_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    `moz-fx-data-shared-prod`.udf.normalize_search_engine(
+      mozfun.map.get_key(event_details, "telemetry_id")
+    ) AS search_engine,
+    mozfun.map.get_key(event_details, "search_access_point") AS search_access_point,
+    COUNTIF(event_name = "issued") AS searches,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_tagged") = "true"
+    ) AS tagged_search_ad_impressions,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
+    ) AS follow_on_search_ad_impressions,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
+      AND mozfun.map.get_key(event_details, "is_tagged") = "true"
+    ) AS tagged_follow_on_search_ad_impressions,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_tagged") = "true"
+    ) AS tagged_search_ad_clicks,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
+    ) AS follow_on_search_ad_clicks,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_follow_on") = "true"
+      AND mozfun.map.get_key(event_details, "is_tagged") = "true"
+    ) AS tagged_follow_on_search_ad_clicks
+  FROM
+    events_unnested
+  WHERE
+    event_category IN ('newtab.search', 'newtab.search.ad')
+    AND event_name IN ('click', 'impression', 'issued')
+  GROUP BY
+    newtab_visit_id,
+    search_engine,
+    search_access_point
+),
+search_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        search_engine,
+        search_access_point,
+        searches,
+        tagged_search_ad_clicks,
+        tagged_search_ad_impressions,
+        follow_on_search_ad_clicks,
+        follow_on_search_ad_impressions,
+        tagged_follow_on_search_ad_clicks,
+        tagged_follow_on_search_ad_impressions
+      )
+    ) AS search_interactions
+  FROM
+    search_events
+  GROUP BY
+    newtab_visit_id
+),
+topsites_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS topsite_tile_position,
+    mozfun.map.get_key(event_details, "advertiser_name") AS topsite_tile_advertiser_name,
+    mozfun.map.get_key(event_details, "tile_id") AS topsite_tile_id,
+    JSON_EXTRACT(sov, "$.assigned") AS topsite_tile_assigned_sov_branch,
+    JSON_EXTRACT(sov, "$.chosen") AS topsite_tile_displayed_sov_branch,
+    COUNTIF(event_name = 'impression') AS topsite_tile_impressions,
+    COUNTIF(event_name = 'click') AS topsite_tile_clicks,
+    COUNTIF(
+      event_name = 'impression'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_topsite_tile_impressions,
+    COUNTIF(
+      event_name = 'impression'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
+    ) AS organic_topsite_tile_impressions,
+    COUNTIF(
+      event_name = 'click'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_topsite_tile_clicks,
+    COUNTIF(
+      event_name = 'click'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
+    ) AS organic_topsite_tile_clicks,
+    COUNTIF(event_name = 'dismiss') AS topsite_tile_dismissals,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_topsite_tile_dismissals,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "false"
+    ) AS organic_topsite_tile_dismissals,
+  FROM
+    events_unnested
+  LEFT JOIN
+    UNNEST(metrics.string_list.newtab_sov_allocation) sov
+    ON SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) = SAFE_CAST(
+      JSON_EXTRACT(sov, "$.pos") AS INT64
+    )
+  WHERE
+    event_category = 'topsites'
+    AND event_name IN ('dismiss', 'click', 'impression')
+  GROUP BY
+    newtab_visit_id,
+    topsite_tile_position,
+    topsite_tile_advertiser_name,
+    topsite_tile_id,
+    topsite_tile_assigned_sov_branch,
+    topsite_tile_displayed_sov_branch
+),
+topsites_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        topsite_tile_advertiser_name,
+        topsite_tile_position,
+        topsite_tile_id,
+        topsite_tile_assigned_sov_branch,
+        topsite_tile_displayed_sov_branch,
+        topsite_tile_clicks,
+        sponsored_topsite_tile_clicks,
+        organic_topsite_tile_clicks,
+        topsite_tile_impressions,
+        sponsored_topsite_tile_impressions,
+        organic_topsite_tile_impressions,
+        topsite_tile_dismissals,
+        sponsored_topsite_tile_dismissals,
+        organic_topsite_tile_dismissals
+      )
+    ) AS topsite_tile_interactions
+  FROM
+    topsites_events
+  GROUP BY
+    newtab_visit_id
+),
+pocket_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position,
+    mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id,
+    mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id,
+    COUNTIF(
+      event_name = 'save'
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS pocket_saves,
+    COUNTIF(
+      event_name = 'click'
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS pocket_clicks,
+    COUNTIF(
+      event_name = 'impression'
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS pocket_impressions,
+    COUNTIF(
+      event_name = 'click'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS sponsored_pocket_clicks,
+    COUNTIF(
+      event_name = 'click'
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS organic_pocket_clicks,
+    COUNTIF(
+      event_name = 'impression'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS sponsored_pocket_impressions,
+    COUNTIF(
+      event_name = 'impression'
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS organic_pocket_impressions,
+    COUNTIF(
+      event_name = 'save'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS sponsored_pocket_saves,
+    COUNTIF(
+      event_name = 'save'
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS organic_pocket_saves,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS sponsored_pocket_dismissals,
+    COUNTIF(
+      event_name = 'dismiss'
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+      AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false"
+    ) AS organic_pocket_dismissals,
+    COUNTIF(
+      event_name = 'thumb_voting_interaction'
+      AND mozfun.map.get_key(event_details, "thumbs_up") = "true"
+    ) AS pocket_thumbs_up,
+    COUNTIF(
+      event_name = 'thumb_voting_interaction'
+      AND mozfun.map.get_key(event_details, "thumbs_down") = "true"
+    ) AS pocket_thumbs_down,
+    SAFE_CAST(mozfun.map.get_key(event_details, "received_rank") AS INT) AS pocket_received_rank,
+    mozfun.map.get_key(
+      event_details,
+      "scheduled_corpus_item_id"
+    ) AS pocket_scheduled_corpus_item_id,
+    mozfun.map.get_key(event_details, "topic") AS pocket_topic,
+    mozfun.map.get_key(event_details, "matches_selected_topic") AS pocket_matches_selected_topic,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+    ) AS list_card_clicks,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+    ) AS organic_list_card_clicks,
+    COUNTIF(
+      event_name = "click"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_list_card_clicks,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+    ) AS list_card_impressions,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+    ) AS organic_list_card_impressions,
+    COUNTIF(
+      event_name = "impression"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_list_card_impressions,
+    COUNTIF(
+      event_name = "save"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+    ) AS list_card_saves,
+    COUNTIF(
+      event_name = "save"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+    ) AS organic_list_card_saves,
+    COUNTIF(
+      event_name = "save"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_list_card_saves,
+    COUNTIF(
+      event_name = "dismiss"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+    ) AS list_card_dismissals,
+    COUNTIF(
+      event_name = "dismiss"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") != "true"
+    ) AS organic_list_card_dismissals,
+    COUNTIF(
+      event_name = "dismiss"
+      AND mozfun.map.get_key(event_details, "is_list_card") = "true"
+      AND mozfun.map.get_key(event_details, "is_sponsored") = "true"
+    ) AS sponsored_list_card_dismissals,
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'pocket'
+    AND event_name IN ('impression', 'click', 'save', 'dismiss', 'thumb_voting_interaction')
+  GROUP BY
+    newtab_visit_id,
+    pocket_story_position,
+    pocket_tile_id,
+    pocket_recommendation_id,
+    pocket_received_rank,
+    pocket_scheduled_corpus_item_id,
+    pocket_topic,
+    pocket_matches_selected_topic
+),
+pocket_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        pocket_story_position,
+        pocket_tile_id,
+        pocket_recommendation_id,
+        pocket_impressions,
+        sponsored_pocket_impressions,
+        organic_pocket_impressions,
+        pocket_clicks,
+        sponsored_pocket_clicks,
+        organic_pocket_clicks,
+        pocket_saves,
+        sponsored_pocket_saves,
+        organic_pocket_saves,
+        sponsored_pocket_dismissals,
+        organic_pocket_dismissals,
+        pocket_thumbs_up,
+        pocket_thumbs_down,
+        pocket_received_rank,
+        pocket_scheduled_corpus_item_id,
+        pocket_topic,
+        pocket_matches_selected_topic,
+        list_card_clicks,
+        organic_list_card_clicks,
+        sponsored_list_card_clicks,
+        list_card_impressions,
+        organic_list_card_impressions,
+        sponsored_list_card_impressions,
+        list_card_saves,
+        organic_list_card_saves,
+        sponsored_list_card_saves,
+        list_card_dismissals,
+        organic_list_card_dismissals,
+        sponsored_list_card_dismissals
+      )
+    ) AS pocket_interactions
+  FROM
+    pocket_events
+  GROUP BY
+    newtab_visit_id
+),
+wallpaper_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper,
+    COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks,
+    COUNTIF(
+      event_name = 'wallpaper_click'
+      AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true"
+    ) AS wallpaper_clicks_had_previous_wallpaper,
+    COUNTIF(
+      event_name = 'wallpaper_click'
+      AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false"
+    ) AS wallpaper_clicks_first_selected_wallpaper,
+    COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks,
+    COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals,
+    COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'wallpaper_click',
+      'wallpaper_category_click',
+      'wallpaper_highlight_cta_clicks',
+      'wallpaper_highlight_dismissed'
+    )
+  GROUP BY
+    newtab_visit_id,
+    wallpaper_selected_wallpaper
+),
+wallpaper_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        wallpaper_selected_wallpaper,
+        wallpaper_clicks,
+        wallpaper_clicks_had_previous_wallpaper,
+        wallpaper_clicks_first_selected_wallpaper,
+        wallpaper_category_clicks,
+        wallpaper_highlight_dismissals,
+        wallpaper_highlight_cta_clicks
+      )
+    ) AS wallpaper_interactions
+  FROM
+    wallpaper_events
+  GROUP BY
+    newtab_visit_id
+),
+weather_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions,
+    COUNTIF(event_name = 'weather_location_selected') AS weather_widget_location_selected,
+    COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks,
+    COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors,
+    COUNTIF(
+      event_name = 'weather_change_display'
+      AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed"
+    ) AS weather_widget_change_display_to_detailed,
+    COUNTIF(
+      event_name = 'weather_change_display'
+      AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple"
+    ) AS weather_widget_change_display_to_simple
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'weather_impression',
+      'weather_open_provider_url',
+      'weather_load_error',
+      'weather_change_display',
+      'weather_location_selected'
+    )
+  GROUP BY
+    newtab_visit_id
+),
+weather_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        weather_widget_impressions,
+        weather_widget_location_selected,
+        weather_widget_clicks,
+        weather_widget_load_errors,
+        weather_widget_change_display_to_detailed,
+        weather_widget_change_display_to_simple
+      )
+    ) AS weather_interactions
+  FROM
+    weather_events
+  GROUP BY
+    newtab_visit_id
+),
+topic_selection_events AS (
+  SELECT
+    mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id,
+    mozfun.map.get_key(event_details, "previous_topics") AS previous_topics,
+    mozfun.map.get_key(event_details, "topics") AS topics,
+    COUNTIF(event_name = 'topic_selection_open') AS topic_selection_open,
+    COUNTIF(event_name = 'topic_selection_dismiss') AS topic_selection_dismiss,
+    COUNTIF(
+      event_name = 'topic_selection_topics_saved'
+      AND mozfun.map.get_key(event_details, "first_save") = "true"
+    ) AS topic_selection_topics_first_saved,
+    COUNTIF(
+      event_name = 'topic_selection_topics_saved'
+      AND mozfun.map.get_key(event_details, "first_save") != "true"
+    ) AS topic_selection_topics_updated,
+  FROM
+    events_unnested
+  WHERE
+    event_category = 'newtab'
+    AND event_name IN (
+      'topic_selection_dismiss',
+      'topic_selection_open',
+      'topic_selection_topics_saved'
+    )
+  GROUP BY
+    newtab_visit_id,
+    previous_topics,
+    topics
+),
+topic_selection_summary AS (
+  SELECT
+    newtab_visit_id,
+    ARRAY_AGG(
+      STRUCT(
+        previous_topics,
+        topics,
+        topic_selection_open,
+        topic_selection_dismiss,
+        topic_selection_topics_first_saved,
+        topic_selection_topics_updated
+      )
+    ) AS topic_selection_interactions
+  FROM
+    topic_selection_events
+  GROUP BY
+    newtab_visit_id
+),
+combined_newtab_activity AS (
+  SELECT
+    *
+  FROM
+    visit_metadata
+  LEFT JOIN
+    search_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    topsites_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    pocket_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    wallpaper_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    weather_summary
+    USING (newtab_visit_id)
+  LEFT JOIN
+    topic_selection_summary
+    USING (newtab_visit_id)
+  WHERE
+    -- Keep only rows with interactions, unless we receive a valid newtab.opened event.
+    -- This is meant to drop only interactions that only have a newtab.closed event on the same partition
+    -- (these are suspected to be from pre-loaded tabs)
+    newtab_open_source IS NOT NULL
+    OR search_interactions IS NOT NULL
+    OR topsite_tile_interactions IS NOT NULL
+    OR pocket_interactions IS NOT NULL
+    OR wallpaper_interactions IS NOT NULL
+    OR weather_interactions IS NOT NULL
+    OR topic_selection_interactions IS NOT NULL
+),
+client_profile_info AS (
+  SELECT
+    client_id AS legacy_telemetry_client_id,
+    ANY_VALUE(is_new_profile) AS is_new_profile,
+    ANY_VALUE(activity_segment) AS activity_segment
+  FROM
+    `moz-fx-data-shared-prod.telemetry_derived.unified_metrics_v1`
+  WHERE
+    submission_date = @submission_date
+  GROUP BY
+    client_id
+)
+SELECT
+  *,
+  CASE
+    WHEN (newtab_open_source = 'about:home')
+      AND (newtab_homepage_category = 'enabled')
+      THEN 'default'
+    WHEN (newtab_open_source = 'about:newtab')
+      AND (newtab_newtab_category = 'enabled')
+      THEN 'default'
+    WHEN newtab_open_source = 'about:welcome'
+      THEN 'welcome'
+    WHEN (newtab_open_source = 'about:home')
+      AND (newtab_homepage_category != 'enabled')
+      THEN 'home - non-default'
+    WHEN (newtab_open_source = 'about:newtab')
+      AND (newtab_newtab_category != 'enabled')
+      THEN 'newtab - non-default'
+    ELSE "other"
+  END AS default_ui,
+FROM
+  combined_newtab_activity
+LEFT JOIN
+  client_profile_info
+  USING (legacy_telemetry_client_id)

Link to full diff

Please sign in to comment.