From 4330322518124ec6a03c02d4bfa0578686d5bded Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Mon, 13 Jan 2025 17:00:08 -0800 Subject: [PATCH 1/8] DENG-6880 updates to include new fields and aggregations --- .../query.sql | 101 +++ .../newtab_clients_daily_v2/query.sql | 355 ++++++++++ .../newtab_visits_v2/query.sql | 621 ++++++++++++++++++ 3 files changed, 1077 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql new file mode 100644 index 00000000000..e3c5abcc0c9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql @@ -0,0 +1,101 @@ +SELECT + submission_date, + normalized_os, + normalized_os_version, + country_code, + locale, + channel, + browser_version, + browser_name, + default_search_engine, + default_private_search_engine, + pocket_is_signed_in, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + topsites_sponsored_enabled, + newtab_homepage_category, + newtab_newtab_category, + topsites_rows, + is_new_profile, + activity_segment, + COUNT(DISTINCT client_id) AS client_count, + COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count, + SUM(newtab_visit_count) AS newtab_visit_count, + SUM(visits_with_non_impression_engagement) AS visits_with_non_impression_engagement, + SUM(visits_with_non_search_engagement) AS visits_with_non_search_engagement, + SUM(searches) AS searches, + SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks, + SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions, + SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, + SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, + SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, + SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, + SUM(topsite_tile_clicks) AS topsite_tile_clicks, + SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks, + SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks, + SUM(topsite_tile_impressions) AS topsite_tile_impressions, + SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions, + SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions, + SUM(topsite_tile_dismissals) AS topsite_tile_dismissals, + SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals, + SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals, + SUM(pocket_impressions) AS pocket_impressions, + SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions, + SUM(organic_pocket_impressions) AS organic_pocket_impressions, + SUM(pocket_clicks) AS pocket_clicks, + SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks, + SUM(organic_pocket_clicks) AS organic_pocket_clicks, + SUM(pocket_saves) AS pocket_saves, + SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, + SUM(organic_pocket_saves) AS organic_pocket_saves, + +-- New additions + sum(pocket_thumb_voting_events) as pocket_thumb_voting_events, + sum(topic_selection_opened) as topic_selection_opened, + sum(topic_selection_updates) as topic_selection_updates, + sum(weather_widget_clicks) as weather_widget_clicks, + sum(weather_widget_change_display_to_detailed) as weather_widget_change_display_to_detailed, + sum(weather_widget_change_display_to_simple) as weather_widget_change_display_to_simple, + sum(wallpaper_clicks) as wallpaper_clicks, + sum(topic_selection_selected_topics_first_time) as topic_selection_selected_topics_first_time, + sum(sponsored_pocket_dismissals) as sponsored_pocket_dismissals, + sum(organic_pocket_dismissals) as organic_pocket_dismissals, + sum(topic_selection_dismissals) as topic_selection_dismissals, + + sum(visits_with_default_ui) as visits_with_default_ui, + sum(topic_selection_opened) as topic_selection_opened, + sum(topic_selection_opened) as topic_selection_opened, + sum(topic_selection_opened) as topic_selection_opened, + sum(topic_selection_opened) as topic_selection_opened, + sum(topic_selection_opened) as topic_selection_opened, + +FROM + `moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1` +WHERE + {% if is_init() %} + submission_date >= "2022-11-01" + {% else %} + submission_date = @submission_date + {% endif %} +GROUP BY + submission_date, + normalized_os, + normalized_os_version, + country_code, + locale, + channel, + browser_version, + browser_name, + default_search_engine, + default_private_search_engine, + pocket_is_signed_in, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + topsites_sponsored_enabled, + newtab_homepage_category, + newtab_newtab_category, + topsites_rows, + is_new_profile, + activity_segment diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql new file mode 100644 index 00000000000..b9f694f71e2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql @@ -0,0 +1,355 @@ +WITH visits_data_base AS ( + SELECT + client_id, + submission_date, + legacy_telemetry_client_id, + newtab_visit_id, + normalized_os, + normalized_os_version, + country_code, + locale, + channel, + browser_version, + browser_name, + default_search_engine, + default_private_search_engine, + pocket_is_signed_in, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + topsites_sponsored_enabled, + newtab_weather_widget_enabled, + newtab_homepage_category, + newtab_newtab_category, + topsites_rows, + experiments, + had_non_impression_engagement, + had_non_search_engagement, + is_new_profile, + activity_segment, + search_interactions, + topsite_tile_interactions, + pocket_interactions, + wallpaper_interactions, + weather_interactions, + topic_selection_interactions, + default_ui, + newtab_selected_topics, + profile_group_id, + topsites_sponsored_tiles_configured, + newtab_open_source + FROM + `moz-fx-data-shared-prod.telemetry_derived.newtab_visits_v1` + WHERE + submission_date = @submission_date +), +visits_data AS ( + SELECT + client_id, + submission_date, + ANY_VALUE(profile_group_id) AS profile_group_id, + ANY_VALUE(legacy_telemetry_client_id) AS legacy_telemetry_client_id, + COUNT(newtab_visit_id) AS newtab_visit_count, + ANY_VALUE(normalized_os) AS normalized_os, + ANY_VALUE(normalized_os_version) AS normalized_os_version, + ANY_VALUE(country_code) AS country_code, + ANY_VALUE(locale) AS locale, + ANY_VALUE(channel) AS channel, + ANY_VALUE(browser_version) AS browser_version, + ANY_VALUE(browser_name) AS browser_name, + ANY_VALUE(default_search_engine) AS default_search_engine, + ANY_VALUE(default_private_search_engine) AS default_private_search_engine, + LOGICAL_OR(pocket_is_signed_in) AS pocket_is_signed_in, + LOGICAL_AND(pocket_enabled) AS pocket_enabled, + LOGICAL_AND(pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled, + LOGICAL_AND(topsites_enabled) AS topsites_enabled, + LOGICAL_AND(topsites_sponsored_enabled) AS topsites_sponsored_enabled, + LOGICAL_AND(newtab_weather_widget_enabled) AS newtab_weather_widget_enabled, + ANY_VALUE(newtab_homepage_category) AS newtab_homepage_category, + ANY_VALUE(newtab_newtab_category) AS newtab_newtab_category, + ANY_VALUE(topsites_rows) AS topsites_rows, + ANY_VALUE(experiments) AS experiments, + COUNTIF(had_non_impression_engagement) AS visits_with_non_impression_engagement, + COUNTIF(had_non_search_engagement) AS visits_with_non_search_engagement, + COUNTIF(default_ui = "default") AS visits_with_default_ui, + COUNTIF(default_ui = "default" AND had_non_impression_engagement) AS visits_with_default_ui_with_non_impression_engagement, + COUNTIF(default_ui = "default" AND had_non_search_engagement) AS visits_with_default_ui_with_non_search_engagement, + COUNTIF(default_ui = "home - non-default" OR default_ui = "newtab - non-default") AS visits_with_non_default_ui, + LOGICAL_OR(is_new_profile) AS is_new_profile, + ANY_VALUE(activity_segment) AS activity_segment, + LOGICAL_OR(ARRAY_LENGTH(newtab_selected_topics) > 0) AS topic_preferences_set, + ANY_VALUE(topsites_sponsored_tiles_configured) AS topsites_sponsored_tiles_configured + FROM + visits_data_base + GROUP BY + client_id, + submission_date +), +search_data AS ( + SELECT + client_id, + SUM(searches) AS searches, + SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks, + SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions, + SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, + SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, + SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, + SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, + FROM + visits_data_base + CROSS JOIN + UNNEST(search_interactions) + GROUP BY + client_id +), +tiles_data AS ( + SELECT + client_id, + SUM(topsite_tile_clicks) AS topsite_tile_clicks, + SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks, + SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks, + SUM(topsite_tile_impressions) AS topsite_tile_impressions, + SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions, + SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions, + SUM(topsite_tile_dismissals) AS topsite_tile_dismissals, + SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals, + SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals, + FROM + visits_data_base + CROSS JOIN + UNNEST(topsite_tile_interactions) + GROUP BY + client_id +), +pocket_data AS ( + SELECT + client_id, + SUM(pocket_impressions) AS pocket_impressions, + SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions, + SUM(organic_pocket_impressions) AS organic_pocket_impressions, + SUM(pocket_clicks) AS pocket_clicks, + SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks, + SUM(organic_pocket_clicks) AS organic_pocket_clicks, + SUM(pocket_saves) AS pocket_saves, + SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, + SUM(organic_pocket_saves) AS organic_pocket_saves, + SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals, + SUM(organic_pocket_dismissals) AS organic_pocket_dismissals, + SUM(pocket_thumbs_up) AS pocket_thumbs_up, + SUM(pocket_thumbs_down) AS pocket_thumbs_down, + SUM(pocket_thumbs_down) + SUM(pocket_thumbs_up) AS pocket_thumb_voting_events, + SUM(list_card_clicks) AS list_card_clicks, + SUM(organic_list_card_clicks) AS organic_list_card_clicks, + SUM(sponsored_list_card_clicks) AS sponsored_list_card_clicks, + SUM(list_card_impressions) AS list_card_impressions, + SUM(organic_list_card_impressions) AS organic_list_card_impressions, + SUM(sponsored_list_card_impressions) AS sponsored_list_card_impressions, + SUM(list_card_saves) AS list_card_saves, + SUM(organic_list_card_saves) AS organic_list_card_saves, + SUM(sponsored_list_card_saves) AS sponsored_list_card_saves, + SUM(list_card_dismissals) AS list_card_dismissals, + SUM(organic_list_card_dismissals) AS organic_list_card_dismissals, + SUM(sponsored_list_card_dismissals) AS sponsored_list_card_dismissals, + FROM + visits_data_base + CROSS JOIN + UNNEST(pocket_interactions) + GROUP BY + client_id +), +wallpaper_data AS ( + SELECT + client_id, + SUM(wallpaper_clicks) AS wallpaper_clicks, + SUM(wallpaper_clicks_had_previous_wallpaper) AS wallpaper_clicks_had_previous_wallpaper, + SUM(wallpaper_clicks_first_selected_wallpaper) AS wallpaper_clicks_first_selected_wallpaper, + SUM(wallpaper_category_clicks) AS wallpaper_category_clicks, + SUM(wallpaper_highlight_dismissals) AS wallpaper_highlight_dismissals, + SUM(wallpaper_highlight_cta_clicks) AS wallpaper_highlight_cta_clicks + FROM + visits_data_base + CROSS JOIN + UNNEST(wallpaper_interactions) + GROUP BY + client_id +), +weather_data AS ( + SELECT + client_id, + SUM(weather_widget_impressions) AS weather_widget_impressions, + SUM(weather_widget_clicks) AS weather_widget_clicks, + SUM(weather_widget_load_errors) AS weather_widget_load_errors, + SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed, + SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple, + SUM(weather_widget_location_selected) AS weather_widget_location_selected + FROM + visits_data_base + CROSS JOIN + UNNEST(weather_interactions) + GROUP BY + client_id +), +topic_selection_data AS ( + SELECT + client_id, + LOGICAL_OR( + topic_selection_topics_first_saved > 0 + ) AS topic_selection_selected_topics_first_time, + SUM(topic_selection_topics_updated) AS topic_selection_updates, + SUM(topic_selection_open) AS topic_selection_opened, + SUM(topic_selection_dismiss) AS topic_selection_dismissals + FROM + visits_data_base + CROSS JOIN + UNNEST(topic_selection_interactions) + GROUP BY + client_id +), +joined AS ( + SELECT + visits_data.client_id, + visits_data.submission_date, + visits_data.legacy_telemetry_client_id, + visits_data.newtab_visit_count, + visits_data.normalized_os, + visits_data.normalized_os_version, + visits_data.country_code, + visits_data.locale, + visits_data.channel, + visits_data.browser_version, + visits_data.browser_name, + visits_data.default_search_engine, + visits_data.default_private_search_engine, + visits_data.pocket_is_signed_in, + visits_data.pocket_enabled, + visits_data.pocket_sponsored_stories_enabled, + visits_data.topsites_enabled, + visits_data.topsites_sponsored_enabled, + visits_data.newtab_weather_widget_enabled, + visits_data.newtab_homepage_category, + visits_data.newtab_newtab_category, + visits_data.topsites_rows, + visits_data.experiments, + visits_data.visits_with_non_impression_engagement, + visits_data.visits_with_non_search_engagement, + visits_data.visits_with_non_default_ui, + visits_data.is_new_profile, + visits_data.activity_segment, + -- COALESCE calls for visits where no interactions with a surface were performed and are all Null + COALESCE(search_data.searches, 0) AS searches, + COALESCE(search_data.tagged_search_ad_clicks, 0) AS tagged_search_ad_clicks, + COALESCE(search_data.tagged_search_ad_impressions, 0) AS tagged_search_ad_impressions, + COALESCE(search_data.follow_on_search_ad_clicks, 0) AS follow_on_search_ad_clicks, + COALESCE(search_data.follow_on_search_ad_impressions, 0) AS follow_on_search_ad_impressions, + COALESCE(search_data.tagged_follow_on_search_ad_clicks, 0) AS tagged_follow_on_search_ad_clicks, + COALESCE( + search_data.tagged_follow_on_search_ad_impressions, + 0 + ) AS tagged_follow_on_search_ad_impressions, + COALESCE(tiles_data.topsite_tile_clicks, 0) AS topsite_tile_clicks, + COALESCE(tiles_data.sponsored_topsite_tile_clicks, 0) AS sponsored_topsite_tile_clicks, + COALESCE(tiles_data.organic_topsite_tile_clicks, 0) AS organic_topsite_tile_clicks, + COALESCE(tiles_data.topsite_tile_impressions, 0) AS topsite_tile_impressions, + COALESCE( + tiles_data.sponsored_topsite_tile_impressions, + 0 + ) AS sponsored_topsite_tile_impressions, + COALESCE(tiles_data.organic_topsite_tile_impressions, 0) AS organic_topsite_tile_impressions, + COALESCE(tiles_data.topsite_tile_dismissals, 0) AS topsite_tile_dismissals, + COALESCE(tiles_data.sponsored_topsite_tile_dismissals, 0) AS sponsored_topsite_tile_dismissals, + COALESCE(tiles_data.organic_topsite_tile_dismissals, 0) AS organic_topsite_tile_dismissals, + COALESCE(pocket_data.pocket_impressions, 0) AS pocket_impressions, + COALESCE(pocket_data.sponsored_pocket_impressions, 0) AS sponsored_pocket_impressions, + COALESCE(pocket_data.organic_pocket_impressions, 0) AS organic_pocket_impressions, + COALESCE(pocket_data.pocket_clicks, 0) AS pocket_clicks, + COALESCE(pocket_data.sponsored_pocket_clicks, 0) AS sponsored_pocket_clicks, + COALESCE(pocket_data.organic_pocket_clicks, 0) AS organic_pocket_clicks, + COALESCE(pocket_data.pocket_saves, 0) AS pocket_saves, + COALESCE(pocket_data.sponsored_pocket_saves, 0) AS sponsored_pocket_saves, + COALESCE(pocket_data.organic_pocket_saves, 0) AS organic_pocket_saves, + COALESCE(wallpaper_clicks, 0) AS wallpaper_clicks, + COALESCE(wallpaper_clicks_had_previous_wallpaper, 0) AS wallpaper_clicks_had_previous_wallpaper, + COALESCE( + wallpaper_clicks_first_selected_wallpaper, + 0 + ) AS wallpaper_clicks_first_selected_wallpaper, + COALESCE(wallpaper_data.wallpaper_category_clicks, 0) AS wallpaper_category_clicks, + COALESCE(wallpaper_data.wallpaper_highlight_dismissals, 0) AS wallpaper_highlight_dismissals, + COALESCE(wallpaper_data.wallpaper_highlight_cta_clicks, 0) AS wallpaper_highlight_cta_clicks, + COALESCE(weather_data.weather_widget_impressions, 0) AS weather_widget_impressions, + COALESCE(weather_data.weather_widget_clicks, 0) AS weather_widget_clicks, + COALESCE(weather_data.weather_widget_load_errors, 0) AS weather_widget_load_errors, + COALESCE( + weather_data.weather_widget_change_display_to_detailed, + 0 + ) AS weather_widget_change_display_to_detailed, + COALESCE( + weather_data.weather_widget_change_display_to_simple, + 0 + ) AS weather_widget_change_display_to_simple, + COALESCE(weather_data.weather_widget_location_selected, 0) AS weather_widget_location_selected, + COALESCE(visits_data.visits_with_default_ui, 0) AS visits_with_default_ui, + COALESCE( + visits_data.visits_with_default_ui_with_non_impression_engagement, + 0 + ) AS visits_with_default_ui_with_non_impression_engagement, + COALESCE( + visits_data.visits_with_default_ui_with_non_search_engagement, + 0 + ) AS visits_with_default_ui_with_non_search_engagement, + COALESCE(visits_data.topic_preferences_set, FALSE) AS topic_preferences_set, + COALESCE(pocket_data.sponsored_pocket_dismissals, 0) AS sponsored_pocket_dismissals, + COALESCE(pocket_data.organic_pocket_dismissals, 0) AS organic_pocket_dismissals, + COALESCE(pocket_data.pocket_thumbs_up, 0) AS pocket_thumbs_up, + COALESCE(pocket_data.pocket_thumbs_down, 0) AS pocket_thumbs_down, + COALESCE(pocket_data.pocket_thumb_voting_events, 0) AS pocket_thumb_voting_events, + COALESCE( + topic_selection_data.topic_selection_selected_topics_first_time, + FALSE + ) AS topic_selection_selected_topics_first_time, + COALESCE(topic_selection_data.topic_selection_updates, 0) AS topic_selection_updates, + COALESCE(topic_selection_data.topic_selection_opened, 0) AS topic_selection_opened, + COALESCE(topic_selection_data.topic_selection_dismissals, 0) AS topic_selection_dismissals, + visits_data.profile_group_id AS profile_group_id, + visits_data.topsites_sponsored_tiles_configured AS topsites_sponsored_tiles_configured, + COALESCE(pocket_data.list_card_clicks, 0) AS list_card_clicks, + COALESCE(pocket_data.organic_list_card_clicks, 0) AS organic_list_card_clicks, + COALESCE(pocket_data.sponsored_list_card_clicks, 0) AS sponsored_list_card_clicks, + COALESCE(pocket_data.list_card_impressions, 0) AS list_card_impressions, + COALESCE(pocket_data.organic_list_card_impressions, 0) AS organic_list_card_impressions, + COALESCE(pocket_data.sponsored_list_card_impressions, 0) AS sponsored_list_card_impressions, + COALESCE(pocket_data.list_card_saves, 0) AS list_card_saves, + COALESCE(pocket_data.organic_list_card_saves, 0) AS organic_list_card_saves, + COALESCE(pocket_data.sponsored_list_card_saves, 0) AS sponsored_list_card_saves, + COALESCE(pocket_data.list_card_dismissals, 0) AS list_card_dismissals, + COALESCE(pocket_data.organic_list_card_dismissals, 0) AS organic_list_card_dismissals, + COALESCE(pocket_data.sponsored_list_card_dismissals, 0) AS sponsored_list_card_dismissals, + FROM + visits_data + LEFT JOIN + search_data + USING (client_id) + LEFT JOIN + tiles_data + USING (client_id) + LEFT JOIN + pocket_data + USING (client_id) + LEFT JOIN + weather_data + USING (client_id) + LEFT JOIN + wallpaper_data + USING (client_id) + LEFT JOIN + topic_selection_data + USING (client_id) +) +SELECT + *, + topsite_tile_clicks + pocket_clicks + pocket_saves + wallpaper_clicks + pocket_thumb_voting_events + topic_selection_updates + topic_selection_opened + CAST( + topic_selection_selected_topics_first_time AS INT + ) + weather_widget_clicks + weather_widget_change_display_to_detailed + weather_widget_change_display_to_simple + topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS non_search_engagement_count, + topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS newtab_dismissal_count +FROM + joined diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql new file mode 100644 index 00000000000..436aaf99677 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql @@ -0,0 +1,621 @@ +WITH events_unnested AS ( + SELECT + DATE(submission_timestamp) AS submission_date, + category AS event_category, + name AS event_name, + timestamp AS event_timestamp, + client_info, + METADATA, + normalized_os, + normalized_os_version, + normalized_country_code, + normalized_channel, + ping_info, + extra AS event_details, + metrics + FROM + -- https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/pings/newtab + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1`, + UNNEST(events) + WHERE + DATE(submission_timestamp) = @submission_date + AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') +), +visit_metadata AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + submission_date, + ANY_VALUE(client_info.client_id) AS client_id, + ANY_VALUE(metrics.uuid.legacy_telemetry_client_id) AS legacy_telemetry_client_id, + ANY_VALUE(metrics.uuid.legacy_telemetry_profile_group_id) AS profile_group_id, + ANY_VALUE(normalized_os) AS normalized_os, + ANY_VALUE(normalized_os_version) AS normalized_os_version, + ANY_VALUE(normalized_country_code) AS country_code, + ANY_VALUE(normalized_channel) AS channel, + ANY_VALUE(client_info.locale) AS locale, + ANY_VALUE(client_info.app_display_version) AS browser_version, + "Firefox Desktop" AS browser_name, + ANY_VALUE(metrics.string.search_engine_default_engine_id) AS default_search_engine, + ANY_VALUE(metrics.string.search_engine_private_engine_id) AS default_private_search_engine, + ANY_VALUE(metrics.boolean.pocket_is_signed_in) AS pocket_is_signed_in, + ANY_VALUE(metrics.boolean.pocket_enabled) AS pocket_enabled, + ANY_VALUE(metrics.boolean.pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled, + ANY_VALUE(metrics.boolean.topsites_enabled) AS topsites_enabled, + ANY_VALUE(metrics.boolean.topsites_sponsored_enabled) AS topsites_sponsored_enabled, + ANY_VALUE( + metrics.quantity.topsites_sponsored_tiles_configured + ) AS topsites_sponsored_tiles_configured, + ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, + ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, + ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, + ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, + ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, + ANY_VALUE(ping_info.experiments) AS experiments, + MIN(IF(event_name = "opened", event_timestamp, NULL)) AS newtab_visit_started_at, + MIN(IF(event_name = "closed", event_timestamp, NULL)) AS newtab_visit_ended_at, + ANY_VALUE( + IF(event_name = "opened", mozfun.map.get_key(event_details, "source"), NULL) + ) AS newtab_open_source, + LOGICAL_OR(event_name IN ("click", "issued", "save")) AS had_non_impression_engagement, + LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement, + ANY_VALUE(metrics.string_list.newtab_selected_topics) AS newtab_selected_topics, + ANY_VALUE( + IF( + event_name = "opened", + SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_height") AS INT), + NULL + ) + ) AS newtab_window_inner_height, + ANY_VALUE( + IF( + event_name = "opened", + SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_width") AS INT), + NULL + ) + ) AS newtab_window_inner_width, + FROM + events_unnested + GROUP BY + newtab_visit_id, + submission_date + HAVING + newtab_visit_id IS NOT NULL +), +search_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + `moz-fx-data-shared-prod`.udf.normalize_search_engine( + mozfun.map.get_key(event_details, "telemetry_id") + ) AS search_engine, + mozfun.map.get_key(event_details, "search_access_point") AS search_access_point, + COUNTIF(event_name = "issued") AS searches, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_tagged") = "true" + ) AS tagged_search_ad_impressions, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_follow_on") = "true" + ) AS follow_on_search_ad_impressions, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_follow_on") = "true" + AND mozfun.map.get_key(event_details, "is_tagged") = "true" + ) AS tagged_follow_on_search_ad_impressions, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_tagged") = "true" + ) AS tagged_search_ad_clicks, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_follow_on") = "true" + ) AS follow_on_search_ad_clicks, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_follow_on") = "true" + AND mozfun.map.get_key(event_details, "is_tagged") = "true" + ) AS tagged_follow_on_search_ad_clicks + FROM + events_unnested + WHERE + event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') + GROUP BY + newtab_visit_id, + search_engine, + search_access_point +), +search_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + search_engine, + search_access_point, + searches, + tagged_search_ad_clicks, + tagged_search_ad_impressions, + follow_on_search_ad_clicks, + follow_on_search_ad_impressions, + tagged_follow_on_search_ad_clicks, + tagged_follow_on_search_ad_impressions + ) + ) AS search_interactions + FROM + search_events + GROUP BY + newtab_visit_id +), +topsites_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS topsite_tile_position, + mozfun.map.get_key(event_details, "advertiser_name") AS topsite_tile_advertiser_name, + mozfun.map.get_key(event_details, "tile_id") AS topsite_tile_id, + JSON_EXTRACT(sov, "$.assigned") AS topsite_tile_assigned_sov_branch, + JSON_EXTRACT(sov, "$.chosen") AS topsite_tile_displayed_sov_branch, + COUNTIF(event_name = 'impression') AS topsite_tile_impressions, + COUNTIF(event_name = 'click') AS topsite_tile_clicks, + COUNTIF( + event_name = 'impression' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_topsite_tile_impressions, + COUNTIF( + event_name = 'impression' + AND mozfun.map.get_key(event_details, "is_sponsored") = "false" + ) AS organic_topsite_tile_impressions, + COUNTIF( + event_name = 'click' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_topsite_tile_clicks, + COUNTIF( + event_name = 'click' + AND mozfun.map.get_key(event_details, "is_sponsored") = "false" + ) AS organic_topsite_tile_clicks, + COUNTIF(event_name = 'dismiss') AS topsite_tile_dismissals, + COUNTIF( + event_name = 'dismiss' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_topsite_tile_dismissals, + COUNTIF( + event_name = 'dismiss' + AND mozfun.map.get_key(event_details, "is_sponsored") = "false" + ) AS organic_topsite_tile_dismissals, + FROM + events_unnested + LEFT JOIN + UNNEST(metrics.string_list.newtab_sov_allocation) sov + ON SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) = SAFE_CAST( + JSON_EXTRACT(sov, "$.pos") AS INT64 + ) + WHERE + event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') + GROUP BY + newtab_visit_id, + topsite_tile_position, + topsite_tile_advertiser_name, + topsite_tile_id, + topsite_tile_assigned_sov_branch, + topsite_tile_displayed_sov_branch +), +topsites_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + topsite_tile_advertiser_name, + topsite_tile_position, + topsite_tile_id, + topsite_tile_assigned_sov_branch, + topsite_tile_displayed_sov_branch, + topsite_tile_clicks, + sponsored_topsite_tile_clicks, + organic_topsite_tile_clicks, + topsite_tile_impressions, + sponsored_topsite_tile_impressions, + organic_topsite_tile_impressions, + topsite_tile_dismissals, + sponsored_topsite_tile_dismissals, + organic_topsite_tile_dismissals + ) + ) AS topsite_tile_interactions + FROM + topsites_events + GROUP BY + newtab_visit_id +), +pocket_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, + COUNTIF( + event_name = 'save' + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS pocket_saves, + COUNTIF( + event_name = 'click' + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS pocket_clicks, + COUNTIF( + event_name = 'impression' + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS pocket_impressions, + COUNTIF( + event_name = 'click' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS sponsored_pocket_clicks, + COUNTIF( + event_name = 'click' + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS organic_pocket_clicks, + COUNTIF( + event_name = 'impression' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS sponsored_pocket_impressions, + COUNTIF( + event_name = 'impression' + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS organic_pocket_impressions, + COUNTIF( + event_name = 'save' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS sponsored_pocket_saves, + COUNTIF( + event_name = 'save' + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS organic_pocket_saves, + COUNTIF( + event_name = 'dismiss' + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS sponsored_pocket_dismissals, + COUNTIF( + event_name = 'dismiss' + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" + ) AS organic_pocket_dismissals, + COUNTIF( + event_name = 'thumb_voting_interaction' + AND mozfun.map.get_key(event_details, "thumbs_up") = "true" + ) AS pocket_thumbs_up, + COUNTIF( + event_name = 'thumb_voting_interaction' + AND mozfun.map.get_key(event_details, "thumbs_down") = "true" + ) AS pocket_thumbs_down, + SAFE_CAST(mozfun.map.get_key(event_details, "received_rank") AS INT) AS pocket_received_rank, + mozfun.map.get_key( + event_details, + "scheduled_corpus_item_id" + ) AS pocket_scheduled_corpus_item_id, + mozfun.map.get_key(event_details, "topic") AS pocket_topic, + mozfun.map.get_key(event_details, "matches_selected_topic") AS pocket_matches_selected_topic, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + ) AS list_card_clicks, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + ) AS organic_list_card_clicks, + COUNTIF( + event_name = "click" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_list_card_clicks, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + ) AS list_card_impressions, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + ) AS organic_list_card_impressions, + COUNTIF( + event_name = "impression" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_list_card_impressions, + COUNTIF( + event_name = "save" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + ) AS list_card_saves, + COUNTIF( + event_name = "save" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + ) AS organic_list_card_saves, + COUNTIF( + event_name = "save" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_list_card_saves, + COUNTIF( + event_name = "dismiss" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + ) AS list_card_dismissals, + COUNTIF( + event_name = "dismiss" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") != "true" + ) AS organic_list_card_dismissals, + COUNTIF( + event_name = "dismiss" + AND mozfun.map.get_key(event_details, "is_list_card") = "true" + AND mozfun.map.get_key(event_details, "is_sponsored") = "true" + ) AS sponsored_list_card_dismissals, + FROM + events_unnested + WHERE + event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save', 'dismiss', 'thumb_voting_interaction') + GROUP BY + newtab_visit_id, + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, + pocket_received_rank, + pocket_scheduled_corpus_item_id, + pocket_topic, + pocket_matches_selected_topic +), +pocket_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, + pocket_impressions, + sponsored_pocket_impressions, + organic_pocket_impressions, + pocket_clicks, + sponsored_pocket_clicks, + organic_pocket_clicks, + pocket_saves, + sponsored_pocket_saves, + organic_pocket_saves, + sponsored_pocket_dismissals, + organic_pocket_dismissals, + pocket_thumbs_up, + pocket_thumbs_down, + pocket_received_rank, + pocket_scheduled_corpus_item_id, + pocket_topic, + pocket_matches_selected_topic, + list_card_clicks, + organic_list_card_clicks, + sponsored_list_card_clicks, + list_card_impressions, + organic_list_card_impressions, + sponsored_list_card_impressions, + list_card_saves, + organic_list_card_saves, + sponsored_list_card_saves, + list_card_dismissals, + organic_list_card_dismissals, + sponsored_list_card_dismissals + ) + ) AS pocket_interactions + FROM + pocket_events + GROUP BY + newtab_visit_id +), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_location_selected') AS weather_widget_location_selected, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display', + 'weather_location_selected' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_location_selected, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), +topic_selection_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "previous_topics") AS previous_topics, + mozfun.map.get_key(event_details, "topics") AS topics, + COUNTIF(event_name = 'topic_selection_open') AS topic_selection_open, + COUNTIF(event_name = 'topic_selection_dismiss') AS topic_selection_dismiss, + COUNTIF( + event_name = 'topic_selection_topics_saved' + AND mozfun.map.get_key(event_details, "first_save") = "true" + ) AS topic_selection_topics_first_saved, + COUNTIF( + event_name = 'topic_selection_topics_saved' + AND mozfun.map.get_key(event_details, "first_save") != "true" + ) AS topic_selection_topics_updated, + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'topic_selection_dismiss', + 'topic_selection_open', + 'topic_selection_topics_saved' + ) + GROUP BY + newtab_visit_id, + previous_topics, + topics +), +topic_selection_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + previous_topics, + topics, + topic_selection_open, + topic_selection_dismiss, + topic_selection_topics_first_saved, + topic_selection_topics_updated + ) + ) AS topic_selection_interactions + FROM + topic_selection_events + GROUP BY + newtab_visit_id +), +combined_newtab_activity AS ( + SELECT + * + FROM + visit_metadata + LEFT JOIN + search_summary + USING (newtab_visit_id) + LEFT JOIN + topsites_summary + USING (newtab_visit_id) + LEFT JOIN + pocket_summary + USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) + LEFT JOIN + topic_selection_summary + USING (newtab_visit_id) + WHERE + -- Keep only rows with interactions, unless we receive a valid newtab.opened event. + -- This is meant to drop only interactions that only have a newtab.closed event on the same partition + -- (these are suspected to be from pre-loaded tabs) + newtab_open_source IS NOT NULL + OR search_interactions IS NOT NULL + OR topsite_tile_interactions IS NOT NULL + OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL + OR topic_selection_interactions IS NOT NULL +), +client_profile_info AS ( + SELECT + client_id AS legacy_telemetry_client_id, + ANY_VALUE(is_new_profile) AS is_new_profile, + ANY_VALUE(activity_segment) AS activity_segment + FROM + `moz-fx-data-shared-prod.telemetry_derived.unified_metrics_v1` + WHERE + submission_date = @submission_date + GROUP BY + client_id +) +SELECT + *, + CASE + WHEN (newtab_open_source = 'about:home') AND (newtab_homepage_category = 'enabled') THEN 'default' + WHEN (newtab_open_source = 'about:newtab') AND (newtab_newtab_category = 'enabled') THEN 'default' + WHEN newtab_open_source = 'about:welcome' THEN 'welcome' + WHEN (newtab_open_source = 'about:home') AND (newtab_homepage_category != 'enabled') THEN 'home - non-default' + WHEN (newtab_open_source = 'about:newtab') AND (newtab_newtab_category != 'enabled') THEN 'newtab - non-default' + ELSE "other" + END AS default_ui, +FROM + combined_newtab_activity +LEFT JOIN + client_profile_info + USING (legacy_telemetry_client_id) From 5593c06faebf2541e9ffea75aad1c949bab1ede3 Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Tue, 21 Jan 2025 10:19:05 -0800 Subject: [PATCH 2/8] adding interactions aggregations --- .../query.sql | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql new file mode 100644 index 00000000000..ecce09ad6b3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql @@ -0,0 +1,84 @@ +select + submission_date, + search_engine, + search_access_point, + pocket_story_position, + default_private_search_engine, + default_search_engine, + os, + os_version, + country_code, + channel, + browser_version, + browser_name, + newtab_open_source, + pocket_is_signed_in, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + newtab_homepage_category, + newtab_newtab_category, + newtab_search_enabled, + topsites_rows, + is_new_profile, + activity_segment, + + COUNT(DISTINCT client_id) AS client_count, + COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count, + COUNT(DISTINCT newtab_visit_id) AS newtab_interactions_visits, + -- Topsite + sum(topsite_clicks) AS topsite_clicks, + sum(sponsored_topsite_clicks) AS sponsored_topsite_clicks, + sum(organic_topsite_clicks) AS organic_topsite_clicks, + sum(topsite_impressions) AS topsite_impressions, + sum(sponsored_topsite_impressions) AS sponsored_topsite_impressions, + sum(organic_topsite_impressions) AS organic_topsite_impressions, + sum(topsite_dismissals) AS topsite_dismissals, + sum(sponsored_topsite_dismissals) AS sponsored_topsite_dismissals, + sum(organic_topsite_dismissals) AS organic_topsite_dismissals, + -- Search + sum(searches) AS searches, + sum(tagged_search_ad_clicks) AS tagged_search_ad_clicks, + sum(tagged_search_ad_impressions) AS tagged_search_ad_impressions, + sum(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, + sum(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, + sum(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, + sum(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, + -- Pocket + sum(pocket_impressions) AS pocket_impressions, + sum(sponsored_pocket_impressions) AS sponsored_pocket_impressions, + sum(organic_pocket_impressions) AS organic_pocket_impressions, + sum(pocket_clicks) AS pocket_clicks, + sum(sponsored_pocket_clicks) AS sponsored_pocket_clicks, + sum(organic_pocket_clicks) AS organic_pocket_clicks, + sum(pocket_saves) AS pocket_saves, + sum(sponsored_pocket_saves) AS sponsored_pocket_saves, + sum(organic_pocket_saves) AS organic_pocket_saves, +from + `moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1` +WHERE + submission_date = @submission_date +GROUP BY + submission_date, + search_engine, + search_access_point, + pocket_story_position, + default_private_search_engine, + default_search_engine, + os, + os_version, + country_code, + channel, + browser_version, + browser_name, + newtab_open_source, + pocket_is_signed_in, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + newtab_homepage_category, + newtab_newtab_category, + newtab_search_enabled, + topsites_rows, + is_new_profile, + activity_segment From 5a0d694b5988e467174dac7186393738f4e02ad2 Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Wed, 22 Jan 2025 16:50:23 -0800 Subject: [PATCH 3/8] removing unwanted aggs --- .../newtab_clients_daily_v2/query.sql | 355 ---------- .../newtab_visits_v2/query.sql | 621 ------------------ 2 files changed, 976 deletions(-) delete mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql delete mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql deleted file mode 100644 index b9f694f71e2..00000000000 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_v2/query.sql +++ /dev/null @@ -1,355 +0,0 @@ -WITH visits_data_base AS ( - SELECT - client_id, - submission_date, - legacy_telemetry_client_id, - newtab_visit_id, - normalized_os, - normalized_os_version, - country_code, - locale, - channel, - browser_version, - browser_name, - default_search_engine, - default_private_search_engine, - pocket_is_signed_in, - pocket_enabled, - pocket_sponsored_stories_enabled, - topsites_enabled, - topsites_sponsored_enabled, - newtab_weather_widget_enabled, - newtab_homepage_category, - newtab_newtab_category, - topsites_rows, - experiments, - had_non_impression_engagement, - had_non_search_engagement, - is_new_profile, - activity_segment, - search_interactions, - topsite_tile_interactions, - pocket_interactions, - wallpaper_interactions, - weather_interactions, - topic_selection_interactions, - default_ui, - newtab_selected_topics, - profile_group_id, - topsites_sponsored_tiles_configured, - newtab_open_source - FROM - `moz-fx-data-shared-prod.telemetry_derived.newtab_visits_v1` - WHERE - submission_date = @submission_date -), -visits_data AS ( - SELECT - client_id, - submission_date, - ANY_VALUE(profile_group_id) AS profile_group_id, - ANY_VALUE(legacy_telemetry_client_id) AS legacy_telemetry_client_id, - COUNT(newtab_visit_id) AS newtab_visit_count, - ANY_VALUE(normalized_os) AS normalized_os, - ANY_VALUE(normalized_os_version) AS normalized_os_version, - ANY_VALUE(country_code) AS country_code, - ANY_VALUE(locale) AS locale, - ANY_VALUE(channel) AS channel, - ANY_VALUE(browser_version) AS browser_version, - ANY_VALUE(browser_name) AS browser_name, - ANY_VALUE(default_search_engine) AS default_search_engine, - ANY_VALUE(default_private_search_engine) AS default_private_search_engine, - LOGICAL_OR(pocket_is_signed_in) AS pocket_is_signed_in, - LOGICAL_AND(pocket_enabled) AS pocket_enabled, - LOGICAL_AND(pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled, - LOGICAL_AND(topsites_enabled) AS topsites_enabled, - LOGICAL_AND(topsites_sponsored_enabled) AS topsites_sponsored_enabled, - LOGICAL_AND(newtab_weather_widget_enabled) AS newtab_weather_widget_enabled, - ANY_VALUE(newtab_homepage_category) AS newtab_homepage_category, - ANY_VALUE(newtab_newtab_category) AS newtab_newtab_category, - ANY_VALUE(topsites_rows) AS topsites_rows, - ANY_VALUE(experiments) AS experiments, - COUNTIF(had_non_impression_engagement) AS visits_with_non_impression_engagement, - COUNTIF(had_non_search_engagement) AS visits_with_non_search_engagement, - COUNTIF(default_ui = "default") AS visits_with_default_ui, - COUNTIF(default_ui = "default" AND had_non_impression_engagement) AS visits_with_default_ui_with_non_impression_engagement, - COUNTIF(default_ui = "default" AND had_non_search_engagement) AS visits_with_default_ui_with_non_search_engagement, - COUNTIF(default_ui = "home - non-default" OR default_ui = "newtab - non-default") AS visits_with_non_default_ui, - LOGICAL_OR(is_new_profile) AS is_new_profile, - ANY_VALUE(activity_segment) AS activity_segment, - LOGICAL_OR(ARRAY_LENGTH(newtab_selected_topics) > 0) AS topic_preferences_set, - ANY_VALUE(topsites_sponsored_tiles_configured) AS topsites_sponsored_tiles_configured - FROM - visits_data_base - GROUP BY - client_id, - submission_date -), -search_data AS ( - SELECT - client_id, - SUM(searches) AS searches, - SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks, - SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions, - SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, - SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, - SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, - SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, - FROM - visits_data_base - CROSS JOIN - UNNEST(search_interactions) - GROUP BY - client_id -), -tiles_data AS ( - SELECT - client_id, - SUM(topsite_tile_clicks) AS topsite_tile_clicks, - SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks, - SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks, - SUM(topsite_tile_impressions) AS topsite_tile_impressions, - SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions, - SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions, - SUM(topsite_tile_dismissals) AS topsite_tile_dismissals, - SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals, - SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals, - FROM - visits_data_base - CROSS JOIN - UNNEST(topsite_tile_interactions) - GROUP BY - client_id -), -pocket_data AS ( - SELECT - client_id, - SUM(pocket_impressions) AS pocket_impressions, - SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions, - SUM(organic_pocket_impressions) AS organic_pocket_impressions, - SUM(pocket_clicks) AS pocket_clicks, - SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks, - SUM(organic_pocket_clicks) AS organic_pocket_clicks, - SUM(pocket_saves) AS pocket_saves, - SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, - SUM(organic_pocket_saves) AS organic_pocket_saves, - SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals, - SUM(organic_pocket_dismissals) AS organic_pocket_dismissals, - SUM(pocket_thumbs_up) AS pocket_thumbs_up, - SUM(pocket_thumbs_down) AS pocket_thumbs_down, - SUM(pocket_thumbs_down) + SUM(pocket_thumbs_up) AS pocket_thumb_voting_events, - SUM(list_card_clicks) AS list_card_clicks, - SUM(organic_list_card_clicks) AS organic_list_card_clicks, - SUM(sponsored_list_card_clicks) AS sponsored_list_card_clicks, - SUM(list_card_impressions) AS list_card_impressions, - SUM(organic_list_card_impressions) AS organic_list_card_impressions, - SUM(sponsored_list_card_impressions) AS sponsored_list_card_impressions, - SUM(list_card_saves) AS list_card_saves, - SUM(organic_list_card_saves) AS organic_list_card_saves, - SUM(sponsored_list_card_saves) AS sponsored_list_card_saves, - SUM(list_card_dismissals) AS list_card_dismissals, - SUM(organic_list_card_dismissals) AS organic_list_card_dismissals, - SUM(sponsored_list_card_dismissals) AS sponsored_list_card_dismissals, - FROM - visits_data_base - CROSS JOIN - UNNEST(pocket_interactions) - GROUP BY - client_id -), -wallpaper_data AS ( - SELECT - client_id, - SUM(wallpaper_clicks) AS wallpaper_clicks, - SUM(wallpaper_clicks_had_previous_wallpaper) AS wallpaper_clicks_had_previous_wallpaper, - SUM(wallpaper_clicks_first_selected_wallpaper) AS wallpaper_clicks_first_selected_wallpaper, - SUM(wallpaper_category_clicks) AS wallpaper_category_clicks, - SUM(wallpaper_highlight_dismissals) AS wallpaper_highlight_dismissals, - SUM(wallpaper_highlight_cta_clicks) AS wallpaper_highlight_cta_clicks - FROM - visits_data_base - CROSS JOIN - UNNEST(wallpaper_interactions) - GROUP BY - client_id -), -weather_data AS ( - SELECT - client_id, - SUM(weather_widget_impressions) AS weather_widget_impressions, - SUM(weather_widget_clicks) AS weather_widget_clicks, - SUM(weather_widget_load_errors) AS weather_widget_load_errors, - SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed, - SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple, - SUM(weather_widget_location_selected) AS weather_widget_location_selected - FROM - visits_data_base - CROSS JOIN - UNNEST(weather_interactions) - GROUP BY - client_id -), -topic_selection_data AS ( - SELECT - client_id, - LOGICAL_OR( - topic_selection_topics_first_saved > 0 - ) AS topic_selection_selected_topics_first_time, - SUM(topic_selection_topics_updated) AS topic_selection_updates, - SUM(topic_selection_open) AS topic_selection_opened, - SUM(topic_selection_dismiss) AS topic_selection_dismissals - FROM - visits_data_base - CROSS JOIN - UNNEST(topic_selection_interactions) - GROUP BY - client_id -), -joined AS ( - SELECT - visits_data.client_id, - visits_data.submission_date, - visits_data.legacy_telemetry_client_id, - visits_data.newtab_visit_count, - visits_data.normalized_os, - visits_data.normalized_os_version, - visits_data.country_code, - visits_data.locale, - visits_data.channel, - visits_data.browser_version, - visits_data.browser_name, - visits_data.default_search_engine, - visits_data.default_private_search_engine, - visits_data.pocket_is_signed_in, - visits_data.pocket_enabled, - visits_data.pocket_sponsored_stories_enabled, - visits_data.topsites_enabled, - visits_data.topsites_sponsored_enabled, - visits_data.newtab_weather_widget_enabled, - visits_data.newtab_homepage_category, - visits_data.newtab_newtab_category, - visits_data.topsites_rows, - visits_data.experiments, - visits_data.visits_with_non_impression_engagement, - visits_data.visits_with_non_search_engagement, - visits_data.visits_with_non_default_ui, - visits_data.is_new_profile, - visits_data.activity_segment, - -- COALESCE calls for visits where no interactions with a surface were performed and are all Null - COALESCE(search_data.searches, 0) AS searches, - COALESCE(search_data.tagged_search_ad_clicks, 0) AS tagged_search_ad_clicks, - COALESCE(search_data.tagged_search_ad_impressions, 0) AS tagged_search_ad_impressions, - COALESCE(search_data.follow_on_search_ad_clicks, 0) AS follow_on_search_ad_clicks, - COALESCE(search_data.follow_on_search_ad_impressions, 0) AS follow_on_search_ad_impressions, - COALESCE(search_data.tagged_follow_on_search_ad_clicks, 0) AS tagged_follow_on_search_ad_clicks, - COALESCE( - search_data.tagged_follow_on_search_ad_impressions, - 0 - ) AS tagged_follow_on_search_ad_impressions, - COALESCE(tiles_data.topsite_tile_clicks, 0) AS topsite_tile_clicks, - COALESCE(tiles_data.sponsored_topsite_tile_clicks, 0) AS sponsored_topsite_tile_clicks, - COALESCE(tiles_data.organic_topsite_tile_clicks, 0) AS organic_topsite_tile_clicks, - COALESCE(tiles_data.topsite_tile_impressions, 0) AS topsite_tile_impressions, - COALESCE( - tiles_data.sponsored_topsite_tile_impressions, - 0 - ) AS sponsored_topsite_tile_impressions, - COALESCE(tiles_data.organic_topsite_tile_impressions, 0) AS organic_topsite_tile_impressions, - COALESCE(tiles_data.topsite_tile_dismissals, 0) AS topsite_tile_dismissals, - COALESCE(tiles_data.sponsored_topsite_tile_dismissals, 0) AS sponsored_topsite_tile_dismissals, - COALESCE(tiles_data.organic_topsite_tile_dismissals, 0) AS organic_topsite_tile_dismissals, - COALESCE(pocket_data.pocket_impressions, 0) AS pocket_impressions, - COALESCE(pocket_data.sponsored_pocket_impressions, 0) AS sponsored_pocket_impressions, - COALESCE(pocket_data.organic_pocket_impressions, 0) AS organic_pocket_impressions, - COALESCE(pocket_data.pocket_clicks, 0) AS pocket_clicks, - COALESCE(pocket_data.sponsored_pocket_clicks, 0) AS sponsored_pocket_clicks, - COALESCE(pocket_data.organic_pocket_clicks, 0) AS organic_pocket_clicks, - COALESCE(pocket_data.pocket_saves, 0) AS pocket_saves, - COALESCE(pocket_data.sponsored_pocket_saves, 0) AS sponsored_pocket_saves, - COALESCE(pocket_data.organic_pocket_saves, 0) AS organic_pocket_saves, - COALESCE(wallpaper_clicks, 0) AS wallpaper_clicks, - COALESCE(wallpaper_clicks_had_previous_wallpaper, 0) AS wallpaper_clicks_had_previous_wallpaper, - COALESCE( - wallpaper_clicks_first_selected_wallpaper, - 0 - ) AS wallpaper_clicks_first_selected_wallpaper, - COALESCE(wallpaper_data.wallpaper_category_clicks, 0) AS wallpaper_category_clicks, - COALESCE(wallpaper_data.wallpaper_highlight_dismissals, 0) AS wallpaper_highlight_dismissals, - COALESCE(wallpaper_data.wallpaper_highlight_cta_clicks, 0) AS wallpaper_highlight_cta_clicks, - COALESCE(weather_data.weather_widget_impressions, 0) AS weather_widget_impressions, - COALESCE(weather_data.weather_widget_clicks, 0) AS weather_widget_clicks, - COALESCE(weather_data.weather_widget_load_errors, 0) AS weather_widget_load_errors, - COALESCE( - weather_data.weather_widget_change_display_to_detailed, - 0 - ) AS weather_widget_change_display_to_detailed, - COALESCE( - weather_data.weather_widget_change_display_to_simple, - 0 - ) AS weather_widget_change_display_to_simple, - COALESCE(weather_data.weather_widget_location_selected, 0) AS weather_widget_location_selected, - COALESCE(visits_data.visits_with_default_ui, 0) AS visits_with_default_ui, - COALESCE( - visits_data.visits_with_default_ui_with_non_impression_engagement, - 0 - ) AS visits_with_default_ui_with_non_impression_engagement, - COALESCE( - visits_data.visits_with_default_ui_with_non_search_engagement, - 0 - ) AS visits_with_default_ui_with_non_search_engagement, - COALESCE(visits_data.topic_preferences_set, FALSE) AS topic_preferences_set, - COALESCE(pocket_data.sponsored_pocket_dismissals, 0) AS sponsored_pocket_dismissals, - COALESCE(pocket_data.organic_pocket_dismissals, 0) AS organic_pocket_dismissals, - COALESCE(pocket_data.pocket_thumbs_up, 0) AS pocket_thumbs_up, - COALESCE(pocket_data.pocket_thumbs_down, 0) AS pocket_thumbs_down, - COALESCE(pocket_data.pocket_thumb_voting_events, 0) AS pocket_thumb_voting_events, - COALESCE( - topic_selection_data.topic_selection_selected_topics_first_time, - FALSE - ) AS topic_selection_selected_topics_first_time, - COALESCE(topic_selection_data.topic_selection_updates, 0) AS topic_selection_updates, - COALESCE(topic_selection_data.topic_selection_opened, 0) AS topic_selection_opened, - COALESCE(topic_selection_data.topic_selection_dismissals, 0) AS topic_selection_dismissals, - visits_data.profile_group_id AS profile_group_id, - visits_data.topsites_sponsored_tiles_configured AS topsites_sponsored_tiles_configured, - COALESCE(pocket_data.list_card_clicks, 0) AS list_card_clicks, - COALESCE(pocket_data.organic_list_card_clicks, 0) AS organic_list_card_clicks, - COALESCE(pocket_data.sponsored_list_card_clicks, 0) AS sponsored_list_card_clicks, - COALESCE(pocket_data.list_card_impressions, 0) AS list_card_impressions, - COALESCE(pocket_data.organic_list_card_impressions, 0) AS organic_list_card_impressions, - COALESCE(pocket_data.sponsored_list_card_impressions, 0) AS sponsored_list_card_impressions, - COALESCE(pocket_data.list_card_saves, 0) AS list_card_saves, - COALESCE(pocket_data.organic_list_card_saves, 0) AS organic_list_card_saves, - COALESCE(pocket_data.sponsored_list_card_saves, 0) AS sponsored_list_card_saves, - COALESCE(pocket_data.list_card_dismissals, 0) AS list_card_dismissals, - COALESCE(pocket_data.organic_list_card_dismissals, 0) AS organic_list_card_dismissals, - COALESCE(pocket_data.sponsored_list_card_dismissals, 0) AS sponsored_list_card_dismissals, - FROM - visits_data - LEFT JOIN - search_data - USING (client_id) - LEFT JOIN - tiles_data - USING (client_id) - LEFT JOIN - pocket_data - USING (client_id) - LEFT JOIN - weather_data - USING (client_id) - LEFT JOIN - wallpaper_data - USING (client_id) - LEFT JOIN - topic_selection_data - USING (client_id) -) -SELECT - *, - topsite_tile_clicks + pocket_clicks + pocket_saves + wallpaper_clicks + pocket_thumb_voting_events + topic_selection_updates + topic_selection_opened + CAST( - topic_selection_selected_topics_first_time AS INT - ) + weather_widget_clicks + weather_widget_change_display_to_detailed + weather_widget_change_display_to_simple + topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS non_search_engagement_count, - topsite_tile_dismissals + organic_pocket_dismissals + sponsored_pocket_dismissals + topic_selection_dismissals AS newtab_dismissal_count -FROM - joined diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql deleted file mode 100644 index 436aaf99677..00000000000 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v2/query.sql +++ /dev/null @@ -1,621 +0,0 @@ -WITH events_unnested AS ( - SELECT - DATE(submission_timestamp) AS submission_date, - category AS event_category, - name AS event_name, - timestamp AS event_timestamp, - client_info, - METADATA, - normalized_os, - normalized_os_version, - normalized_country_code, - normalized_channel, - ping_info, - extra AS event_details, - metrics - FROM - -- https://dictionary.telemetry.mozilla.org/apps/firefox_desktop/pings/newtab - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1`, - UNNEST(events) - WHERE - DATE(submission_timestamp) = @submission_date - AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') -), -visit_metadata AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - submission_date, - ANY_VALUE(client_info.client_id) AS client_id, - ANY_VALUE(metrics.uuid.legacy_telemetry_client_id) AS legacy_telemetry_client_id, - ANY_VALUE(metrics.uuid.legacy_telemetry_profile_group_id) AS profile_group_id, - ANY_VALUE(normalized_os) AS normalized_os, - ANY_VALUE(normalized_os_version) AS normalized_os_version, - ANY_VALUE(normalized_country_code) AS country_code, - ANY_VALUE(normalized_channel) AS channel, - ANY_VALUE(client_info.locale) AS locale, - ANY_VALUE(client_info.app_display_version) AS browser_version, - "Firefox Desktop" AS browser_name, - ANY_VALUE(metrics.string.search_engine_default_engine_id) AS default_search_engine, - ANY_VALUE(metrics.string.search_engine_private_engine_id) AS default_private_search_engine, - ANY_VALUE(metrics.boolean.pocket_is_signed_in) AS pocket_is_signed_in, - ANY_VALUE(metrics.boolean.pocket_enabled) AS pocket_enabled, - ANY_VALUE(metrics.boolean.pocket_sponsored_stories_enabled) AS pocket_sponsored_stories_enabled, - ANY_VALUE(metrics.boolean.topsites_enabled) AS topsites_enabled, - ANY_VALUE(metrics.boolean.topsites_sponsored_enabled) AS topsites_sponsored_enabled, - ANY_VALUE( - metrics.quantity.topsites_sponsored_tiles_configured - ) AS topsites_sponsored_tiles_configured, - ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, - ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, - ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, - ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, - ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, - ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, - ANY_VALUE(ping_info.experiments) AS experiments, - MIN(IF(event_name = "opened", event_timestamp, NULL)) AS newtab_visit_started_at, - MIN(IF(event_name = "closed", event_timestamp, NULL)) AS newtab_visit_ended_at, - ANY_VALUE( - IF(event_name = "opened", mozfun.map.get_key(event_details, "source"), NULL) - ) AS newtab_open_source, - LOGICAL_OR(event_name IN ("click", "issued", "save")) AS had_non_impression_engagement, - LOGICAL_OR(event_name IN ("click", "save")) AS had_non_search_engagement, - ANY_VALUE(metrics.string_list.newtab_selected_topics) AS newtab_selected_topics, - ANY_VALUE( - IF( - event_name = "opened", - SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_height") AS INT), - NULL - ) - ) AS newtab_window_inner_height, - ANY_VALUE( - IF( - event_name = "opened", - SAFE_CAST(mozfun.map.get_key(event_details, "window_inner_width") AS INT), - NULL - ) - ) AS newtab_window_inner_width, - FROM - events_unnested - GROUP BY - newtab_visit_id, - submission_date - HAVING - newtab_visit_id IS NOT NULL -), -search_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - `moz-fx-data-shared-prod`.udf.normalize_search_engine( - mozfun.map.get_key(event_details, "telemetry_id") - ) AS search_engine, - mozfun.map.get_key(event_details, "search_access_point") AS search_access_point, - COUNTIF(event_name = "issued") AS searches, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_tagged") = "true" - ) AS tagged_search_ad_impressions, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_follow_on") = "true" - ) AS follow_on_search_ad_impressions, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_follow_on") = "true" - AND mozfun.map.get_key(event_details, "is_tagged") = "true" - ) AS tagged_follow_on_search_ad_impressions, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_tagged") = "true" - ) AS tagged_search_ad_clicks, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_follow_on") = "true" - ) AS follow_on_search_ad_clicks, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_follow_on") = "true" - AND mozfun.map.get_key(event_details, "is_tagged") = "true" - ) AS tagged_follow_on_search_ad_clicks - FROM - events_unnested - WHERE - event_category IN ('newtab.search', 'newtab.search.ad') - AND event_name IN ('click', 'impression', 'issued') - GROUP BY - newtab_visit_id, - search_engine, - search_access_point -), -search_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - search_engine, - search_access_point, - searches, - tagged_search_ad_clicks, - tagged_search_ad_impressions, - follow_on_search_ad_clicks, - follow_on_search_ad_impressions, - tagged_follow_on_search_ad_clicks, - tagged_follow_on_search_ad_impressions - ) - ) AS search_interactions - FROM - search_events - GROUP BY - newtab_visit_id -), -topsites_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS topsite_tile_position, - mozfun.map.get_key(event_details, "advertiser_name") AS topsite_tile_advertiser_name, - mozfun.map.get_key(event_details, "tile_id") AS topsite_tile_id, - JSON_EXTRACT(sov, "$.assigned") AS topsite_tile_assigned_sov_branch, - JSON_EXTRACT(sov, "$.chosen") AS topsite_tile_displayed_sov_branch, - COUNTIF(event_name = 'impression') AS topsite_tile_impressions, - COUNTIF(event_name = 'click') AS topsite_tile_clicks, - COUNTIF( - event_name = 'impression' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_topsite_tile_impressions, - COUNTIF( - event_name = 'impression' - AND mozfun.map.get_key(event_details, "is_sponsored") = "false" - ) AS organic_topsite_tile_impressions, - COUNTIF( - event_name = 'click' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_topsite_tile_clicks, - COUNTIF( - event_name = 'click' - AND mozfun.map.get_key(event_details, "is_sponsored") = "false" - ) AS organic_topsite_tile_clicks, - COUNTIF(event_name = 'dismiss') AS topsite_tile_dismissals, - COUNTIF( - event_name = 'dismiss' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_topsite_tile_dismissals, - COUNTIF( - event_name = 'dismiss' - AND mozfun.map.get_key(event_details, "is_sponsored") = "false" - ) AS organic_topsite_tile_dismissals, - FROM - events_unnested - LEFT JOIN - UNNEST(metrics.string_list.newtab_sov_allocation) sov - ON SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) = SAFE_CAST( - JSON_EXTRACT(sov, "$.pos") AS INT64 - ) - WHERE - event_category = 'topsites' - AND event_name IN ('dismiss', 'click', 'impression') - GROUP BY - newtab_visit_id, - topsite_tile_position, - topsite_tile_advertiser_name, - topsite_tile_id, - topsite_tile_assigned_sov_branch, - topsite_tile_displayed_sov_branch -), -topsites_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - topsite_tile_advertiser_name, - topsite_tile_position, - topsite_tile_id, - topsite_tile_assigned_sov_branch, - topsite_tile_displayed_sov_branch, - topsite_tile_clicks, - sponsored_topsite_tile_clicks, - organic_topsite_tile_clicks, - topsite_tile_impressions, - sponsored_topsite_tile_impressions, - organic_topsite_tile_impressions, - topsite_tile_dismissals, - sponsored_topsite_tile_dismissals, - organic_topsite_tile_dismissals - ) - ) AS topsite_tile_interactions - FROM - topsites_events - GROUP BY - newtab_visit_id -), -pocket_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, - mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, - mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, - COUNTIF( - event_name = 'save' - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS pocket_saves, - COUNTIF( - event_name = 'click' - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS pocket_clicks, - COUNTIF( - event_name = 'impression' - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS pocket_impressions, - COUNTIF( - event_name = 'click' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS sponsored_pocket_clicks, - COUNTIF( - event_name = 'click' - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS organic_pocket_clicks, - COUNTIF( - event_name = 'impression' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS sponsored_pocket_impressions, - COUNTIF( - event_name = 'impression' - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS organic_pocket_impressions, - COUNTIF( - event_name = 'save' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS sponsored_pocket_saves, - COUNTIF( - event_name = 'save' - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS organic_pocket_saves, - COUNTIF( - event_name = 'dismiss' - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS sponsored_pocket_dismissals, - COUNTIF( - event_name = 'dismiss' - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - AND COALESCE(mozfun.map.get_key(event_details, "is_list_card"), "false") = "false" - ) AS organic_pocket_dismissals, - COUNTIF( - event_name = 'thumb_voting_interaction' - AND mozfun.map.get_key(event_details, "thumbs_up") = "true" - ) AS pocket_thumbs_up, - COUNTIF( - event_name = 'thumb_voting_interaction' - AND mozfun.map.get_key(event_details, "thumbs_down") = "true" - ) AS pocket_thumbs_down, - SAFE_CAST(mozfun.map.get_key(event_details, "received_rank") AS INT) AS pocket_received_rank, - mozfun.map.get_key( - event_details, - "scheduled_corpus_item_id" - ) AS pocket_scheduled_corpus_item_id, - mozfun.map.get_key(event_details, "topic") AS pocket_topic, - mozfun.map.get_key(event_details, "matches_selected_topic") AS pocket_matches_selected_topic, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - ) AS list_card_clicks, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - ) AS organic_list_card_clicks, - COUNTIF( - event_name = "click" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_list_card_clicks, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - ) AS list_card_impressions, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - ) AS organic_list_card_impressions, - COUNTIF( - event_name = "impression" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_list_card_impressions, - COUNTIF( - event_name = "save" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - ) AS list_card_saves, - COUNTIF( - event_name = "save" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - ) AS organic_list_card_saves, - COUNTIF( - event_name = "save" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_list_card_saves, - COUNTIF( - event_name = "dismiss" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - ) AS list_card_dismissals, - COUNTIF( - event_name = "dismiss" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") != "true" - ) AS organic_list_card_dismissals, - COUNTIF( - event_name = "dismiss" - AND mozfun.map.get_key(event_details, "is_list_card") = "true" - AND mozfun.map.get_key(event_details, "is_sponsored") = "true" - ) AS sponsored_list_card_dismissals, - FROM - events_unnested - WHERE - event_category = 'pocket' - AND event_name IN ('impression', 'click', 'save', 'dismiss', 'thumb_voting_interaction') - GROUP BY - newtab_visit_id, - pocket_story_position, - pocket_tile_id, - pocket_recommendation_id, - pocket_received_rank, - pocket_scheduled_corpus_item_id, - pocket_topic, - pocket_matches_selected_topic -), -pocket_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - pocket_story_position, - pocket_tile_id, - pocket_recommendation_id, - pocket_impressions, - sponsored_pocket_impressions, - organic_pocket_impressions, - pocket_clicks, - sponsored_pocket_clicks, - organic_pocket_clicks, - pocket_saves, - sponsored_pocket_saves, - organic_pocket_saves, - sponsored_pocket_dismissals, - organic_pocket_dismissals, - pocket_thumbs_up, - pocket_thumbs_down, - pocket_received_rank, - pocket_scheduled_corpus_item_id, - pocket_topic, - pocket_matches_selected_topic, - list_card_clicks, - organic_list_card_clicks, - sponsored_list_card_clicks, - list_card_impressions, - organic_list_card_impressions, - sponsored_list_card_impressions, - list_card_saves, - organic_list_card_saves, - sponsored_list_card_saves, - list_card_dismissals, - organic_list_card_dismissals, - sponsored_list_card_dismissals - ) - ) AS pocket_interactions - FROM - pocket_events - GROUP BY - newtab_visit_id -), -wallpaper_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, - COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, - COUNTIF( - event_name = 'wallpaper_click' - AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" - ) AS wallpaper_clicks_had_previous_wallpaper, - COUNTIF( - event_name = 'wallpaper_click' - AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" - ) AS wallpaper_clicks_first_selected_wallpaper, - COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, - COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, - COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks - FROM - events_unnested - WHERE - event_category = 'newtab' - AND event_name IN ( - 'wallpaper_click', - 'wallpaper_category_click', - 'wallpaper_highlight_cta_clicks', - 'wallpaper_highlight_dismissed' - ) - GROUP BY - newtab_visit_id, - wallpaper_selected_wallpaper -), -wallpaper_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - wallpaper_selected_wallpaper, - wallpaper_clicks, - wallpaper_clicks_had_previous_wallpaper, - wallpaper_clicks_first_selected_wallpaper, - wallpaper_category_clicks, - wallpaper_highlight_dismissals, - wallpaper_highlight_cta_clicks - ) - ) AS wallpaper_interactions - FROM - wallpaper_events - GROUP BY - newtab_visit_id -), -weather_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, - COUNTIF(event_name = 'weather_location_selected') AS weather_widget_location_selected, - COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, - COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, - COUNTIF( - event_name = 'weather_change_display' - AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" - ) AS weather_widget_change_display_to_detailed, - COUNTIF( - event_name = 'weather_change_display' - AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" - ) AS weather_widget_change_display_to_simple - FROM - events_unnested - WHERE - event_category = 'newtab' - AND event_name IN ( - 'weather_impression', - 'weather_open_provider_url', - 'weather_load_error', - 'weather_change_display', - 'weather_location_selected' - ) - GROUP BY - newtab_visit_id -), -weather_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - weather_widget_impressions, - weather_widget_location_selected, - weather_widget_clicks, - weather_widget_load_errors, - weather_widget_change_display_to_detailed, - weather_widget_change_display_to_simple - ) - ) AS weather_interactions - FROM - weather_events - GROUP BY - newtab_visit_id -), -topic_selection_events AS ( - SELECT - mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, - mozfun.map.get_key(event_details, "previous_topics") AS previous_topics, - mozfun.map.get_key(event_details, "topics") AS topics, - COUNTIF(event_name = 'topic_selection_open') AS topic_selection_open, - COUNTIF(event_name = 'topic_selection_dismiss') AS topic_selection_dismiss, - COUNTIF( - event_name = 'topic_selection_topics_saved' - AND mozfun.map.get_key(event_details, "first_save") = "true" - ) AS topic_selection_topics_first_saved, - COUNTIF( - event_name = 'topic_selection_topics_saved' - AND mozfun.map.get_key(event_details, "first_save") != "true" - ) AS topic_selection_topics_updated, - FROM - events_unnested - WHERE - event_category = 'newtab' - AND event_name IN ( - 'topic_selection_dismiss', - 'topic_selection_open', - 'topic_selection_topics_saved' - ) - GROUP BY - newtab_visit_id, - previous_topics, - topics -), -topic_selection_summary AS ( - SELECT - newtab_visit_id, - ARRAY_AGG( - STRUCT( - previous_topics, - topics, - topic_selection_open, - topic_selection_dismiss, - topic_selection_topics_first_saved, - topic_selection_topics_updated - ) - ) AS topic_selection_interactions - FROM - topic_selection_events - GROUP BY - newtab_visit_id -), -combined_newtab_activity AS ( - SELECT - * - FROM - visit_metadata - LEFT JOIN - search_summary - USING (newtab_visit_id) - LEFT JOIN - topsites_summary - USING (newtab_visit_id) - LEFT JOIN - pocket_summary - USING (newtab_visit_id) - LEFT JOIN - wallpaper_summary - USING (newtab_visit_id) - LEFT JOIN - weather_summary - USING (newtab_visit_id) - LEFT JOIN - topic_selection_summary - USING (newtab_visit_id) - WHERE - -- Keep only rows with interactions, unless we receive a valid newtab.opened event. - -- This is meant to drop only interactions that only have a newtab.closed event on the same partition - -- (these are suspected to be from pre-loaded tabs) - newtab_open_source IS NOT NULL - OR search_interactions IS NOT NULL - OR topsite_tile_interactions IS NOT NULL - OR pocket_interactions IS NOT NULL - OR wallpaper_interactions IS NOT NULL - OR weather_interactions IS NOT NULL - OR topic_selection_interactions IS NOT NULL -), -client_profile_info AS ( - SELECT - client_id AS legacy_telemetry_client_id, - ANY_VALUE(is_new_profile) AS is_new_profile, - ANY_VALUE(activity_segment) AS activity_segment - FROM - `moz-fx-data-shared-prod.telemetry_derived.unified_metrics_v1` - WHERE - submission_date = @submission_date - GROUP BY - client_id -) -SELECT - *, - CASE - WHEN (newtab_open_source = 'about:home') AND (newtab_homepage_category = 'enabled') THEN 'default' - WHEN (newtab_open_source = 'about:newtab') AND (newtab_newtab_category = 'enabled') THEN 'default' - WHEN newtab_open_source = 'about:welcome' THEN 'welcome' - WHEN (newtab_open_source = 'about:home') AND (newtab_homepage_category != 'enabled') THEN 'home - non-default' - WHEN (newtab_open_source = 'about:newtab') AND (newtab_newtab_category != 'enabled') THEN 'newtab - non-default' - ELSE "other" - END AS default_ui, -FROM - combined_newtab_activity -LEFT JOIN - client_profile_info - USING (legacy_telemetry_client_id) From c95300d79020fda3a2104aae6247f8faab2766ce Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Thu, 23 Jan 2025 22:36:03 -0800 Subject: [PATCH 4/8] DENG-6880 - Daily aggregations for NewTab --- .../metadata.yaml | 5 + .../view.sql | 7 + .../metadata.yaml | 5 + .../view.sql | 7 + .../query.sql | 14 ++ .../query.sql | 101 ----------- .../metadata.yaml | 23 +++ .../query.sql | 171 ++++++++++++++++++ .../schema.yaml | 156 ++++++++++++++++ .../metadata.yaml | 24 +++ .../query.sql | 55 +++--- .../schema.yaml | 149 +++++++++++++++ 12 files changed, 588 insertions(+), 129 deletions(-) create mode 100644 sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/view.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/view.sql delete mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml diff --git a/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/metadata.yaml new file mode 100644 index 00000000000..b1b245d27f9 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/metadata.yaml @@ -0,0 +1,5 @@ +friendly_name: Newtab Conditional Daily Aggregates +description: |- + Conditional computations for daily aggregation for newtab. +owners: +- gkatre@mozilla.com diff --git a/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/view.sql b/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/view.sql new file mode 100644 index 00000000000..a9ad4b5b23c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/newtab_conditional_daily_aggregates/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.newtab_conditional_daily_aggregates` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.newtab_conditional_daily_aggregates_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/metadata.yaml new file mode 100644 index 00000000000..41a6b7b2ff3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/metadata.yaml @@ -0,0 +1,5 @@ +friendly_name: Newtab Interactions Daily Aggregates +description: |- + A daily aggregation of newtab interactions, partitioned by day. +owners: +- gkatre@mozilla.com diff --git a/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/view.sql b/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/view.sql new file mode 100644 index 00000000000..2291e9beade --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry/newtab_daily_interactions_aggregates/view.sql @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.newtab_daily_interactions_aggregates` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.telemetry_derived.newtab_daily_interactions_aggregates_v1` diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/query.sql index 70ca6078cd3..39f1eba1a69 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/query.sql @@ -49,6 +49,20 @@ SELECT SUM(pocket_saves) AS pocket_saves, SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, SUM(organic_pocket_saves) AS organic_pocket_saves, + SUM(pocket_thumb_voting_events) AS pocket_thumb_voting_events, + SUM(topic_selection_opened) AS topic_selection_opened, + SUM(topic_selection_updates) AS topic_selection_updates, + SUM(weather_widget_clicks) AS weather_widget_clicks, + SUM(weather_widget_change_display_to_detailed) AS weather_widget_change_display_to_detailed, + SUM(weather_widget_change_display_to_simple) AS weather_widget_change_display_to_simple, + SUM(wallpaper_clicks) AS wallpaper_clicks, + SUM( + IF(topic_selection_selected_topics_first_time, 1, 0) + ) AS topic_selection_selected_topics_first_time_count, + SUM(sponsored_pocket_dismissals) AS sponsored_pocket_dismissals, + SUM(organic_pocket_dismissals) AS organic_pocket_dismissals, + SUM(topic_selection_dismissals) AS topic_selection_dismissals, + SUM(visits_with_default_ui) AS visits_with_default_ui, FROM `moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1` WHERE diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql deleted file mode 100644 index e3c5abcc0c9..00000000000 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v2/query.sql +++ /dev/null @@ -1,101 +0,0 @@ -SELECT - submission_date, - normalized_os, - normalized_os_version, - country_code, - locale, - channel, - browser_version, - browser_name, - default_search_engine, - default_private_search_engine, - pocket_is_signed_in, - pocket_enabled, - pocket_sponsored_stories_enabled, - topsites_enabled, - topsites_sponsored_enabled, - newtab_homepage_category, - newtab_newtab_category, - topsites_rows, - is_new_profile, - activity_segment, - COUNT(DISTINCT client_id) AS client_count, - COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count, - SUM(newtab_visit_count) AS newtab_visit_count, - SUM(visits_with_non_impression_engagement) AS visits_with_non_impression_engagement, - SUM(visits_with_non_search_engagement) AS visits_with_non_search_engagement, - SUM(searches) AS searches, - SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks, - SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions, - SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, - SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, - SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, - SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, - SUM(topsite_tile_clicks) AS topsite_tile_clicks, - SUM(sponsored_topsite_tile_clicks) AS sponsored_topsite_tile_clicks, - SUM(organic_topsite_tile_clicks) AS organic_topsite_tile_clicks, - SUM(topsite_tile_impressions) AS topsite_tile_impressions, - SUM(sponsored_topsite_tile_impressions) AS sponsored_topsite_tile_impressions, - SUM(organic_topsite_tile_impressions) AS organic_topsite_tile_impressions, - SUM(topsite_tile_dismissals) AS topsite_tile_dismissals, - SUM(sponsored_topsite_tile_dismissals) AS sponsored_topsite_tile_dismissals, - SUM(organic_topsite_tile_dismissals) AS organic_topsite_tile_dismissals, - SUM(pocket_impressions) AS pocket_impressions, - SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions, - SUM(organic_pocket_impressions) AS organic_pocket_impressions, - SUM(pocket_clicks) AS pocket_clicks, - SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks, - SUM(organic_pocket_clicks) AS organic_pocket_clicks, - SUM(pocket_saves) AS pocket_saves, - SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, - SUM(organic_pocket_saves) AS organic_pocket_saves, - --- New additions - sum(pocket_thumb_voting_events) as pocket_thumb_voting_events, - sum(topic_selection_opened) as topic_selection_opened, - sum(topic_selection_updates) as topic_selection_updates, - sum(weather_widget_clicks) as weather_widget_clicks, - sum(weather_widget_change_display_to_detailed) as weather_widget_change_display_to_detailed, - sum(weather_widget_change_display_to_simple) as weather_widget_change_display_to_simple, - sum(wallpaper_clicks) as wallpaper_clicks, - sum(topic_selection_selected_topics_first_time) as topic_selection_selected_topics_first_time, - sum(sponsored_pocket_dismissals) as sponsored_pocket_dismissals, - sum(organic_pocket_dismissals) as organic_pocket_dismissals, - sum(topic_selection_dismissals) as topic_selection_dismissals, - - sum(visits_with_default_ui) as visits_with_default_ui, - sum(topic_selection_opened) as topic_selection_opened, - sum(topic_selection_opened) as topic_selection_opened, - sum(topic_selection_opened) as topic_selection_opened, - sum(topic_selection_opened) as topic_selection_opened, - sum(topic_selection_opened) as topic_selection_opened, - -FROM - `moz-fx-data-shared-prod.telemetry_derived.newtab_clients_daily_v1` -WHERE - {% if is_init() %} - submission_date >= "2022-11-01" - {% else %} - submission_date = @submission_date - {% endif %} -GROUP BY - submission_date, - normalized_os, - normalized_os_version, - country_code, - locale, - channel, - browser_version, - browser_name, - default_search_engine, - default_private_search_engine, - pocket_is_signed_in, - pocket_enabled, - pocket_sponsored_stories_enabled, - topsites_enabled, - topsites_sponsored_enabled, - newtab_homepage_category, - newtab_newtab_category, - topsites_rows, - is_new_profile, - activity_segment diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/metadata.yaml new file mode 100644 index 00000000000..ad8deda5a47 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/metadata.yaml @@ -0,0 +1,23 @@ +friendly_name: Newtab Conditional Daily Aggregates +description: |- + Conditional computations for daily aggregation for newtab. +owners: + - gkatre@mozilla.com +labels: + application: firefox + incremental: true + schedule: daily + dag: bqetl_newtab + owner1: gkatre +scheduling: + dag_name: bqetl_newtab +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + clustering: + fields: + - country_code +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql new file mode 100644 index 00000000000..270e9af3693 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql @@ -0,0 +1,171 @@ +SELECT + submission_date, + country_code, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled, + COUNT(DISTINCT client_id) AS total_client_count, + COUNT(DISTINCT newtab_visit_id) AS total_visit_count, + -- conditional + COUNT(DISTINCT IF(channel = 'release', client_id, NULL)) AS client_count, + COUNT(DISTINCT IF(channel = 'release', newtab_visit_id, NULL)) AS visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:home' + AND newtab_homepage_category = 'enabled', + client_id, + NULL + ) + ) AS home_default_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:home' + AND newtab_homepage_category = 'enabled', + newtab_visit_id, + NULL + ) + ) AS home_default_visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:newtab' + AND newtab_newtab_category = 'enabled', + client_id, + NULL + ) + ) AS newtab_default_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:newtab' + AND newtab_newtab_category = 'enabled', + newtab_visit_id, + NULL + ) + ) AS newtab_default_visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:home' + AND newtab_homepage_category != 'enabled', + client_id, + NULL + ) + ) AS home_nondefault_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:home' + AND newtab_homepage_category != 'enabled', + newtab_visit_id, + NULL + ) + ) AS home_nondefault_visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:newtab' + AND newtab_newtab_category != 'enabled', + client_id, + NULL + ) + ) AS newtab_nondefault_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_open_source = 'about:newtab' + AND newtab_newtab_category != 'enabled', + newtab_visit_id, + NULL + ) + ) AS newtab_nondefault_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND newtab_open_source = 'about:welcome', client_id, NULL) + ) AS welcome_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND newtab_open_source = 'about:welcome', newtab_visit_id, NULL) + ) AS welcome_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND searches > 0, client_id, NULL) + ) AS searched_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND searches > 0, newtab_visit_id, NULL) + ) AS searched_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_topsite_impressions > 0, client_id, NULL) + ) AS sponsored_topsite_impressed_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_topsite_impressions > 0, newtab_visit_id, NULL) + ) AS sponsored_topsite_impressed_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_topsite_clicks > 0, client_id, NULL) + ) AS sponsored_topsite_clicked_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_topsite_clicks > 0, newtab_visit_id, NULL) + ) AS sponsored_topsite_clicked_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND topsite_impressions > 0, client_id, NULL) + ) AS topsite_impressed_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND topsite_impressions > 0, newtab_visit_id, NULL) + ) AS topsite_impressed_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND topsite_clicks > 0, client_id, NULL) + ) AS topsite_clicked_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND topsite_clicks > 0, newtab_visit_id, NULL) + ) AS topsite_clicked_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_pocket_impressions > 0, client_id, NULL) + ) AS sponsored_pocket_impressed_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_pocket_impressions > 0, newtab_visit_id, NULL) + ) AS sponsored_pocket_impressed_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_pocket_clicks > 0, client_id, NULL) + ) AS sponsored_pocket_clicked_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND sponsored_pocket_clicks > 0, newtab_visit_id, NULL) + ) AS sponsored_pocket_clicked_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND organic_pocket_impressions > 0, client_id, NULL) + ) AS organic_pocket_impressed_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND organic_pocket_impressions > 0, newtab_visit_id, NULL) + ) AS organic_pocket_impressed_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND organic_pocket_clicks > 0, client_id, NULL) + ) AS organic_pocket_clicked_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND organic_pocket_clicks > 0, newtab_visit_id, NULL) + ) AS organic_pocket_clicked_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_impressions > 0, client_id, NULL) + ) AS pocket_impressed_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_impressions > 0, newtab_visit_id, NULL) + ) AS pocket_impressed_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_clicks > 0, client_id, NULL) + ) AS pocket_clicked_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_clicks > 0, newtab_visit_id, NULL) + ) AS pocket_clicked_visit_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_is_signed_in, client_id, NULL) + ) AS pocket_signed_in_client_count, + COUNT( + DISTINCT IF(channel = 'release' AND pocket_is_signed_in, newtab_visit_id, NULL) + ) AS pocket_signed_in_visit_count, +FROM + `moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1` +WHERE + submission_date = @submission_date +GROUP BY + submission_date, + country_code, + pocket_enabled, + pocket_sponsored_stories_enabled, + topsites_enabled diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml new file mode 100644 index 00000000000..a6c94a80b38 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml @@ -0,0 +1,156 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: country_code + type: STRING + mode: NULLABLE +- name: pocket_enabled + type: BOOLEAN + mode: NULLABLE +- name: pocket_sponsored_stories_enabled + type: BOOLEAN + mode: NULLABLE +- name: topsites_enabled + type: BOOLEAN + mode: NULLABLE + +- name: total_client_count + type: INTEGER + mode: NULLABLE +- name: total_visit_count + type: INTEGER + mode: NULLABLE + +- name: client_count + type: INTEGER + mode: NULLABLE +- name: visit_count + type: INTEGER + mode: NULLABLE + +- name: home_default_client_count + type: INTEGER + mode: NULLABLE +- name: home_default_visit_count + type: INTEGER + mode: NULLABLE + +- name: newtab_default_client_count + type: INTEGER + mode: NULLABLE +- name: newtab_default_visit_count + type: INTEGER + mode: NULLABLE + +- name: home_nondefault_client_count + type: INTEGER + mode: NULLABLE +- name: home_nondefault_visit_count + type: INTEGER + mode: NULLABLE + +- name: newtab_nondefault_client_count + type: INTEGER + mode: NULLABLE +- name: newtab_nondefault_visit_count + type: INTEGER + mode: NULLABLE + +- name: welcome_client_count + type: INTEGER + mode: NULLABLE +- name: welcome_visit_count + type: INTEGER + mode: NULLABLE + +- name: searched_client_count + type: INTEGER + mode: NULLABLE +- name: searched_visit_count + type: INTEGER + mode: NULLABLE + +- name: sponsored_topsite_impressed_client_count + type: INTEGER + mode: NULLABLE +- name: sponsored_topsite_impressed_visit_count + type: INTEGER + mode: NULLABLE + +- name: client_count + type: INTEGER + mode: NULLABLE +- name: visit_count + type: INTEGER + mode: NULLABLE + +- name: sponsored_topsite_clicked_client_count + type: INTEGER + mode: NULLABLE +- name: sponsored_topsite_clicked_visit_count + type: INTEGER + mode: NULLABLE + +- name: topsite_impressed_client_count + type: INTEGER + mode: NULLABLE +- name: topsite_impressed_visit_count + type: INTEGER + mode: NULLABLE + +- name: topsite_clicked_client_count + type: INTEGER + mode: NULLABLE +- name: topsite_clicked_visit_count + type: INTEGER + mode: NULLABLE + +- name: sponsored_pocket_impressed_client_count + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_impressed_visit_count + type: INTEGER + mode: NULLABLE + +- name: sponsored_pocket_clicked_client_count + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_clicked_visit_count + type: INTEGER + mode: NULLABLE + +- name: organic_pocket_impressed_client_count + type: INTEGER + mode: NULLABLE +- name: organic_pocket_impressed_visit_count + type: INTEGER + mode: NULLABLE + +- name: organic_pocket_clicked_client_count + type: INTEGER + mode: NULLABLE +- name: organic_pocket_clicked_visit_count + type: INTEGER + mode: NULLABLE + +- name: pocket_impressed_client_count + type: INTEGER + mode: NULLABLE +- name: pocket_impressed_visit_count + type: INTEGER + mode: NULLABLE + +- name: pocket_clicked_client_count + type: INTEGER + mode: NULLABLE +- name: pocket_clicked_visit_count + type: INTEGER + mode: NULLABLE + +- name: pocket_signed_in_client_count + type: INTEGER + mode: NULLABLE +- name: pocket_signed_in_visit_count + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml new file mode 100644 index 00000000000..1e70a9b7172 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/metadata.yaml @@ -0,0 +1,24 @@ +friendly_name: Newtab Interactions Daily Aggregates +description: |- + A daily aggregation of newtab interactions, partitioned by day. +owners: + - gkatre@mozilla.com +labels: + application: firefox + incremental: true + schedule: daily + dag: bqetl_newtab + owner1: gkatre +scheduling: + dag_name: bqetl_newtab +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + clustering: + fields: + - channel + - country_code +references: {} diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql index ecce09ad6b3..da8a6695ffa 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/query.sql @@ -1,4 +1,4 @@ -select +SELECT submission_date, search_engine, search_access_point, @@ -22,39 +22,38 @@ select topsites_rows, is_new_profile, activity_segment, - COUNT(DISTINCT client_id) AS client_count, COUNT(DISTINCT legacy_telemetry_client_id) AS legacy_telemetry_client_count, COUNT(DISTINCT newtab_visit_id) AS newtab_interactions_visits, -- Topsite - sum(topsite_clicks) AS topsite_clicks, - sum(sponsored_topsite_clicks) AS sponsored_topsite_clicks, - sum(organic_topsite_clicks) AS organic_topsite_clicks, - sum(topsite_impressions) AS topsite_impressions, - sum(sponsored_topsite_impressions) AS sponsored_topsite_impressions, - sum(organic_topsite_impressions) AS organic_topsite_impressions, - sum(topsite_dismissals) AS topsite_dismissals, - sum(sponsored_topsite_dismissals) AS sponsored_topsite_dismissals, - sum(organic_topsite_dismissals) AS organic_topsite_dismissals, + SUM(topsite_clicks) AS topsite_clicks, + SUM(sponsored_topsite_clicks) AS sponsored_topsite_clicks, + SUM(organic_topsite_clicks) AS organic_topsite_clicks, + SUM(topsite_impressions) AS topsite_impressions, + SUM(sponsored_topsite_impressions) AS sponsored_topsite_impressions, + SUM(organic_topsite_impressions) AS organic_topsite_impressions, + SUM(topsite_dismissals) AS topsite_dismissals, + SUM(sponsored_topsite_dismissals) AS sponsored_topsite_dismissals, + SUM(organic_topsite_dismissals) AS organic_topsite_dismissals, -- Search - sum(searches) AS searches, - sum(tagged_search_ad_clicks) AS tagged_search_ad_clicks, - sum(tagged_search_ad_impressions) AS tagged_search_ad_impressions, - sum(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, - sum(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, - sum(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, - sum(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, + SUM(searches) AS searches, + SUM(tagged_search_ad_clicks) AS tagged_search_ad_clicks, + SUM(tagged_search_ad_impressions) AS tagged_search_ad_impressions, + SUM(follow_on_search_ad_clicks) AS follow_on_search_ad_clicks, + SUM(follow_on_search_ad_impressions) AS follow_on_search_ad_impressions, + SUM(tagged_follow_on_search_ad_clicks) AS tagged_follow_on_search_ad_clicks, + SUM(tagged_follow_on_search_ad_impressions) AS tagged_follow_on_search_ad_impressions, -- Pocket - sum(pocket_impressions) AS pocket_impressions, - sum(sponsored_pocket_impressions) AS sponsored_pocket_impressions, - sum(organic_pocket_impressions) AS organic_pocket_impressions, - sum(pocket_clicks) AS pocket_clicks, - sum(sponsored_pocket_clicks) AS sponsored_pocket_clicks, - sum(organic_pocket_clicks) AS organic_pocket_clicks, - sum(pocket_saves) AS pocket_saves, - sum(sponsored_pocket_saves) AS sponsored_pocket_saves, - sum(organic_pocket_saves) AS organic_pocket_saves, -from + SUM(pocket_impressions) AS pocket_impressions, + SUM(sponsored_pocket_impressions) AS sponsored_pocket_impressions, + SUM(organic_pocket_impressions) AS organic_pocket_impressions, + SUM(pocket_clicks) AS pocket_clicks, + SUM(sponsored_pocket_clicks) AS sponsored_pocket_clicks, + SUM(organic_pocket_clicks) AS organic_pocket_clicks, + SUM(pocket_saves) AS pocket_saves, + SUM(sponsored_pocket_saves) AS sponsored_pocket_saves, + SUM(organic_pocket_saves) AS organic_pocket_saves, +FROM `moz-fx-data-shared-prod.telemetry_derived.newtab_interactions_v1` WHERE submission_date = @submission_date diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml new file mode 100644 index 00000000000..097ef108eeb --- /dev/null +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml @@ -0,0 +1,149 @@ +fields: +- name: submission_date + type: DATE + mode: NULLABLE +- name: country_code + type: STRING + mode: NULLABLE +- name: channel + type: STRING + mode: NULLABLE +- name: browser_version + type: STRING + mode: NULLABLE +- name: browser_name + type: STRING + mode: NULLABLE +- name: search_engine + type: STRING + mode: NULLABLE +- name: search_access_point + type: STRING + mode: NULLABLE +- name: default_search_engine + type: STRING + mode: NULLABLE +- name: default_private_search_engine + type: STRING + mode: NULLABLE +- name: pocket_story_position + type: INTEGER + mode: NULLABLE +- name: newtab_open_source + type: STRING + mode: NULLABLE +- name: pocket_is_signed_in + type: BOOLEAN + mode: NULLABLE +- name: pocket_enabled + type: BOOLEAN + mode: NULLABLE +- name: pocket_sponsored_stories_enabled + type: BOOLEAN + mode: NULLABLE +- name: topsites_enabled + type: BOOLEAN + mode: NULLABLE +- name: newtab_homepage_category + type: STRING + mode: NULLABLE +- name: newtab_newtab_category + type: STRING + mode: NULLABLE +- name: topsites_rows + type: INTEGER + mode: NULLABLE +- name: is_new_profile + type: BOOLEAN + mode: NULLABLE +- name: activity_segment + type: STRING + mode: NULLABLE + +- name: client_count + type: INTEGER + mode: NULLABLE +- name: legacy_telemetry_client_count + type: INTEGER + mode: NULLABLE +- name: newtab_interactions_visits + type: INTEGER + mode: NULLABLE + +- name: topsite_clicks + type: INTEGER + mode: NULLABLE +- name: sponsored_topsite_clicks + type: INTEGER + mode: NULLABLE +- name: organic_topsite_clicks + type: INTEGER + mode: NULLABLE +- name: topsite_impressions + type: INTEGER + mode: NULLABLE +- name: sponsored_topsite_impressions + type: INTEGER + mode: NULLABLE +- name: organic_topsite_impressions + type: INTEGER + mode: NULLABLE +- name: topsite_dismissals + type: INTEGER + mode: NULLABLE +- name: sponsored_topsite_dismissals + type: INTEGER + mode: NULLABLE +- name: organic_topsite_dismissals + type: INTEGER + mode: NULLABLE + +- name: searches + type: INTEGER + mode: NULLABLE +- name: tagged_search_ad_clicks + type: INTEGER + mode: NULLABLE +- name: tagged_search_ad_impressions + type: INTEGER + mode: NULLABLE +- name: follow_on_search_ad_clicks + type: INTEGER + mode: NULLABLE +- name: follow_on_search_ad_impressions + type: INTEGER + mode: NULLABLE +- name: tagged_follow_on_search_ad_clicks + type: INTEGER + mode: NULLABLE +- name: tagged_follow_on_search_ad_impressions + type: INTEGER + mode: NULLABLE + +- name: pocket_impressions + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_impressions + type: INTEGER + mode: NULLABLE +- name: organic_pocket_impressions + type: INTEGER + mode: NULLABLE +- name: pocket_clicks + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_clicks + type: INTEGER + mode: NULLABLE +- name: organic_pocket_clicks + type: INTEGER + mode: NULLABLE +- name: pocket_saves + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_saves + type: INTEGER + mode: NULLABLE +- name: organic_pocket_saves + type: INTEGER + mode: NULLABLE From b58055e7704c9e8db9cb105f3580972b87d06444 Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Fri, 24 Jan 2025 10:10:41 -0800 Subject: [PATCH 5/8] updating schema.yaml --- .../schema.yaml | 36 +++++++++++++++++++ .../schema.yaml | 20 ----------- .../schema.yaml | 4 --- 3 files changed, 36 insertions(+), 24 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/schema.yaml index 39eb2247503..fe9dc0031cc 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_clients_daily_aggregates_v1/schema.yaml @@ -149,3 +149,39 @@ fields: - name: organic_pocket_saves type: INTEGER mode: NULLABLE +- name: pocket_thumb_voting_events + type: INTEGER + mode: NULLABLE +- name: topic_selection_opened + type: INTEGER + mode: NULLABLE +- name: topic_selection_updates + type: INTEGER + mode: NULLABLE +- name: weather_widget_clicks + type: INTEGER + mode: NULLABLE +- name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE +- name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE +- name: wallpaper_clicks + type: INTEGER + mode: NULLABLE +- name: topic_selection_selected_topics_first_time_count + type: INTEGER + mode: NULLABLE +- name: sponsored_pocket_dismissals + type: INTEGER + mode: NULLABLE +- name: organic_pocket_dismissals + type: INTEGER + mode: NULLABLE +- name: topic_selection_dismissals + type: INTEGER + mode: NULLABLE +- name: visits_with_default_ui + type: INTEGER + mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml index a6c94a80b38..d2455b48aac 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml @@ -14,140 +14,120 @@ fields: - name: topsites_enabled type: BOOLEAN mode: NULLABLE - - name: total_client_count type: INTEGER mode: NULLABLE - name: total_visit_count type: INTEGER mode: NULLABLE - - name: client_count type: INTEGER mode: NULLABLE - name: visit_count type: INTEGER mode: NULLABLE - - name: home_default_client_count type: INTEGER mode: NULLABLE - name: home_default_visit_count type: INTEGER mode: NULLABLE - - name: newtab_default_client_count type: INTEGER mode: NULLABLE - name: newtab_default_visit_count type: INTEGER mode: NULLABLE - - name: home_nondefault_client_count type: INTEGER mode: NULLABLE - name: home_nondefault_visit_count type: INTEGER mode: NULLABLE - - name: newtab_nondefault_client_count type: INTEGER mode: NULLABLE - name: newtab_nondefault_visit_count type: INTEGER mode: NULLABLE - - name: welcome_client_count type: INTEGER mode: NULLABLE - name: welcome_visit_count type: INTEGER mode: NULLABLE - - name: searched_client_count type: INTEGER mode: NULLABLE - name: searched_visit_count type: INTEGER mode: NULLABLE - - name: sponsored_topsite_impressed_client_count type: INTEGER mode: NULLABLE - name: sponsored_topsite_impressed_visit_count type: INTEGER mode: NULLABLE - - name: client_count type: INTEGER mode: NULLABLE - name: visit_count type: INTEGER mode: NULLABLE - - name: sponsored_topsite_clicked_client_count type: INTEGER mode: NULLABLE - name: sponsored_topsite_clicked_visit_count type: INTEGER mode: NULLABLE - - name: topsite_impressed_client_count type: INTEGER mode: NULLABLE - name: topsite_impressed_visit_count type: INTEGER mode: NULLABLE - - name: topsite_clicked_client_count type: INTEGER mode: NULLABLE - name: topsite_clicked_visit_count type: INTEGER mode: NULLABLE - - name: sponsored_pocket_impressed_client_count type: INTEGER mode: NULLABLE - name: sponsored_pocket_impressed_visit_count type: INTEGER mode: NULLABLE - - name: sponsored_pocket_clicked_client_count type: INTEGER mode: NULLABLE - name: sponsored_pocket_clicked_visit_count type: INTEGER mode: NULLABLE - - name: organic_pocket_impressed_client_count type: INTEGER mode: NULLABLE - name: organic_pocket_impressed_visit_count type: INTEGER mode: NULLABLE - - name: organic_pocket_clicked_client_count type: INTEGER mode: NULLABLE - name: organic_pocket_clicked_visit_count type: INTEGER mode: NULLABLE - - name: pocket_impressed_client_count type: INTEGER mode: NULLABLE - name: pocket_impressed_visit_count type: INTEGER mode: NULLABLE - - name: pocket_clicked_client_count type: INTEGER mode: NULLABLE - name: pocket_clicked_visit_count type: INTEGER mode: NULLABLE - - name: pocket_signed_in_client_count type: INTEGER mode: NULLABLE diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml index 097ef108eeb..4a2a8f499b4 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_daily_interactions_aggregates_v1/schema.yaml @@ -59,7 +59,6 @@ fields: - name: activity_segment type: STRING mode: NULLABLE - - name: client_count type: INTEGER mode: NULLABLE @@ -69,7 +68,6 @@ fields: - name: newtab_interactions_visits type: INTEGER mode: NULLABLE - - name: topsite_clicks type: INTEGER mode: NULLABLE @@ -97,7 +95,6 @@ fields: - name: organic_topsite_dismissals type: INTEGER mode: NULLABLE - - name: searches type: INTEGER mode: NULLABLE @@ -119,7 +116,6 @@ fields: - name: tagged_follow_on_search_ad_impressions type: INTEGER mode: NULLABLE - - name: pocket_impressions type: INTEGER mode: NULLABLE From 793aa7b296a535d1a78718a5231fd423e26bd95a Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Fri, 24 Jan 2025 10:59:21 -0800 Subject: [PATCH 6/8] Update sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml Co-authored-by: Ben Wu <12437227+BenWu@users.noreply.github.com> --- .../newtab_conditional_daily_aggregates_v1/schema.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml index d2455b48aac..0d17761eb64 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml @@ -68,12 +68,6 @@ fields: - name: sponsored_topsite_impressed_visit_count type: INTEGER mode: NULLABLE -- name: client_count - type: INTEGER - mode: NULLABLE -- name: visit_count - type: INTEGER - mode: NULLABLE - name: sponsored_topsite_clicked_client_count type: INTEGER mode: NULLABLE From fa5746a5da3f8481cfe83ca399bdabb8736250af Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Fri, 24 Jan 2025 15:04:03 -0800 Subject: [PATCH 7/8] Adding homepage_category and newtab_category computations --- .../query.sql | 32 +++++++++++++++++++ .../schema.yaml | 12 +++++++ 2 files changed, 44 insertions(+) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql index 270e9af3693..3274f3c0ee5 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql @@ -9,6 +9,38 @@ SELECT -- conditional COUNT(DISTINCT IF(channel = 'release', client_id, NULL)) AS client_count, COUNT(DISTINCT IF(channel = 'release', newtab_visit_id, NULL)) AS visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_homepage_category = 'enabled', + client_id, + NULL + ) + ) AS homepage_category_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_homepage_category = 'enabled', + newtab_visit_id, + NULL + ) + ) AS homepage_category_visit_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_newtab_category = 'enabled', + client_id, + NULL + ) + ) AS newtab_category_client_count, + COUNT( + DISTINCT IF( + channel = 'release' + AND newtab_newtab_category = 'enabled', + newtab_visit_id, + NULL + ) + ) AS newtab_category_visit_count, COUNT( DISTINCT IF( channel = 'release' diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml index 0d17761eb64..6c39d63644b 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/schema.yaml @@ -26,6 +26,18 @@ fields: - name: visit_count type: INTEGER mode: NULLABLE +- name: homepage_category_client_count + type: INTEGER + mode: NULLABLE +- name: homepage_category_visit_count + type: INTEGER + mode: NULLABLE +- name: newtab_category_client_count + type: INTEGER + mode: NULLABLE +- name: newtab_category_visit_count + type: INTEGER + mode: NULLABLE - name: home_default_client_count type: INTEGER mode: NULLABLE From 4a1f958aab2cb126dbc6367ed1f0d0c38cbbaf1b Mon Sep 17 00:00:00 2001 From: Gaurang Katre Date: Fri, 24 Jan 2025 15:17:48 -0800 Subject: [PATCH 8/8] formatting --- .../query.sql | 28 +++---------------- 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql index 3274f3c0ee5..3295f716893 100644 --- a/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql +++ b/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_conditional_daily_aggregates_v1/query.sql @@ -10,36 +10,16 @@ SELECT COUNT(DISTINCT IF(channel = 'release', client_id, NULL)) AS client_count, COUNT(DISTINCT IF(channel = 'release', newtab_visit_id, NULL)) AS visit_count, COUNT( - DISTINCT IF( - channel = 'release' - AND newtab_homepage_category = 'enabled', - client_id, - NULL - ) + DISTINCT IF(channel = 'release' AND newtab_homepage_category = 'enabled', client_id, NULL) ) AS homepage_category_client_count, COUNT( - DISTINCT IF( - channel = 'release' - AND newtab_homepage_category = 'enabled', - newtab_visit_id, - NULL - ) + DISTINCT IF(channel = 'release' AND newtab_homepage_category = 'enabled', newtab_visit_id, NULL) ) AS homepage_category_visit_count, COUNT( - DISTINCT IF( - channel = 'release' - AND newtab_newtab_category = 'enabled', - client_id, - NULL - ) + DISTINCT IF(channel = 'release' AND newtab_newtab_category = 'enabled', client_id, NULL) ) AS newtab_category_client_count, COUNT( - DISTINCT IF( - channel = 'release' - AND newtab_newtab_category = 'enabled', - newtab_visit_id, - NULL - ) + DISTINCT IF(channel = 'release' AND newtab_newtab_category = 'enabled', newtab_visit_id, NULL) ) AS newtab_category_visit_count, COUNT( DISTINCT IF(