Add back thunderbird checks

bigquery_etl/cli/monitoring.py

@@ -376,7 +376,10 @@ def _update_bigconfig(
 
                     raise Exception(err_message)
 
-                if metric.metric_type.predefined_metric in default_metrics:
+                if (
+                    metric.metric_type
+                    and metric.metric_type.predefined_metric in default_metrics
+                ):
                     default_metrics.remove(metric.metric_type.predefined_metric)
 
         if metadata.monitoring.collection and collection.collection is None:

bqetl_project.yaml

@@ -296,6 +296,10 @@ generate:
     - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/bigconfig.yml
     - sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_last_seen_v1/checks.sql
     - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/checks.sql
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml
     skip_apps:
     - mlhackweek_search
     - moso_mastodon_android

sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml

@@ -0,0 +1,205 @@
+type: BIGCONFIG_FILE
+row_creation_times:
+  column_selectors:
+  - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date
+saved_metric_definitions:
+  metrics:
+  - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: PERCENT_NULL
+    metric_name: Null (%)
+    group_by:
+    - submission_date
+    threshold:
+      type: CONSTANT
+      upper_bound: 0.0
+      lower_bound: 0.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: COUNT_VALUE_IN_LIST
+    metric_name: Value in list (#)
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    parameters:
+    - key: list
+      string_value: nightly, aurora, release, Other, beta, esr
+      column_name: ''
+      number_value: 0.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived
+    metric_type:
+      type: TEMPLATE
+      template_id: 945
+      aggregation_type: COUNT
+      template_name: country_regex_check
+    metric_name: COUNT of country_regex_check
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    parameters:
+    - key: column
+      string_value: country
+    - key: column_name
+      string_value: country
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: FRESHNESS
+    metric_name: FRESHNESS [warn]
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: true
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: HOUR
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: VOLUME
+    metric_name: VOLUME [fail]
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: HOUR
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: COUNT_DUPLICATES
+    metric_name: Duplicates (#)
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: STRING_LENGTH_MAX
+    metric_name: Maximum string length
+    threshold:
+      type: CONSTANT
+      upper_bound: 36.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+tag_deployments:
+- collection:
+    name: Thunderbird
+    description: Checks related to thunderbird datasets
+  deployments:
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.sample_id
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.first_seen_date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_created_profile_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_active_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id
+    metrics:
+    - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.normalized_channel
+    metrics:
+    - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.*
+    metrics:
+    - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.*
+    metrics:
+    - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived
+    - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id
+    metrics:
+    - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived
+    - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived

sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml

@@ -0,0 +1,51 @@
+friendly_name: Baseline Clients Last Seen
+description: |-
+  Captures activity history of each client in 28-day
+  windows for each submission date based on baseline pings.
+
+  Clustering fields: `normalized_channel`, `sample_id`
+
+  See also: `baseline_clients_daily`
+owners:
+- [email protected]
+labels:
+  incremental: true
+  schedule: daily
+  dag: bqetl_glean_usage
+  owner1: ascholtz
+scheduling:
+  dag_name: bqetl_glean_usage
+  task_group: thunderbird_android
+  depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - normalized_channel
+    - sample_id
+schema:
+  derived_from:
+  - table:
+    - moz-fx-data-shared-prod
+    - net_thunderbird_android_beta_derived
+    - baseline_clients_daily_v1
+    exclude: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+monitoring:
+  enabled: true
+references:
+  checks.sql:
+  - ..
+  - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1
+  - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1
+  query.sql:
+  - net_thunderbird_android_beta_derived.baseline_clients_daily_v1
+  - net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1