Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Undo service_type transformation in stg_rate_sheet_daily and fix service_type filters in cost_per_query and hourly_spend #151

Merged
merged 5 commits into from
May 6, 2024
Merged

Undo service_type transformation in stg_rate_sheet_daily and fix service_type filters in cost_per_query and hourly_spend #151

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
798d7d1
Undo service_type transformation in stg_rate_sheet_daily
stumelius Apr 3, 2024
970bc7e
Replace service_type='COMPUTE' filters with service_type='WAREHOUSE_M…
stumelius Apr 3, 2024
f2c6209
Fix service_type filters in hourly_spend
stumelius Apr 3, 2024
4ebd1b6
Merge branch 'main' into feat/undo-service-type-transformation
NiallRees May 6, 2024
d0458d5
Cost per query adjustment
NiallRees May 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions models/cost_per_query.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ query_cost as (
and query_seconds_per_hour.hour = credits_billed_hourly.hour
inner join {{ ref('daily_rates') }} as daily_rates
on date(query_seconds_per_hour.start_time) = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'WAREHOUSE_METERING'
and daily_rates.usage_type = 'compute'
),

Expand Down Expand Up @@ -188,10 +188,10 @@ inner join credits_billed_daily
on date(all_queries.start_time) = credits_billed_daily.date
left join {{ ref('daily_rates') }} as daily_rates
on date(all_queries.start_time) = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'WAREHOUSE_METERING'
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should probably be CLOUD_SERVICES based on the data in daily rates

and daily_rates.usage_type = 'cloud services'
inner join {{ ref('daily_rates') }} as current_rates
on current_rates.is_latest_rate
and current_rates.service_type = 'COMPUTE'
and current_rates.service_type = 'WAREHOUSE_METERING'
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here - CLOUD_SERVICES

and current_rates.usage_type = 'cloud services'
order by all_queries.start_time asc
32 changes: 15 additions & 17 deletions models/hourly_spend.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ hybrid_table_storage_spend_hourly as (
left join _hybrid_table_terabytes_daily on hours.date = convert_timezone('UTC', _hybrid_table_terabytes_daily.date)
left join {{ ref('daily_rates') }} as daily_rates
on _hybrid_table_terabytes_daily.date = daily_rates.date
and daily_rates.service_type = 'STORAGE'
and daily_rates.service_type = 'HYBRID_TABLE_STORAGE'
and daily_rates.usage_type = 'hybrid table storage'
group by 1, 2, 3, 4, 5
),
Expand Down Expand Up @@ -146,7 +146,7 @@ hybrid_table_requests_spend_hourly as (
and stg_metering_history.service_type = 'HYBRID_TABLE_REQUESTS'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'HYBRID_TABLE_REQUESTS'
and daily_rates.usage_type = 'hybrid table requests'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -262,7 +262,7 @@ compute_spend_hourly as (
)
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'WAREHOUSE_METERING'
and daily_rates.usage_type = 'compute'
where
stg_metering_history.service_type = 'WAREHOUSE_METERING' and stg_metering_history.name != 'CLOUD_SERVICES_ONLY'
Expand All @@ -289,7 +289,7 @@ serverless_task_spend_hourly as (
hours.hour = date_trunc('hour', stg_serverless_task_history.start_time)
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'SERVERLESS_TASK'
and daily_rates.usage_type = 'serverless tasks'
group by 1, 2, 3, 4, 5
),
Expand All @@ -314,7 +314,7 @@ adj_for_incl_cloud_services_hourly as (
hours.hour = stg_metering_daily_history.date
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'CLOUD_SERVICES'
and daily_rates.usage_type = 'cloud services'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -377,7 +377,7 @@ cloud_services_spend_hourly as (
_cloud_services_usage_hourly.date = _cloud_services_billed_daily.date
left join {{ ref('daily_rates') }} as daily_rates
on _cloud_services_usage_hourly.date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'CLOUD_SERVICES'
and daily_rates.usage_type = 'cloud services'

),
Expand Down Expand Up @@ -405,7 +405,7 @@ automatic_clustering_spend_hourly as (
and stg_metering_history.service_type = 'AUTO_CLUSTERING'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'AUTOMATIC_CLUSTERING'
and daily_rates.usage_type = 'automatic clustering'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -433,7 +433,7 @@ materialized_view_spend_hourly as (
and stg_metering_history.service_type = 'MATERIALIZED_VIEW'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'MATERIALIZED_VIEW'
and daily_rates.usage_type = 'materialized views'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -461,7 +461,7 @@ snowpipe_spend_hourly as (
and stg_metering_history.service_type = 'PIPE'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'SNOWPIPE'
and daily_rates.usage_type = 'snowpipe'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -489,7 +489,7 @@ snowpipe_streaming_spend_hourly as (
and stg_metering_history.service_type = 'SNOWPIPE_STREAMING'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'SNOWPIPE_STREAMING'
and daily_rates.usage_type = 'snowpipe streaming'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -517,7 +517,7 @@ query_acceleration_spend_hourly as (
and stg_metering_history.service_type = 'QUERY_ACCELERATION'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'QUERY_ACCELERATION'
and daily_rates.usage_type = 'query acceleration'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -545,7 +545,7 @@ replication_spend_hourly as (
and stg_metering_history.service_type = 'REPLICATION'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'REPLICATION'
and daily_rates.usage_type = 'replication'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -573,7 +573,7 @@ search_optimization_spend_hourly as (
and stg_metering_history.service_type = 'SEARCH_OPTIMIZATION'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'SEARCH_OPTIMIZATION'
and daily_rates.usage_type = 'search optimization'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -601,7 +601,7 @@ snowpark_container_services_spend_hourly as (
and stg_metering_history.service_type = 'SNOWPARK_CONTAINER_SERVICES'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'SNOWPARK_CONTAINER_SERVICES'
and daily_rates.usage_type = 'snowpark container services'
group by 1, 2, 3, 4
),
Expand Down Expand Up @@ -629,13 +629,11 @@ copy_files_spend_hourly as (
and stg_metering_history.service_type = 'COPY_FILES'
left join {{ ref('daily_rates') }} as daily_rates
on hours.hour::date = daily_rates.date
and daily_rates.service_type = 'COMPUTE'
and daily_rates.service_type = 'COPY_FILES'
and daily_rates.usage_type = 'copy files'
group by 1, 2, 3, 4
),



unioned as (
select * from storage_spend_hourly
union all
Expand Down
9 changes: 1 addition & 8 deletions models/staging/stg_rate_sheet_daily.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,6 @@ select
usage_type,
currency,
effective_rate,
case
-- For most Snowflake accounts, the service_type field is always COMPUTE or STORAGE
-- Have recently seen new values introduced for one account: WAREHOUSE_METERING and CLOUD_SERVICES
-- For now, we'll force these to either be COMPUTE or STORAGE since that's what the downstream models expect
-- May adjust this in the future if Snowflake is permanently changing these fields for all accounts and starts offering different credit rates per usage_type
when service_type in ('STORAGE', 'HYBRID_TABLE_STORAGE') then 'STORAGE'
else 'COMPUTE'
end as service_type
service_type
from {{ source('snowflake_organization_usage', 'rate_sheet_daily') }}
order by date
Loading