From c9562c9a522e3b8f3df7857092b0d1e0dc872e27 Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Wed, 15 Jan 2025 13:09:08 +0000 Subject: [PATCH 1/8] handle EDP and Private rate discounts for amortised cost calculations --- .../reporting_awscostentrylineitem_daily_summary.sql | 2 ++ ...g_awscostentrylineitem_summary_by_ec2_compute_p.sql | 2 ++ .../reporting_ocpawscostlineitem_daily_summary.sql | 10 ++++++++-- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index f8f99d3e9e..be1aebfa07 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -114,6 +114,8 @@ FROM ( CASE WHEN lineitem_lineitemtype='Tax' OR lineitem_lineitemtype='Usage' + OR lineitem_lineitemtype='EdpDiscount' + OR lineitem_lineitemtype='PrivateRateDiscount' THEN lineitem_unblendedcost ELSE savingsplan_savingsplaneffectivecost END diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index 0b014be4a1..b54f34335c 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -158,6 +158,8 @@ FROM ( CASE WHEN lineitem_lineitemtype='Tax' OR lineitem_lineitemtype='Usage' + OR lineitem_lineitemtype='EdpDiscount' + OR lineitem_lineitemtype='PrivateRateDiscount' THEN lineitem_unblendedcost ELSE savingsplan_savingsplaneffectivecost END diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index 677fa1dc0a..2104ffb954 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -698,7 +698,7 @@ WHERE ocp.source = {{ocp_source_uuid}} AND aws.resource_id_matched = True AND aws_disk.year = {{year}} AND aws_disk.month = {{month}} - AND aws_disk.ocp_source = {{ocp_source_uuid}} + -- AND aws_disk.ocp_source = {{ocp_source_uuid}} GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels, ocp.volume_labels {% endif %} ; @@ -822,7 +822,7 @@ WHERE ocp.source = {{ocp_source_uuid}} AND aws_disk.capacity != pv_cap.total_pv_capacity -- prevent inserting zero cost rows AND aws_disk.year = {{year}} AND aws_disk.month = {{month}} - AND aws_disk.ocp_source = {{ocp_source_uuid}} + -- AND aws_disk.ocp_source = {{ocp_source_uuid}} GROUP BY aws.uuid, aws.resource_id {% endif %} ; @@ -953,6 +953,12 @@ SELECT aws.uuid as aws_uuid, -- Filter out Node Network Costs since they cannot be attributed to a namespace and are accounted for later AND aws.data_transfer_direction IS NULL AND aws_disk.resource_id is NULL -- exclude any resource used in disk capacity calculations +<<<<<<< Updated upstream +======= + AND aws_disk.year = {{year}} + AND aws_disk.month = {{month}} + -- AND aws_disk.ocp_source = {{ocp_source_uuid}} +>>>>>>> Stashed changes GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels ; From 653803f31c6d283cd7e8093960975f96c0bff2e1 Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Wed, 15 Jan 2025 13:38:34 +0000 Subject: [PATCH 2/8] fix stashed push --- .../reporting_ocpawscostlineitem_daily_summary.sql | 7 ------- 1 file changed, 7 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index 2104ffb954..72d894571b 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -822,7 +822,6 @@ WHERE ocp.source = {{ocp_source_uuid}} AND aws_disk.capacity != pv_cap.total_pv_capacity -- prevent inserting zero cost rows AND aws_disk.year = {{year}} AND aws_disk.month = {{month}} - -- AND aws_disk.ocp_source = {{ocp_source_uuid}} GROUP BY aws.uuid, aws.resource_id {% endif %} ; @@ -953,12 +952,6 @@ SELECT aws.uuid as aws_uuid, -- Filter out Node Network Costs since they cannot be attributed to a namespace and are accounted for later AND aws.data_transfer_direction IS NULL AND aws_disk.resource_id is NULL -- exclude any resource used in disk capacity calculations -<<<<<<< Updated upstream -======= - AND aws_disk.year = {{year}} - AND aws_disk.month = {{month}} - -- AND aws_disk.ocp_source = {{ocp_source_uuid}} ->>>>>>> Stashed changes GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels ; From a8f7beed7ab37f2c1b1a46fdf69141ba1b9d458f Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Wed, 15 Jan 2025 13:39:46 +0000 Subject: [PATCH 3/8] fix stashed push again --- .../trino_sql/reporting_ocpawscostlineitem_daily_summary.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index 72d894571b..677fa1dc0a 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -698,7 +698,7 @@ WHERE ocp.source = {{ocp_source_uuid}} AND aws.resource_id_matched = True AND aws_disk.year = {{year}} AND aws_disk.month = {{month}} - -- AND aws_disk.ocp_source = {{ocp_source_uuid}} + AND aws_disk.ocp_source = {{ocp_source_uuid}} GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels, ocp.volume_labels {% endif %} ; @@ -822,6 +822,7 @@ WHERE ocp.source = {{ocp_source_uuid}} AND aws_disk.capacity != pv_cap.total_pv_capacity -- prevent inserting zero cost rows AND aws_disk.year = {{year}} AND aws_disk.month = {{month}} + AND aws_disk.ocp_source = {{ocp_source_uuid}} GROUP BY aws.uuid, aws.resource_id {% endif %} ; From f784a92ddc39e2f921cdf316fbfe37562de5265c Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Wed, 15 Jan 2025 13:54:43 +0000 Subject: [PATCH 4/8] switch statement to handle all outliers --- ...porting_awscostentrylineitem_daily_summary.sql | 15 ++++++--------- ...costentrylineitem_summary_by_ec2_compute_p.sql | 9 +++------ 2 files changed, 9 insertions(+), 15 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index be1aebfa07..026a3de6ed 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -111,15 +111,12 @@ FROM ( sum(lineitem_blendedcost) as blended_cost, sum(savingsplan_savingsplaneffectivecost) as savingsplan_effective_cost, sum( - CASE - WHEN lineitem_lineitemtype='Tax' - OR lineitem_lineitemtype='Usage' - OR lineitem_lineitemtype='EdpDiscount' - OR lineitem_lineitemtype='PrivateRateDiscount' - THEN lineitem_unblendedcost - ELSE savingsplan_savingsplaneffectivecost - END - ) as calculated_amortized_cost, + CASE + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + THEN savingsplan_savingsplaneffectivecost + ELSE lineitem_unblendedcost + END + ) as calculated_amortized_cost, sum(pricing_publicondemandcost) as public_on_demand_cost, max(pricing_publicondemandrate) as public_on_demand_rate, array_agg(DISTINCT lineitem_resourceid) as resource_ids, diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index b54f34335c..20e4aff6c4 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -156,12 +156,9 @@ FROM ( sum(savingsplan_savingsplaneffectivecost) as savingsplan_effective_cost, sum( CASE - WHEN lineitem_lineitemtype='Tax' - OR lineitem_lineitemtype='Usage' - OR lineitem_lineitemtype='EdpDiscount' - OR lineitem_lineitemtype='PrivateRateDiscount' - THEN lineitem_unblendedcost - ELSE savingsplan_savingsplaneffectivecost + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + THEN savingsplan_savingsplaneffectivecost + ELSE lineitem_unblendedcost END ) as calculated_amortized_cost, sum(pricing_publicondemandcost) as public_on_demand_cost, From 3d6a296db498c1c493e39647a50a641891939aee Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Wed, 15 Jan 2025 20:21:48 +0000 Subject: [PATCH 5/8] Update reporting_awscostentrylineitem_daily_summary.sql --- .../reporting_awscostentrylineitem_daily_summary.sql | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index 026a3de6ed..7095512938 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -111,12 +111,12 @@ FROM ( sum(lineitem_blendedcost) as blended_cost, sum(savingsplan_savingsplaneffectivecost) as savingsplan_effective_cost, sum( - CASE - WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' - THEN savingsplan_savingsplaneffectivecost - ELSE lineitem_unblendedcost - END - ) as calculated_amortized_cost, + CASE + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + THEN savingsplan_savingsplaneffectivecost + ELSE lineitem_unblendedcost + END + ) as calculated_amortized_cost, sum(pricing_publicondemandcost) as public_on_demand_cost, max(pricing_publicondemandrate) as public_on_demand_rate, array_agg(DISTINCT lineitem_resourceid) as resource_ids, From 9a9e98f919b6d2edaf7553a8fe9f9c67589a040d Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Thu, 16 Jan 2025 11:59:04 +0000 Subject: [PATCH 6/8] correcting sql indent --- ...ing_awscostentrylineitem_daily_summary.sql | 1 + ...entrylineitem_summary_by_ec2_compute_p.sql | 99 ++++++++++--------- 2 files changed, 51 insertions(+), 49 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index 7095512938..c289080594 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -113,6 +113,7 @@ FROM ( sum( CASE WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + OR lineitem_lineitemtype='SavingsPlanNegation' THEN savingsplan_savingsplaneffectivecost ELSE lineitem_unblendedcost END diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index 20e4aff6c4..ed2c367511 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -112,57 +112,58 @@ SELECT uuid() as uuid, aa.id as account_alias_id FROM ( SELECT min(date(lineitem_usagestartdate)) as usage_start, - max(date(lineitem_usagestartdate)) as usage_end, - max(lineitem_usageaccountid) as usage_account_id, - lineitem_resourceid as resource_id, - max(nullif(product_operatingsystem, '')) as operating_system, - max(nullif(product_region, '')) as region, - max(nullif(pricing_unit, '')) as unit, - -- SavingsPlanNegation needs to be negated to prevent duplicate usage COST-5369 - sum( - CASE - WHEN lineitem_lineitemtype='SavingsPlanNegation' - THEN 0.0 - ELSE lineitem_usageamount - END - ) as usage_amount, - max(lineitem_normalizationfactor) as normalization_factor, - sum(lineitem_normalizedusageamount) as normalized_usage_amount, - max(lineitem_currencycode) as currency_code, - max(lineitem_unblendedrate) as unblended_rate, - /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items - that offset that cost. - https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html - */ - sum( - CASE - WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' - THEN 0.0 - ELSE lineitem_unblendedcost - END - ) as unblended_cost, - max(lineitem_blendedrate) as blended_rate, + max(date(lineitem_usagestartdate)) as usage_end, + max(lineitem_usageaccountid) as usage_account_id, + lineitem_resourceid as resource_id, + max(nullif(product_operatingsystem, '')) as operating_system, + max(nullif(product_region, '')) as region, + max(nullif(pricing_unit, '')) as unit, + -- SavingsPlanNegation needs to be negated to prevent duplicate usage COST-5369 + sum( + CASE + WHEN lineitem_lineitemtype='SavingsPlanNegation' + THEN 0.0 + ELSE lineitem_usageamount + END + ) as usage_amount, + max(lineitem_normalizationfactor) as normalization_factor, + sum(lineitem_normalizedusageamount) as normalized_usage_amount, + max(lineitem_currencycode) as currency_code, + max(lineitem_unblendedrate) as unblended_rate, /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items - that offset that cost. - https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html + that offset that cost. + https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html */ - sum( - CASE - WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' - THEN 0.0 - ELSE lineitem_blendedcost - END - ) as blended_cost, - sum(savingsplan_savingsplaneffectivecost) as savingsplan_effective_cost, - sum( - CASE - WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' - THEN savingsplan_savingsplaneffectivecost - ELSE lineitem_unblendedcost - END - ) as calculated_amortized_cost, - sum(pricing_publicondemandcost) as public_on_demand_cost, - max(pricing_publicondemandrate) as public_on_demand_rate + sum( + CASE + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + THEN 0.0 + ELSE lineitem_unblendedcost + END + ) as unblended_cost, + max(lineitem_blendedrate) as blended_rate, + /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items + that offset that cost. + https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html + */ + sum( + CASE + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + THEN 0.0 + ELSE lineitem_blendedcost + END + ) as blended_cost, + sum(savingsplan_savingsplaneffectivecost) as savingsplan_effective_cost, + sum( + CASE + WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage' + OR lineitem_lineitemtype='SavingsPlanNegation' + THEN savingsplan_savingsplaneffectivecost + ELSE lineitem_unblendedcost + END + ) as calculated_amortized_cost, + sum(pricing_publicondemandcost) as public_on_demand_cost, + max(pricing_publicondemandrate) as public_on_demand_rate FROM hive.{{schema | sqlsafe}}.aws_line_items_daily as lid WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' From 896572fce7a38748264a7401159c328efddcda7a Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Thu, 16 Jan 2025 12:01:07 +0000 Subject: [PATCH 7/8] comment indent --- ...eporting_awscostentrylineitem_summary_by_ec2_compute_p.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index ed2c367511..f9c9e4af2a 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -143,8 +143,8 @@ FROM ( ) as unblended_cost, max(lineitem_blendedrate) as blended_rate, /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items - that offset that cost. - https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html + that offset that cost. + https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html */ sum( CASE From 49639224a88bed39683f1f24291db2e2a35870ba Mon Sep 17 00:00:00 2001 From: Luke Couzens Date: Thu, 16 Jan 2025 12:03:14 +0000 Subject: [PATCH 8/8] comment indent2 --- ...ting_awscostentrylineitem_summary_by_ec2_compute_p.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index f9c9e4af2a..2b674ab73a 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -142,10 +142,10 @@ FROM ( END ) as unblended_cost, max(lineitem_blendedrate) as blended_rate, - /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items - that offset that cost. - https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html - */ + /* SavingsPlanCoveredUsage entries have corresponding SavingsPlanNegation line items + that offset that cost. + https://docs.aws.amazon.com/cur/latest/userguide/cur-sp.html + */ sum( CASE WHEN lineitem_lineitemtype='SavingsPlanCoveredUsage'