From f74851c2bcae996786ced9465b36f739b94dfeb0 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 15 Jan 2025 19:34:23 -0500 Subject: [PATCH] add trino_schema_prefix to sql --- .../sql/reporting_aws_hcs_daily_summary.sql | 2 +- .../sql/reporting_azure_hcs_daily_summary.sql | 2 +- .../sql/reporting_gcp_hcs_daily_summary.sql | 2 +- koku/koku/test_trino_db_utils.py | 16 ++-- .../openshift/managed_aws_openshift_daily.sql | 12 +-- .../reporting_ocpaws_compute_summary_p.sql | 2 +- ...rting_ocpaws_cost_summary_by_account_p.sql | 2 +- ...orting_ocpaws_cost_summary_by_region_p.sql | 2 +- ...rting_ocpaws_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpaws_cost_summary_p.sql | 2 +- .../reporting_ocpaws_database_summary_p.sql | 2 +- .../reporting_ocpaws_network_summary_p.sql | 2 +- .../reporting_ocpaws_storage_summary_p.sql | 2 +- ...porting_ocpinfrastructure_provider_map.sql | 4 +- .../managed_azure_openshift_daily.sql | 12 +-- .../reporting_ocpazure_compute_summary_p.sql | 2 +- ...ing_ocpazure_cost_summary_by_account_p.sql | 2 +- ...ng_ocpazure_cost_summary_by_location_p.sql | 2 +- ...ing_ocpazure_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpazure_cost_summary_p.sql | 2 +- .../reporting_ocpazure_database_summary_p.sql | 2 +- .../reporting_ocpazure_network_summary_p.sql | 2 +- .../reporting_ocpazure_storage_summary_p.sql | 2 +- ...porting_ocpinfrastructure_provider_map.sql | 4 +- .../openshift/managed_gcp_openshift_daily.sql | 12 +-- .../reporting_ocpgcp_compute_summary_p.sql | 2 +- ...rting_ocpgcp_cost_summary_by_account_p.sql | 2 +- ...g_ocpgcp_cost_summary_by_gcp_project_p.sql | 2 +- ...orting_ocpgcp_cost_summary_by_region_p.sql | 2 +- ...rting_ocpgcp_cost_summary_by_service_p.sql | 2 +- .../reporting_ocpgcp_cost_summary_p.sql | 2 +- .../reporting_ocpgcp_database_summary_p.sql | 2 +- .../reporting_ocpgcp_matched_tags.sql | 4 +- .../reporting_ocpgcp_network_summary_p.sql | 2 +- .../reporting_ocpgcp_storage_summary_p.sql | 2 +- ...rting_ocpgcpcostlineitem_daily_summary.sql | 26 +++---- ...costlineitem_daily_summary_resource_id.sql | 38 +++++----- ...porting_ocpinfrastructure_provider_map.sql | 6 +- .../trino_sql/ocp_special_matched_tags.sql | 4 +- ...ing_awscostentrylineitem_daily_summary.sql | 2 +- ...entrylineitem_summary_by_ec2_compute_p.sql | 6 +- ...g_azurecostentrylineitem_daily_summary.sql | 2 +- ...ing_gcpcostentrylineitem_daily_summary.sql | 2 +- ...ing_ocicostentrylineitem_daily_summary.sql | 4 +- .../reporting_ocpaws_matched_tags.sql | 4 +- ...rting_ocpawscostlineitem_daily_summary.sql | 76 +++++++++---------- .../reporting_ocpazure_matched_tags.sql | 4 +- ...ing_ocpazurecostlineitem_daily_summary.sql | 68 ++++++++--------- ...porting_ocpusagelineitem_daily_summary.sql | 24 +++--- .../gcp/managed_ocp_on_gcp_verification.sql | 4 +- .../verify/gcp/managed_resources.sql | 6 +- .../managed_ocp_on_aws_verification.sql | 4 +- .../managed_ocp_on_azure_verification.sql | 4 +- .../aws/determine_ids_for_provider.sql | 2 +- ...termine_resource_ids_for_usage_account.sql | 2 +- koku/subs/trino_sql/aws/subs_row_count.sql | 2 +- koku/subs/trino_sql/aws/subs_summary.sql | 2 +- .../azure/determine_ids_for_provider.sql | 2 +- ...termine_resource_ids_for_usage_account.sql | 2 +- koku/subs/trino_sql/azure/subs_row_count.sql | 2 +- koku/subs/trino_sql/azure/subs_summary.sql | 2 +- 61 files changed, 211 insertions(+), 209 deletions(-) diff --git a/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql index 173d7dca04..24199ab832 100644 --- a/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql @@ -52,7 +52,7 @@ SELECT {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id FROM - hive.{{schema | sqlsafe}}.{{table | sqlsafe}} + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE source = {{provider_uuid}} AND year = {{year}} diff --git a/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql index c51313a2a0..f5908b51cf 100644 --- a/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_azure_hcs_daily_summary.sql @@ -3,7 +3,7 @@ SELECT {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id FROM - hive.{{schema | sqlsafe}}.{{table | sqlsafe}} + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE source = {{provider_uuid}} AND year = {{year}} diff --git a/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql b/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql index d80cdafc96..c6429876f4 100644 --- a/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql +++ b/koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql @@ -1,5 +1,5 @@ SELECT *, {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id -FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}} +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} WHERE ( sku_description LIKE '%RedHat%' OR sku_description LIKE '%Red Hat%' OR service_description LIKE '%Red Hat%') diff --git a/koku/koku/test_trino_db_utils.py b/koku/koku/test_trino_db_utils.py index e7732be08f..f154e43701 100644 --- a/koku/koku/test_trino_db_utils.py +++ b/koku/koku/test_trino_db_utils.py @@ -30,7 +30,7 @@ def test_executescript_jinja_conditionals(self): """ sqlscript = """ {% if populate %} -SELECT * FROM hive.{{schema | sqlsafe}}.fake_table +SELECT * FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.fake_table {% endif %} ; """ @@ -61,23 +61,25 @@ def test_executescript(self): Test execution of a buffer containing multiple statements """ sqlscript = """ -drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; -create table hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} +drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; +create table hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} ( id varchar, i_data integer, t_data varchar ); -insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) +insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) values (cast(uuid() as varchar), 10, 'default'); -insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) +insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data) values (cast(uuid() as varchar), {{int_data}}, {{txt_data}}); -select t_data from hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} where i_data = {{int_data}}; +select t_data + from hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} +where i_data = {{int_data}}; -drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; +drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}; """ conn = FakeTrinoConn() params = { diff --git a/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql b/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql index 65d70bb6ae..38a1879690 100644 --- a/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql +++ b/koku/masu/database/trino_sql/aws/openshift/managed_aws_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily ( lineitem_resourceid varchar, lineitem_usagestartdate timestamp(3), @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily ( lineitem_resourceid, lineitem_usagestartdate, bill_payeraccountid, @@ -85,7 +85,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily ( ) WITH cte_aws_resource_names AS ( SELECT DISTINCT lineitem_resourceid - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily WHERE source = {{aws_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -94,7 +94,7 @@ WITH cte_aws_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT resource_id - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND resource_id != '' AND year = {{year}} @@ -104,7 +104,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND persistentvolume != '' AND year = {{year}} @@ -174,7 +174,7 @@ SELECT aws.lineitem_resourceid, aws.year, aws.month, cast(day(aws.lineitem_usagestartdate) as varchar) as day -FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws LEFT JOIN cte_matchable_resource_names AS resource_names ON resource_names.lineitem_resourceid = aws.lineitem_resourceid LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index 3b20e001ac..c663fc26c7 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -42,7 +42,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 2cdc055dcd..ff850851e5 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -34,7 +34,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_accou sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index 352feacb63..cbd37f5bf2 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_regio sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index ef74f74521..b8fd333686 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_servi sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 1938331aff..0889a449b8 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -32,7 +32,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid, max(cost_category_id) - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index 03abcbd246..d1b65b1d33 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 04e1f702d1..9136747baa 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index a376e83d33..ce6de6528a 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql index 11ce0b6397..e6b01d8d99 100644 --- a/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/aws/reporting_ocpinfrastructure_provider_map.sql @@ -2,7 +2,7 @@ WITH cte_aws_resource_ids AS ( SELECT DISTINCT lineitem_resourceid, aws.source - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws WHERE aws.lineitem_usagestartdate >= {{start_date}} AND aws.lineitem_usagestartdate < date_add('day', 1, {{end_date}}) AND aws.lineitem_resourceid IS NOT NULL @@ -16,7 +16,7 @@ WITH cte_aws_resource_ids AS ( cte_ocp_resource_ids AS ( SELECT DISTINCT resource_id, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.resource_id IS NOT NULL diff --git a/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql b/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql index c342e1989c..2da50a3195 100644 --- a/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql +++ b/koku/masu/database/trino_sql/azure/openshift/managed_azure_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily ( accountname varchar, additionalinfo varchar, @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_dai ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily ( accountname, additionalinfo, billingcurrency, @@ -89,7 +89,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily ( ) WITH cte_azure_resource_names AS ( SELECT DISTINCT resourceid - FROM hive.{{schema | sqlsafe}}.azure_line_items + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{azure_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -98,7 +98,7 @@ WITH cte_azure_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -107,7 +107,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -180,7 +180,7 @@ SELECT azure.accountname, azure.year, azure.month, cast(day(azure.date) as varchar) as day -FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure LEFT JOIN cte_matchable_resource_names AS resource_names ON azure.resourceid = resource_names.resourceid LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index bd323cc2c2..7832cc1efe 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -30,7 +30,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 7d6ff2fb50..5601f4c6a7 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -22,7 +22,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_acc sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index 4df03bdbec..8bab6c5086 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_loc sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 9943268054..1e2e25cf9a 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_ser sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index 2233cc81df..81c8193302 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -20,7 +20,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid, max(cost_category_id) as cost_category_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 3ebe4d4dfe..1b124c80d9 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_database_summary_p sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 150118b550..4930019aeb 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 29d6c87236..8fb3559f9e 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql index 2b7eb0ef10..c4b44414cc 100644 --- a/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/azure/reporting_ocpinfrastructure_provider_map.sql @@ -2,7 +2,7 @@ WITH cte_azure_instances AS ( SELECT DISTINCT split_part(azure.resourceid, '/', 9) as instance, azure.source - FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure WHERE azure.date >= {{start_date}} AND azure.date < date_add('day', 1, {{end_date}}) {% if azure_provider_uuid %} @@ -14,7 +14,7 @@ WITH cte_azure_instances AS ( cte_ocp_nodes AS ( SELECT DISTINCT ocp.node, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.node IS NOT NULL diff --git a/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql b/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql index 8718576a0e..b1c1f040a3 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/managed_gcp_openshift_daily.sql @@ -1,5 +1,5 @@ -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily ( invoice_month varchar, billing_account_id varchar, @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ; -- Direct resource matching -INSERT INTO hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily ( invoice_month, billing_account_id, project_id, @@ -63,7 +63,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily ( ) WITH cte_gcp_resource_names AS ( SELECT DISTINCT resource_name - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily WHERE source = {{gcp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -72,7 +72,7 @@ WITH cte_gcp_resource_names AS ( ), cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -81,7 +81,7 @@ cte_array_agg_nodes AS ( ), cte_array_agg_volumes AS ( SELECT DISTINCT persistentvolume, csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -141,7 +141,7 @@ SELECT gcp.invoice_month, gcp.year, gcp.month, cast(day(gcp.usage_start_time) as varchar) as day -FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp LEFT JOIN cte_matchable_resource_names AS resource_names ON gcp.resource_name = resource_names.resource_name LEFT JOIN cte_agg_tags AS tag_matches diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index bb99732422..bbaad6a8bd 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -31,7 +31,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index 9fdbe05d37..2647a6f7cc 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_accou cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index 2fafd275ca..448d0e2585 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_p cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index ef85f0a62c..81e692ff5b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_regio cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index c135be4690..70fac01c78 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -29,7 +29,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_servi cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index 780ccac4c2..da2ea2d37b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index fafbd6ead9..a9ec9c1004 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( cast({{gcp_source_uuid}} as uuid) as gcp_source, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql index f38a50e803..360a2c6b8f 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_matched_tags.sql @@ -1,7 +1,7 @@ WITH cte_unnested_gcp_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp CROSS JOIN UNNEST(cast(json_parse(labels) as map(varchar, varchar))) AS tags(key, value) WHERE source = {{gcp_source_uuid}} AND year = {{year}} @@ -14,7 +14,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 2cd2b457b2..52d7734c7f 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 7231fd38e5..3e9136354a 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql index f82528848b..1881cc9506 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql @@ -1,5 +1,5 @@ -- First we'll store the data in a "temp" table to do our grouping against -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid varchar, cluster_id varchar, @@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid varchar, cluster_id varchar, @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- OCP ON GCP kubernetes-io-cluster-{cluster_id} label is applied on the VM and is exclusively a pod cost -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -210,8 +210,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND (strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_id | sqlsafe}}') != 0 -- THIS IS THE SPECIFIC TO OCP ON GCP TAG MATCH OR strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_alias | sqlsafe}}') != 0) @@ -232,7 +232,7 @@ GROUP BY gcp.uuid, ocp.namespace, gcp.invoice_month, ocp.data_source, ocp.pod_la ; -- direct tag matching, these costs are split evenly between pod and storage since we don't have the info to quantify them separately -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -328,8 +328,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -343,7 +343,7 @@ JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp AND ocp.namespace != 'Platform unallocated' AND ocp.namespace != 'Network unattributed' AND ocp.namespace != 'Storage unattributed' -LEFT JOIN hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds +LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds ON gcp.uuid = pds.gcp_uuid WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} @@ -361,7 +361,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, gcp.invoice_month ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -413,7 +413,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.gcp_uuid, count(*) as gcp_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds GROUP BY gcp_uuid ) SELECT pds.gcp_uuid, @@ -498,7 +498,7 @@ SELECT pds.gcp_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds JOIN cte_rankings as r ON pds.gcp_uuid = r.gcp_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -580,7 +580,7 @@ SELECT uuid(), cast(gcp_source as UUID), credit_amount, invoice_month -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql index 15f0fec8d6..342bd9d024 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid varchar, cluster_id varchar, @@ -114,7 +114,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid varchar, cluster_id varchar, @@ -171,7 +171,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineite ) WITH(format = 'PARQUET', partitioned_by=ARRAY['gcp_source', 'ocp_source', 'year', 'month', 'day']) ; -INSERT INTO hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp ( uuid, usage_start, account_id, @@ -225,7 +225,7 @@ SELECT cast(uuid() as varchar), {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} AND gcp.month = {{month}} @@ -243,7 +243,7 @@ GROUP BY gcp.usage_start_time, 10 -- data transfer direction ; -INSERT INTO hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp ( uuid, usage_start, account_id, @@ -309,7 +309,7 @@ SELECT cast(uuid() as varchar), {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as gcp CROSS JOIN cte_enabled_tag_keys as etk WHERE gcp.source = {{gcp_source_uuid}} AND gcp.year = {{year}} @@ -328,7 +328,7 @@ GROUP BY gcp.usage_start_time, ; -- Direct resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -430,8 +430,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( (strpos(gcp.resource_name, ocp.node) != 0 AND ocp.data_source='Pod') @@ -451,7 +451,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, ocp.pod_labels, ocp.volume_la ; -- direct tag matching, these costs are split evenly between pod and storage since we don't have the info to quantify them separately -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp ( gcp_uuid, cluster_id, cluster_alias, @@ -553,8 +553,8 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -580,7 +580,7 @@ GROUP BY gcp.uuid, ocp.namespace, ocp.data_source, gcp.invoice_month ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -635,7 +635,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.gcp_uuid, count(*) as gcp_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp AS pds GROUP BY gcp_uuid ) SELECT pds.gcp_uuid, @@ -717,7 +717,7 @@ SELECT pds.gcp_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_temp as pds JOIN cte_rankings as r ON pds.gcp_uuid = r.gcp_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -728,7 +728,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month -- and costs are split out by pod metrics, this puts all network costs per node -- into a "Network unattributed" project with no cost split and one record per -- data direction -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary ( gcp_uuid, cluster_id, cluster_alias, @@ -806,8 +806,8 @@ SELECT gcp.uuid as gcp_uuid, cast(year(max(gcp.usage_start)) as varchar) as year, cast(month(max(gcp.usage_start)) as varchar) as month, cast(day(max(gcp.usage_start)) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( (strpos(gcp.resource_name, ocp.node) != 0 AND ocp.data_source='Pod') @@ -923,7 +923,7 @@ SELECT uuid(), cast(gcp_source as UUID), credit_amount, invoice_month -FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql b/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql index 741293fc17..30be38b3c3 100644 --- a/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql +++ b/koku/masu/database/trino_sql/gcp/reporting_ocpinfrastructure_provider_map.sql @@ -8,7 +8,7 @@ cte_distinct_gcp_labels AS ( SELECT DISTINCT labels, source - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily WHERE source = {{gcp_provider_uuid}} AND year = {{year}} AND month = {{month}} @@ -34,7 +34,7 @@ WITH cte_gcp_resource_name AS ( SELECT DISTINCT gcp.resource_name, gcp.source - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily AS gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily AS gcp WHERE gcp.usage_start_time >= {{start_date}} AND gcp.usage_start_time < date_add('day', 1, {{end_date}}) {% if gcp_provider_uuid %} @@ -46,7 +46,7 @@ cte_ocp_nodes AS ( SELECT DISTINCT ocp.node, ocp.source - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp WHERE ocp.interval_start >= {{start_date}} AND ocp.interval_start < date_add('day', 1, {{end_date}}) AND ocp.node IS NOT NULL diff --git a/koku/masu/database/trino_sql/ocp_special_matched_tags.sql b/koku/masu/database/trino_sql/ocp_special_matched_tags.sql index 4b39c4ad41..1d62ae7dba 100644 --- a/koku/masu/database/trino_sql/ocp_special_matched_tags.sql +++ b/koku/masu/database/trino_sql/ocp_special_matched_tags.sql @@ -1,6 +1,6 @@ WITH cte_array_agg_nodes AS ( SELECT DISTINCT node - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{month}} @@ -34,7 +34,7 @@ cte_tag_matches AS ( UNION SELECT distinct format('"openshift_project": "%s"', namespace) - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily WHERE source = {{ocp_source_uuid}} AND month = {{month}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql index f8f99d3e9e..e52d8060f3 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_daily_summary.sql @@ -122,7 +122,7 @@ FROM ( max(pricing_publicondemandrate) as public_on_demand_rate, array_agg(DISTINCT lineitem_resourceid) as resource_ids, count(DISTINCT lineitem_resourceid) as resource_count - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql index 0b014be4a1..ab5d111d6d 100644 --- a/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql +++ b/koku/masu/database/trino_sql/reporting_awscostentrylineitem_summary_by_ec2_compute_p.sql @@ -48,7 +48,7 @@ cte_latest_values as ( costcategory as cost_category, nullif(product_memory, '') as memory, cast(nullif(product_vcpu, '') AS INTEGER) as vcpu - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily as alid + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily as alid WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' @@ -57,7 +57,7 @@ cte_latest_values as ( AND lineitem_resourceid != '' AND lineitem_usagestartdate = ( SELECT max(date(lv.lineitem_usagestartdate)) AS usage_start - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS lv + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS lv WHERE lineitem_resourceid = alid.lineitem_resourceid AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' @@ -164,7 +164,7 @@ FROM ( ) as calculated_amortized_cost, sum(pricing_publicondemandcost) as public_on_demand_cost, max(pricing_publicondemandrate) as public_on_demand_rate - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily as lid + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily as lid WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql index 0315d23ca2..4c16e9e9f3 100644 --- a/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_azurecostentrylineitem_daily_summary.sql @@ -46,7 +46,7 @@ WITH cte_line_items AS ( THEN split_part(unitofmeasure, ' ', 2) ELSE unitofmeasure END as unit_of_measure - FROM hive.{{schema | sqlsafe}}.azure_line_items + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = '{{source_uuid | sqlsafe}}' AND year = '{{year | sqlsafe}}' AND month = '{{month | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql index 8b274d1201..f205880b6d 100644 --- a/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_gcpcostentrylineitem_daily_summary.sql @@ -57,7 +57,7 @@ SELECT uuid() as uuid, UUID '{{source_uuid | sqlsafe}}' as source_uuid, invoice_month, sum(((cast(COALESCE(json_extract_scalar(json_parse(credits), '$["amount"]'), '0')AS decimal(24,9)))*1000000)/1000000) as credit_amount -FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}} +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}} CROSS JOIN cte_pg_enabled_keys as pek WHERE source = '{{source_uuid | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql index 80324b255e..78c288697f 100644 --- a/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocicostentrylineitem_daily_summary.sql @@ -87,8 +87,8 @@ FROM ( max(c.cost_currencycode) as currency, sum(c.cost_mycost) as cost, c.tags as tags - FROM hive.{{schema | sqlsafe}}.oci_cost_line_items as c - JOIN hive.{{schema | sqlsafe}}.oci_usage_line_items as u + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.oci_cost_line_items as c + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.oci_usage_line_items as u ON c.lineItem_intervalUsageStart = u.lineItem_intervalUsageStart AND c.product_resourceId = u.product_resourceId WHERE c.source = '{{source_uuid | sqlsafe}}' diff --git a/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql b/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql index 0e8809707a..c28b2f4b8c 100644 --- a/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql +++ b/koku/masu/database/trino_sql/reporting_ocpaws_matched_tags.sql @@ -13,7 +13,7 @@ WITH cte_enabled_tag_keys AS ( cte_unnested_aws_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws CROSS JOIN UNNEST(cast(json_parse(resourcetags) as map(varchar, varchar))) AS tags(key, value) JOIN cte_enabled_tag_keys AS etk ON any_match(etk.key_array, x->strpos(aws.resourcetags, x) != 0) @@ -28,7 +28,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql index 677fa1dc0a..a48d863ab9 100644 --- a/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpawscostlineitem_daily_summary.sql @@ -54,7 +54,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.aws_openshift_daily_tag_matched_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid varchar, cluster_id varchar, @@ -113,7 +113,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineite ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid varchar, cluster_id varchar, @@ -162,7 +162,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpawscostlineite ) WITH(format = 'PARQUET', partitioned_by=ARRAY['aws_source', 'ocp_source', 'year', 'month', 'day']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( resource_id varchar, capacity integer, @@ -172,7 +172,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.aws_openshift_disk_capaciti month varchar ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']); -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp ( uuid, usage_start, resource_id, @@ -255,7 +255,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws WHERE aws.source = {{aws_source_uuid}} AND aws.year = {{year}} AND aws.month = {{month}} @@ -276,7 +276,7 @@ GROUP BY aws.lineitem_usagestartdate, lineitem_operation ; -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -362,7 +362,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws CROSS JOIN cte_enabled_tag_keys as etk WHERE aws.source = {{aws_source_uuid}} AND aws.year = {{year}} @@ -387,7 +387,7 @@ GROUP BY aws.lineitem_usagestartdate, -- Developer notes -- We can't use the aws_openshift_daily table to calcualte the capacity -- because it has already aggregated cost per each hour. -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp ( resource_id, capacity, usage_start, @@ -405,8 +405,8 @@ cte_ocp_filtered_resources as ( DATE(aws.usage_start) as usage_start, aws.year as year, aws.month as month - FROM hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws - JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null @@ -427,7 +427,7 @@ calculated_capacity AS ( {{ocp_source_uuid}} as ocp_source, {{year}} as year, {{month}} as month - FROM hive.{{schema | sqlsafe}}.aws_line_items as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items as aws INNER JOIN cte_ocp_filtered_resources as ocpaws ON aws.lineitem_resourceid = ocpaws.resource_id AND DATE(aws.lineitem_usagestartdate) = ocpaws.usage_start @@ -448,7 +448,7 @@ WHERE capacity > 0 -- Maintain tag matching logic for disk resources -- until unattributed storage is released -- FIXME: Remove this section when the unleash flag is removed -INSERT INTO hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -484,7 +484,7 @@ WITH cte_enabled_tag_keys AS ( ), cte_csi_volume_handles as ( SELECT distinct csi_volume_handle as csi_volume_handle - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as ocp WHERE ocp.source = {{ocp_source_uuid}} AND ocp.year = {{year}} AND ocp.month = {{month}} @@ -541,7 +541,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as aws CROSS JOIN cte_enabled_tag_keys as etk CROSS JOIN cte_csi_volume_handles as csi WHERE aws.source = {{aws_source_uuid}} @@ -573,7 +573,7 @@ GROUP BY aws.lineitem_usagestartdate, -- Algorhtim: -- (PV Capacity) / Disk Capacity * Cost of Disk -- PV without PVCs are unattributed storage -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -675,13 +675,13 @@ SELECT cast(uuid() as varchar) as aws_uuid, -- need a new uuid or it will dedup {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null AND ocp.csi_volume_handle != '' -JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id WHERE ocp.source = {{ocp_source_uuid}} @@ -706,7 +706,7 @@ GROUP BY aws.uuid, ocp.namespace, ocp.pod_labels, ocp.volume_labels {% if unattributed_storage %} -- Unattributed Storage Cost: -- ((Disk Capacity - Sum(PV capacity) / Disk Capacity) * Cost of Disk -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -749,8 +749,8 @@ WITH cte_total_pv_capacity as ( ocp.persistentvolume, max(ocp.persistentvolumeclaim_capacity_gigabyte) as capacity, aws.resource_id as aws_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON (aws.usage_start = ocp.usage_start) AND strpos(aws.resource_id, ocp.csi_volume_handle) > 0 AND ocp.csi_volume_handle is not null @@ -797,13 +797,13 @@ SELECT cast(uuid() as varchar) as aws_uuid, -- need a new uuid or it will dedup {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.csi_volume_handle) != 0 AND ocp.csi_volume_handle is not null AND ocp.csi_volume_handle != '' -JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id LEFT JOIN cte_total_pv_capacity as pv_cap @@ -828,7 +828,7 @@ GROUP BY aws.uuid, aws.resource_id ; -- Direct resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -932,11 +932,11 @@ SELECT aws.uuid as aws_uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp as aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.resource_id) != 0 - LEFT JOIN hive.{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk + LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_disk_capacities_temp AS aws_disk ON aws_disk.usage_start = aws.usage_start AND aws_disk.resource_id = aws.resource_id AND aws_disk.year = aws.year @@ -957,7 +957,7 @@ SELECT aws.uuid as aws_uuid, ; -- Tag matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp ( aws_uuid, cluster_id, cluster_alias, @@ -1061,8 +1061,8 @@ SELECT aws.uuid as aws_uuid, {{ocp_source_uuid}} as ocp_source, max(aws.year) as year, max(aws.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp as aws + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_tag_matched_temp as aws ON aws.usage_start = ocp.usage_start AND ( json_query(aws.tags, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -1087,7 +1087,7 @@ SELECT aws.uuid as aws_uuid, ; -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid, cluster_id, cluster_alias, @@ -1134,7 +1134,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily WITH cte_rankings AS ( SELECT pds.aws_uuid, count(*) as aws_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds GROUP BY aws_uuid ) SELECT pds.aws_uuid, @@ -1220,7 +1220,7 @@ SELECT pds.aws_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_temp AS pds JOIN cte_rankings as r ON pds.aws_uuid = r.aws_uuid LEFT JOIN postgres.{{schema | sqlsafe}}.reporting_awsaccountalias AS aa @@ -1229,7 +1229,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND year = {{year}} AND month = {{mon ; -- Put Node Network Costs into the Network unattributed namespace -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary ( aws_uuid, cluster_id, cluster_alias, @@ -1314,8 +1314,8 @@ SELECT max(cast(year(aws.usage_start) AS varchar)) AS year, max(cast(month(aws.usage_start) AS varchar)) AS month, max(cast(day(aws.usage_start) AS varchar)) AS day -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp -JOIN hive.{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp AS aws +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp +JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily_resource_matched_temp AS aws ON aws.usage_start = ocp.usage_start AND strpos(aws.resource_id, ocp.resource_id) != 0 LEFT JOIN postgres.{{schema | sqlsafe}}.reporting_awsaccountalias AS aa @@ -1430,7 +1430,7 @@ SELECT uuid(), json_parse(aws_cost_category), cost_category_id, cast(aws_source as UUID) -FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql b/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql index cd55dbc35e..fea5dd92ff 100644 --- a/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql +++ b/koku/masu/database/trino_sql/reporting_ocpazure_matched_tags.sql @@ -3,7 +3,7 @@ WITH cte_unnested_azure_tags AS ( SELECT DISTINCT key, value - FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure CROSS JOIN UNNEST(cast(json_parse(tags) as map(varchar, varchar))) AS tags(key, value) WHERE source = {{azure_source_uuid}} AND year = {{year}} @@ -16,7 +16,7 @@ cte_unnested_ocp_tags AS ( pod_value, volume_key, volume_value - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS ocp CROSS JOIN UNNEST( cast(json_parse(pod_labels) as map(varchar, varchar)), cast(json_parse(volume_labels) as map(varchar, varchar)) diff --git a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql index 0334e1d042..6ffcb9a886 100644 --- a/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpazurecostlineitem_daily_summary.sql @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS {{schema | sqlsafe}}.azure_openshift_daily_tag_matche ) WITH(format = 'PARQUET', partitioned_by=ARRAY['ocp_source', 'year', 'month']) ; -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid varchar, cluster_id varchar, @@ -95,7 +95,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlinei ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid varchar, cluster_id varchar, @@ -136,7 +136,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpazurecostlinei ; -- Now create our proper table if it does not exist -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( resource_id varchar, capacity integer, @@ -148,7 +148,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.azure_openshift_disk_capaci ; -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp ( uuid, usage_start, resource_id, @@ -198,7 +198,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month -FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as azure WHERE azure.source = {{azure_source_uuid}} AND azure.year = {{year}} AND azure.month = {{month}} @@ -218,7 +218,7 @@ GROUP BY azure.date, ; -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp ( uuid, usage_start, resource_id, @@ -279,7 +279,7 @@ SELECT cast(uuid() as varchar) as uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month -FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as azure +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as azure CROSS JOIN cte_enabled_tag_keys as etk WHERE azure.source = {{azure_source_uuid}} AND azure.year = {{year}} @@ -299,7 +299,7 @@ GROUP BY azure.date, ; {% if unattributed_storage %} -INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( resource_id, capacity, usage_start, @@ -310,8 +310,8 @@ INSERT INTO hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp ( WITH cte_ocp_filtered_resources as ( SELECT distinct azure.resource_id as azure_partial_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') @@ -356,7 +356,7 @@ GROUP BY ocp_filtered.azure_partial_resource_id, date(date) -- (PV’s Capacity) / Disk Capacity * Cost of Disk -- PV without PVCs are Unattributed Storage -- 2 volumes can share the same disk id -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -434,15 +434,15 @@ SELECT cast(uuid() as varchar) as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') OR (lower(ocp.csi_volume_handle) = lower(azure.resource_id)) ) - JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk ON az_disk.usage_start = azure.usage_start AND az_disk.resource_id = azure.resource_id WHERE ocp.source = {{ocp_source_uuid}} @@ -467,7 +467,7 @@ SELECT cast(uuid() as varchar) as azure_uuid, {% if unattributed_storage %} -- Unallocated Cost: ((Disk Capacity - Sum(PV capacity) / Disk Capacity) * Cost of Disk -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -502,8 +502,8 @@ WITH cte_total_pv_capacity as ( ocp.persistentvolume, max(ocp.persistentvolumeclaim_capacity_gigabyte) as capacity, azure.resource_id as azure_resource_id - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') @@ -544,15 +544,15 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') OR (lower(ocp.csi_volume_handle) = lower(azure.resource_id)) ) - JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp AS az_disk ON az_disk.usage_start = azure.usage_start AND az_disk.resource_id = azure.resource_id LEFT JOIN cte_total_pv_capacity as pv_cap @@ -574,7 +574,7 @@ SELECT cast(uuid() as varchar) as azure_uuid, -- need a new uuid or it will dedu ; -- Directly Pod resource_id matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -662,14 +662,14 @@ SELECT azure.uuid as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON (azure.usage_start = ocp.usage_start) AND ( (replace(lower(azure.resource_id), '_osdisk', '') = lower(ocp.node) AND ocp.data_source = 'Pod') OR (strpos(azure.resource_id, ocp.persistentvolume) > 0 AND ocp.data_source = 'Storage') ) - LEFT JOIN hive.{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp as disk_cap + LEFT JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_disk_capacities_temp as disk_cap ON azure.resource_id = disk_cap.resource_id AND disk_cap.year = azure.year AND disk_cap.month = azure.month @@ -690,7 +690,7 @@ SELECT azure.uuid as azure_uuid, ; -- Tag matching -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp ( azure_uuid, cluster_id, cluster_alias, @@ -752,8 +752,8 @@ SELECT azure.uuid as azure_uuid, {{ocp_source_uuid}} as ocp_source, max(azure.year) as year, max(azure.month) as month - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_tag_matched_temp as azure ON azure.usage_start = ocp.usage_start AND ( json_query(azure.tags, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace @@ -780,7 +780,7 @@ SELECT azure.uuid as azure_uuid, -- Group by to calculate proper cost per project -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid, cluster_id, cluster_alias, @@ -818,7 +818,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_dai WITH cte_rankings AS ( SELECT pds.azure_uuid, count(*) as azure_uuid_count - FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds GROUP BY azure_uuid ) SELECT pds.azure_uuid, @@ -877,7 +877,7 @@ SELECT pds.azure_uuid, cast(year(usage_start) as varchar) as year, cast(month(usage_start) as varchar) as month, cast(day(usage_start) as varchar) as day -FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_temp AS pds JOIN cte_rankings as r ON pds.azure_uuid = r.azure_uuid WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month = {{month}} @@ -888,7 +888,7 @@ WHERE pds.ocp_source = {{ocp_source_uuid}} AND pds.year = {{year}} AND pds.month -- and costs are split out by pod metrics, this puts all network costs per node -- into a "Network unattributed" project with no cost split and one record per -- data direction -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary ( azure_uuid, cluster_id, cluster_alias, @@ -952,8 +952,8 @@ SELECT azure.uuid as azure_uuid, max(azure.year) as year, max(azure.month) as month, cast(day(max(azure.usage_start)) as varchar) as day - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp - JOIN hive.{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily_resource_matched_temp as azure ON azure.usage_start = ocp.usage_start AND (azure.resource_id = ocp.node AND ocp.data_source = 'Pod') WHERE ocp.source = {{ocp_source_uuid}} @@ -1049,7 +1049,7 @@ SELECT uuid(), json_parse(tags), cost_category_id, cast(azure_source as UUID) -FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql b/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql index 65b857d067..1d52397b0d 100644 --- a/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/reporting_ocpusagelineitem_daily_summary.sql @@ -3,7 +3,7 @@ * This SQL will utilize Trino for the raw line-item data aggregating * and store the results into the koku database summary tables. */ -CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid varchar, report_period_id int, cluster_id varchar, @@ -48,7 +48,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_ ) WITH(format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month', 'day']) ; -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid, report_period_id, cluster_id, @@ -107,7 +107,7 @@ cte_ocp_node_label_line_item_daily AS ( (k,v) -> contains(pek.keys, k) ) as json ) as node_labels - FROM hive.{{schema | sqlsafe}}.openshift_node_labels_line_items_daily AS nli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_node_labels_line_items_daily AS nli CROSS JOIN cte_pg_enabled_keys AS pek WHERE nli.source = {{source}} AND nli.year = {{year}} @@ -128,7 +128,7 @@ cte_ocp_namespace_label_line_item_daily AS ( (k,v) -> contains(pek.keys, k) ) as json ) as namespace_labels - FROM hive.{{schema | sqlsafe}}.openshift_namespace_labels_line_items_daily AS nli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_namespace_labels_line_items_daily AS nli CROSS JOIN cte_pg_enabled_keys AS pek WHERE nli.source = {{source}} AND nli.year = {{year}} @@ -150,7 +150,7 @@ cte_ocp_node_capacity AS ( li.node, max(li.node_capacity_cpu_core_seconds) as node_capacity_cpu_core_seconds, max(li.node_capacity_memory_byte_seconds) as node_capacity_memory_byte_seconds - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items AS li + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items AS li WHERE li.source = {{source}} AND li.year = {{year}} AND li.month = {{month}} @@ -180,8 +180,8 @@ cte_volume_nodes AS ( sli.namespace, uli.node, uli.resource_id - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as sli - JOIN hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as uli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily as sli + JOIN hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as uli ON uli.source = sli.source AND uli.namespace = sli.namespace AND uli.pod = sli.pod @@ -286,7 +286,7 @@ FROM ( max(nc.node_capacity_memory_byte_seconds) / 3600.0 * power(2, -30) as node_capacity_memory_gigabyte_hours, max(cc.cluster_capacity_cpu_core_seconds) / 3600.0 as cluster_capacity_cpu_core_hours, max(cc.cluster_capacity_memory_byte_seconds) / 3600.0 * power(2, -30) as cluster_capacity_memory_gigabyte_hours - FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as li + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily as li CROSS JOIN cte_pg_enabled_keys AS pek LEFT JOIN cte_ocp_node_label_line_item_daily as nli ON nli.node = li.node @@ -408,7 +408,7 @@ FROM ( -- Divide volume usage and requests by the number of nodes that volume is mounted on sum(sli.volume_request_storage_byte_seconds) / max(nc.node_count) as volume_request_storage_byte_seconds, sum(sli.persistentvolumeclaim_usage_byte_seconds) / max(nc.node_count) as persistentvolumeclaim_usage_byte_seconds - FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily sli + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily sli CROSS JOIN cte_pg_enabled_keys AS pek LEFT JOIN cte_volume_nodes as vn ON vn.usage_start = date(sli.interval_start) @@ -457,7 +457,7 @@ What was selected from unallocated capacity. AND lids.namespace != 'Platform unallocated' AND lids.namespace != 'Worker unallocated' */ -INSERT INTO hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( +INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary ( uuid, report_period_id, cluster_id, @@ -528,7 +528,7 @@ cte_unallocated_capacity AS ( cast(year(lids.usage_start) as varchar) as year, cast(month(lids.usage_start) as varchar) as month, cast(day(lids.usage_start) as varchar) as day - FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as lids LEFT JOIN cte_node_role AS nodes ON lids.node = nodes.node AND lids.resource_id = nodes.resource_id @@ -658,7 +658,7 @@ SELECT uuid(), cast(source_uuid as UUID), json_parse(infrastructure_usage_cost), cost_category_id -FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary AS lids WHERE lids.source = {{source}} AND lids.year = {{year}} AND lpad(lids.month, 2, '0') = {{month}} -- Zero pad the month when fewer than 2 characters diff --git a/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql b/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql index b842d1d94c..0dff2ab8cc 100644 --- a/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql +++ b/koku/masu/database/trino_sql/verify/gcp/managed_ocp_on_gcp_verification.sql @@ -9,7 +9,7 @@ SELECT FROM ( SELECT sum(cost) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -17,7 +17,7 @@ FROM ) t1, ( SELECT sum(cost) AS parquet_total_cost - FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.labels, x) != 0) AND parquet_table.ocp_matched = False diff --git a/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql b/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql index 46f3520b4e..ccaa00f6ab 100644 --- a/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql +++ b/koku/masu/database/trino_sql/verify/gcp/managed_resources.sql @@ -9,7 +9,7 @@ cte_resource_breakdown AS ( SUM(cost) AS cost FROM ( SELECT 'parquet' AS source_type, resource_name, usage_start_time, cost - FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_openshift_daily parquet_table WHERE source = {{cloud_source_uuid}} AND year = {{year}} AND month = {{month}} AND (ocp_matched = TRUE OR EXISTS ( @@ -19,7 +19,7 @@ cte_resource_breakdown AS ( )) UNION ALL SELECT 'managed' AS source_type, resource_name, usage_start_time, cost - FROM hive.{{schema | sqlsafe}}.managed_gcp_openshift_daily + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_gcp_openshift_daily WHERE source = {{cloud_source_uuid}} AND year = {{year}} AND month = {{month}} AND (resource_id_matched = TRUE OR matched_tag != '') @@ -51,7 +51,7 @@ cte_initial_cost_check AS ( WHEN SUM(gcp.cost) < MAX(d.parquet_cost) AND SUM(gcp.cost) = MAX(d.managed_cost) THEN TRUE ELSE FALSE END AS parquet_issue - FROM hive.{{schema | sqlsafe}}.gcp_line_items_daily gcp + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.gcp_line_items_daily gcp JOIN cte_discrepancies d ON gcp.resource_name = d.resource_name AND gcp.usage_start_time = d.usage_start_time diff --git a/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql b/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql index 4806b0bc87..6fc97f1ec0 100644 --- a/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql +++ b/koku/masu/database/trino_sql/verify/managed_ocp_on_aws_verification.sql @@ -11,7 +11,7 @@ SELECT FROM ( SELECT sum(lineitem_unblendedcost) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_aws_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -21,7 +21,7 @@ FROM ) t1, ( SELECT sum(lineitem_unblendedcost) as parquet_total_cost - FROM hive.{{schema | sqlsafe}}.aws_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.resourcetags, x) != 0) AND parquet_table.resource_id_matched = False diff --git a/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql b/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql index 7f0d68abc6..5f0e0c1c99 100644 --- a/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql +++ b/koku/masu/database/trino_sql/verify/managed_ocp_on_azure_verification.sql @@ -11,7 +11,7 @@ SELECT FROM ( SELECT sum(costinbillingcurrency) AS managed_total_cost - FROM hive.{{schema | sqlsafe}}.managed_azure_openshift_daily as managed_ocpcloud + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily as managed_ocpcloud WHERE managed_ocpcloud.source = {{cloud_source_uuid}} AND managed_ocpcloud.year = {{year}} AND managed_ocpcloud.month = {{month}} @@ -19,7 +19,7 @@ FROM ) t1, ( SELECT sum(costinbillingcurrency) AS parquet_total_cost - FROM hive.{{schema | sqlsafe}}.azure_openshift_daily as parquet_table + FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_openshift_daily as parquet_table LEFT JOIN cte_agg_tags AS tag_matches ON any_match(tag_matches.matched_tags, x->strpos(parquet_table.tags, x) != 0) AND parquet_table.resource_id_matched = False diff --git a/koku/subs/trino_sql/aws/determine_ids_for_provider.sql b/koku/subs/trino_sql/aws/determine_ids_for_provider.sql index a6bcfad89b..029e9b816c 100644 --- a/koku/subs/trino_sql/aws/determine_ids_for_provider.sql +++ b/koku/subs/trino_sql/aws/determine_ids_for_provider.sql @@ -1,6 +1,6 @@ SELECT DISTINCT lineitem_usageaccountid -FROM hive.{{schema | sqlsafe}}.aws_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql b/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql index a697c56577..9fc7fed824 100644 --- a/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql +++ b/koku/subs/trino_sql/aws/determine_resource_ids_for_usage_account.sql @@ -1,7 +1,7 @@ SELECT lineitem_resourceid, max(lineitem_usagestartdate) -FROM hive.{{schema | sqlsafe}}.aws_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/aws/subs_row_count.sql b/koku/subs/trino_sql/aws/subs_row_count.sql index 7bbb67ba80..a60c25e712 100644 --- a/koku/subs/trino_sql/aws/subs_row_count.sql +++ b/koku/subs/trino_sql/aws/subs_row_count.sql @@ -1,6 +1,6 @@ SELECT count(*) FROM - hive.{{schema | sqlsafe}}.aws_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/aws/subs_summary.sql b/koku/subs/trino_sql/aws/subs_summary.sql index 480fad481d..a5d2520e1d 100644 --- a/koku/subs/trino_sql/aws/subs_summary.sql +++ b/koku/subs/trino_sql/aws/subs_summary.sql @@ -58,7 +58,7 @@ FROM ) as json ) as tags from - hive.{{schema | sqlsafe}}.aws_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/azure/determine_ids_for_provider.sql b/koku/subs/trino_sql/azure/determine_ids_for_provider.sql index b50e9a7bfe..a19a931cbc 100644 --- a/koku/subs/trino_sql/azure/determine_ids_for_provider.sql +++ b/koku/subs/trino_sql/azure/determine_ids_for_provider.sql @@ -1,6 +1,6 @@ SELECT DISTINCT COALESCE(NULLIF(subscriptionid, ''), subscriptionguid) -FROM hive.{{schema | sqlsafe}}.azure_line_items +FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source={{source_uuid}} AND year={{year}} AND month={{month}} diff --git a/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql b/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql index 58f194b5b3..d89de5f46d 100644 --- a/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql +++ b/koku/subs/trino_sql/azure/determine_resource_ids_for_usage_account.sql @@ -14,7 +14,7 @@ FROM ( COALESCE(json_extract_scalar(lower(additionalinfo), '$.vmname'), regexp_extract(resourceid, '([^/]+$)')) as vmname, max(date) max_date FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source={{source_uuid}} AND year={{year}} diff --git a/koku/subs/trino_sql/azure/subs_row_count.sql b/koku/subs/trino_sql/azure/subs_row_count.sql index 7e30c310ca..85540bd69e 100644 --- a/koku/subs/trino_sql/azure/subs_row_count.sql +++ b/koku/subs/trino_sql/azure/subs_row_count.sql @@ -1,7 +1,7 @@ SELECT COUNT(*) FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{ source_uuid }} AND year = {{ year }} diff --git a/koku/subs/trino_sql/azure/subs_summary.sql b/koku/subs/trino_sql/azure/subs_summary.sql index 24d5e4ec0d..7268f62aa8 100644 --- a/koku/subs/trino_sql/azure/subs_summary.sql +++ b/koku/subs/trino_sql/azure/subs_summary.sql @@ -39,7 +39,7 @@ SELECT -- if the VMName isn't present in additionalinfo, the end of the resourceid should be the VMName COALESCE(json_extract_scalar(lower(additionalinfo), '$.vmname'), regexp_extract(resourceid, '([^/]+$)')) as subs_vmname FROM - hive.{{schema | sqlsafe}}.azure_line_items + hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items WHERE source = {{ source_uuid }} AND year = {{ year }}