Skip to content

Commit

Permalink
add trino_schema_prefix to sql
Browse files Browse the repository at this point in the history
  • Loading branch information
maskarb committed Jan 16, 2025
1 parent d21adc7 commit f74851c
Show file tree
Hide file tree
Showing 61 changed files with 211 additions and 209 deletions.
2 changes: 1 addition & 1 deletion koku/hcs/database/sql/reporting_aws_hcs_daily_summary.sql
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ SELECT
{{ebs_acct_num}} as ebs_account_id,
{{org_id}} as org_id
FROM
hive.{{schema | sqlsafe}}.{{table | sqlsafe}}
hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}}
WHERE
source = {{provider_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ SELECT
{{ebs_acct_num}} as ebs_account_id,
{{org_id}} as org_id
FROM
hive.{{schema | sqlsafe}}.{{table | sqlsafe}}
hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}}
WHERE
source = {{provider_uuid}}
AND year = {{year}}
Expand Down
2 changes: 1 addition & 1 deletion koku/hcs/database/sql/reporting_gcp_hcs_daily_summary.sql
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
SELECT *, {{ebs_acct_num}} as ebs_account_id, {{org_id}} as org_id
FROM hive.{{schema | sqlsafe}}.{{table | sqlsafe}}
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.{{table | sqlsafe}}
WHERE ( sku_description LIKE '%RedHat%'
OR sku_description LIKE '%Red Hat%'
OR service_description LIKE '%Red Hat%')
Expand Down
16 changes: 9 additions & 7 deletions koku/koku/test_trino_db_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ def test_executescript_jinja_conditionals(self):
"""
sqlscript = """
{% if populate %}
SELECT * FROM hive.{{schema | sqlsafe}}.fake_table
SELECT * FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.fake_table
{% endif %}
;
"""
Expand Down Expand Up @@ -61,23 +61,25 @@ def test_executescript(self):
Test execution of a buffer containing multiple statements
"""
sqlscript = """
drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}};
create table hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}
drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}};
create table hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}
(
id varchar,
i_data integer,
t_data varchar
);
insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data)
insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data)
values (cast(uuid() as varchar), 10, 'default');
insert into hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data)
insert into hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} (id, i_data, t_data)
values (cast(uuid() as varchar), {{int_data}}, {{txt_data}});
select t_data from hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}} where i_data = {{int_data}};
select t_data
from hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}}
where i_data = {{int_data}};
drop table if exists hive.{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}};
drop table if exists hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.__test_{{uuid | sqlsafe}};
"""
conn = FakeTrinoConn()
params = {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Now create our proper table if it does not exist
CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily
CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily
(
lineitem_resourceid varchar,
lineitem_usagestartdate timestamp(3),
Expand Down Expand Up @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_aws_openshift_daily
;

-- Direct resource matching
INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily (
INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_aws_openshift_daily (
lineitem_resourceid,
lineitem_usagestartdate,
bill_payeraccountid,
Expand Down Expand Up @@ -85,7 +85,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_aws_openshift_daily (
)
WITH cte_aws_resource_names AS (
SELECT DISTINCT lineitem_resourceid
FROM hive.{{schema | sqlsafe}}.aws_line_items_daily
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily
WHERE source = {{aws_source_uuid}}
AND year = {{year}}
AND month = {{month}}
Expand All @@ -94,7 +94,7 @@ WITH cte_aws_resource_names AS (
),
cte_array_agg_nodes AS (
SELECT DISTINCT resource_id
FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily
WHERE source = {{ocp_source_uuid}}
AND resource_id != ''
AND year = {{year}}
Expand All @@ -104,7 +104,7 @@ cte_array_agg_nodes AS (
),
cte_array_agg_volumes AS (
SELECT DISTINCT persistentvolume, csi_volume_handle
FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily
WHERE source = {{ocp_source_uuid}}
AND persistentvolume != ''
AND year = {{year}}
Expand Down Expand Up @@ -174,7 +174,7 @@ SELECT aws.lineitem_resourceid,
aws.year,
aws.month,
cast(day(aws.lineitem_usagestartdate) as varchar) as day
FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws
LEFT JOIN cte_matchable_resource_names AS resource_names
ON resource_names.lineitem_resourceid = aws.lineitem_resourceid
LEFT JOIN cte_agg_tags AS tag_matches
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p (
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_accou
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_regio
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_servi
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p (
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid,
max(cost_category_id)
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_database_summary_p (
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_network_summary_p (
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p (
sum(markup_cost_amortized),
max(currency_code),
cast({{aws_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary
WHERE aws_source = {{aws_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
WITH cte_aws_resource_ids AS (
SELECT DISTINCT lineitem_resourceid,
aws.source
FROM hive.{{schema | sqlsafe}}.aws_line_items_daily AS aws
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.aws_line_items_daily AS aws
WHERE aws.lineitem_usagestartdate >= {{start_date}}
AND aws.lineitem_usagestartdate < date_add('day', 1, {{end_date}})
AND aws.lineitem_resourceid IS NOT NULL
Expand All @@ -16,7 +16,7 @@ WITH cte_aws_resource_ids AS (
cte_ocp_resource_ids AS (
SELECT DISTINCT resource_id,
ocp.source
FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp
WHERE ocp.interval_start >= {{start_date}}
AND ocp.interval_start < date_add('day', 1, {{end_date}})
AND ocp.resource_id IS NOT NULL
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Now create our proper table if it does not exist
CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_daily
CREATE TABLE IF NOT EXISTS hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily
(
accountname varchar,
additionalinfo varchar,
Expand Down Expand Up @@ -45,7 +45,7 @@ CREATE TABLE IF NOT EXISTS hive.{{schema | sqlsafe}}.managed_azure_openshift_dai
;

-- Direct resource matching
INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily (
INSERT INTO hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.managed_azure_openshift_daily (
accountname,
additionalinfo,
billingcurrency,
Expand Down Expand Up @@ -89,7 +89,7 @@ INSERT INTO hive.{{schema | sqlsafe}}.managed_azure_openshift_daily (
)
WITH cte_azure_resource_names AS (
SELECT DISTINCT resourceid
FROM hive.{{schema | sqlsafe}}.azure_line_items
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items
WHERE source = {{azure_source_uuid}}
AND year = {{year}}
AND month = {{month}}
Expand All @@ -98,7 +98,7 @@ WITH cte_azure_resource_names AS (
),
cte_array_agg_nodes AS (
SELECT DISTINCT node
FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily
WHERE source = {{ocp_source_uuid}}
AND year = {{year}}
AND month = {{month}}
Expand All @@ -107,7 +107,7 @@ cte_array_agg_nodes AS (
),
cte_array_agg_volumes AS (
SELECT DISTINCT persistentvolume, csi_volume_handle
FROM hive.{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_storage_usage_line_items_daily
WHERE source = {{ocp_source_uuid}}
AND year = {{year}}
AND month = {{month}}
Expand Down Expand Up @@ -180,7 +180,7 @@ SELECT azure.accountname,
azure.year,
azure.month,
cast(day(azure.date) as varchar) as day
FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure
LEFT JOIN cte_matchable_resource_names AS resource_names
ON azure.resourceid = resource_names.resourceid
LEFT JOIN cte_agg_tags AS tag_matches
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p (
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_acc
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_loc
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_ser
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p (
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid,
max(cost_category_id) as cost_category_id
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_database_summary_p
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_network_summary_p (
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p (
sum(markup_cost) as markup_cost,
max(currency) as currency,
cast({{azure_source_uuid}} as uuid) as source_uuid
FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary
WHERE azure_source = {{azure_source_uuid}}
AND ocp_source = {{ocp_source_uuid}}
AND year = {{year}}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
WITH cte_azure_instances AS (
SELECT DISTINCT split_part(azure.resourceid, '/', 9) as instance,
azure.source
FROM hive.{{schema | sqlsafe}}.azure_line_items AS azure
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.azure_line_items AS azure
WHERE azure.date >= {{start_date}}
AND azure.date < date_add('day', 1, {{end_date}})
{% if azure_provider_uuid %}
Expand All @@ -14,7 +14,7 @@ WITH cte_azure_instances AS (
cte_ocp_nodes AS (
SELECT DISTINCT ocp.node,
ocp.source
FROM hive.{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp
FROM hive.{{trino_schema_prefix | sqlsafe}}{{schema | sqlsafe}}.openshift_pod_usage_line_items_daily AS ocp
WHERE ocp.interval_start >= {{start_date}}
AND ocp.interval_start < date_add('day', 1, {{end_date}})
AND ocp.node IS NOT NULL
Expand Down
Loading

0 comments on commit f74851c

Please sign in to comment.