diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index 514031bcdc..91b615d86f 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -22,6 +22,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: @@ -209,6 +215,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: @@ -387,6 +399,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: APP_POD_NAME valueFrom: fieldRef: @@ -444,6 +462,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: KAFKA_CONNECT value: ${KAFKA_CONNECT} - name: AWS_SHARED_CREDENTIALS_FILE @@ -590,6 +614,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: MASU value: "true" - name: DJANGO_SECRET_KEY @@ -788,6 +818,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -948,6 +984,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER @@ -1115,6 +1157,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER @@ -1279,6 +1327,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1461,6 +1515,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1649,6 +1709,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -1837,6 +1903,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2025,6 +2097,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2217,6 +2295,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2409,6 +2493,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2601,6 +2691,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2791,6 +2887,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -2981,6 +3083,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3171,6 +3279,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3367,6 +3481,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3563,6 +3683,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3759,6 +3885,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -3947,6 +4079,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4135,6 +4273,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4323,6 +4467,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4515,6 +4665,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4707,6 +4863,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -4899,6 +5061,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5085,6 +5253,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5259,6 +5433,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS @@ -5434,6 +5614,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: PROMETHEUS_MULTIPROC_DIR @@ -5467,6 +5653,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: MGMT_IMAGE @@ -5521,25 +5713,6 @@ objects: kind: Secret metadata: name: koku-secret -- apiVersion: v1 - data: - aws-credentials: ${AWS_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-aws -- apiVersion: v1 - data: - gcp-credentials: ${GCP_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-gcp -- apiVersion: v1 - data: - oci-config: ${OCI_CONFIG_EPH} - oci-credentials: ${OCI_CREDENTIALS_EPH} - kind: Secret - metadata: - name: koku-oci - apiVersion: v1 data: nginx.conf: |- @@ -5637,6 +5810,12 @@ parameters: name: IMAGE required: true value: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/koku +- description: DEVELOPMENT ONLY - used to prevent clashing Glue databases during ephemeral + testing + name: SCHEMA_SUFFIX + value: "" +- name: TRINO_SCHEMA_PREFIX + value: "" - name: S3_BUCKET_NAME value: hccm-eph-s3 - name: S3_ROS_BUCKET_NAME @@ -5948,6 +6127,12 @@ parameters: name: AWS_CREDENTIALS_EPH required: true value: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFXU19BQ0NFU1NfS0VZX0lECmF3c19zZWNyZXRfYWNjZXNzX2tleT1BV1NfU0VDUkVUX0FDQ0VTU19LRVkK +- name: AWS_ACCESS_KEY_ID_EPH + required: false + value: AWS_ACCESS_KEY_ID +- name: AWS_SECRET_ACCESS_KEY_EPH + required: false + value: AWS_SECRET_ACCESS_KEY - displayName: GCP Credentials (Ephemeral) name: GCP_CREDENTIALS_EPH required: true diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml index 7722916698..488bc3ca87 100644 --- a/deploy/kustomize/base/base.yaml +++ b/deploy/kustomize/base/base.yaml @@ -73,6 +73,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: PROMETHEUS_MULTIPROC_DIR @@ -109,6 +115,12 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DEVELOPMENT value: ${DEVELOPMENT} - name: MGMT_IMAGE @@ -153,25 +165,28 @@ objects: name: koku-secret data: django-secret-key: "${SECRET_KEY}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-aws - data: - aws-credentials: "${AWS_CREDENTIALS_EPH}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-gcp - data: - gcp-credentials: "${GCP_CREDENTIALS_EPH}" -- apiVersion: v1 - kind: Secret # For ephemeral/local environment only - metadata: - name: koku-oci - data: - oci-credentials: "${OCI_CREDENTIALS_EPH}" - oci-config: "${OCI_CONFIG_EPH}" +# - apiVersion: v1 +# kind: Secret +# metadata: +# name: koku-aws +# data: +# aws-credentials: ${AWS_CREDENTIALS_EPH} +# stringData: +# aws-access-key-id: "${AWS_ACCESS_KEY_ID_EPH}" +# aws-secret-access-key: "${AWS_SECRET_ACCESS_KEY_EPH}" +# - apiVersion: v1 +# kind: Secret # For ephemeral/local environment only +# metadata: +# name: koku-gcp +# data: +# gcp-credentials: "${GCP_CREDENTIALS_EPH}" +# - apiVersion: v1 +# kind: Secret # For ephemeral/local environment only +# metadata: +# name: koku-oci +# data: +# oci-credentials: "${OCI_CREDENTIALS_EPH}" +# oci-config: "${OCI_CONFIG_EPH}" - apiVersion: v1 kind: ConfigMap @@ -275,6 +290,13 @@ parameters: name: IMAGE required: true value: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/koku +- name: SCHEMA_SUFFIX + description: DEVELOPMENT ONLY - used to prevent clashing Glue databases during ephemeral testing + value: "" +- name: TRINO_SCHEMA_PREFIX + value: "" +- name: TRINO_S3A_OR_S3 + value: "s3a" - name: S3_BUCKET_NAME value: hccm-eph-s3 - name: S3_ROS_BUCKET_NAME @@ -601,6 +623,12 @@ parameters: name: AWS_CREDENTIALS_EPH required: true value: W2RlZmF1bHRdCmF3c19hY2Nlc3Nfa2V5X2lkPUFXU19BQ0NFU1NfS0VZX0lECmF3c19zZWNyZXRfYWNjZXNzX2tleT1BV1NfU0VDUkVUX0FDQ0VTU19LRVkK +- name: AWS_ACCESS_KEY_ID_EPH + required: false + value: AWS_ACCESS_KEY_ID +- name: AWS_SECRET_ACCESS_KEY_EPH + required: false + value: AWS_SECRET_ACCESS_KEY - name: GCP_CREDENTIALS_EPH displayName: GCP Credentials (Ephemeral) required: true diff --git a/deploy/kustomize/patches/koku-reads.yaml b/deploy/kustomize/patches/koku-reads.yaml index 646b53bde2..bc052e5883 100644 --- a/deploy/kustomize/patches/koku-reads.yaml +++ b/deploy/kustomize/patches/koku-reads.yaml @@ -13,6 +13,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: diff --git a/deploy/kustomize/patches/koku-writes.yaml b/deploy/kustomize/patches/koku-writes.yaml index ce730a4cfb..06146c181d 100644 --- a/deploy/kustomize/patches/koku-writes.yaml +++ b/deploy/kustomize/patches/koku-writes.yaml @@ -20,6 +20,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: DJANGO_SECRET_KEY valueFrom: secretKeyRef: diff --git a/deploy/kustomize/patches/listener.yaml b/deploy/kustomize/patches/listener.yaml index 86462896d7..020e8c13c6 100644 --- a/deploy/kustomize/patches/listener.yaml +++ b/deploy/kustomize/patches/listener.yaml @@ -17,6 +17,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: KAFKA_CONNECT value: ${KAFKA_CONNECT} - name: AWS_SHARED_CREDENTIALS_FILE diff --git a/deploy/kustomize/patches/masu.yaml b/deploy/kustomize/patches/masu.yaml index 6c087557e5..71c262dacd 100644 --- a/deploy/kustomize/patches/masu.yaml +++ b/deploy/kustomize/patches/masu.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: MASU value: "true" - name: DJANGO_SECRET_KEY diff --git a/deploy/kustomize/patches/nginx.yaml b/deploy/kustomize/patches/nginx.yaml index ede4369b5c..35fb8759c7 100644 --- a/deploy/kustomize/patches/nginx.yaml +++ b/deploy/kustomize/patches/nginx.yaml @@ -18,6 +18,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: APP_POD_NAME valueFrom: fieldRef: diff --git a/deploy/kustomize/patches/scheduler.yaml b/deploy/kustomize/patches/scheduler.yaml index 2f0bbc6576..08808a93f3 100644 --- a/deploy/kustomize/patches/scheduler.yaml +++ b/deploy/kustomize/patches/scheduler.yaml @@ -29,6 +29,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/sources-client.yaml b/deploy/kustomize/patches/sources-client.yaml index 04d9dd6d5b..56aaf66d01 100644 --- a/deploy/kustomize/patches/sources-client.yaml +++ b/deploy/kustomize/patches/sources-client.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER diff --git a/deploy/kustomize/patches/sources-listener.yaml b/deploy/kustomize/patches/sources-listener.yaml index 28627a9226..d1449ba07c 100644 --- a/deploy/kustomize/patches/sources-listener.yaml +++ b/deploy/kustomize/patches/sources-listener.yaml @@ -23,6 +23,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: SOURCES value: "true" - name: SOURCES_PROBE_HEADER diff --git a/deploy/kustomize/patches/worker-celery.yaml b/deploy/kustomize/patches/worker-celery.yaml index 648ba377f8..1c6a7b8fa8 100644 --- a/deploy/kustomize/patches/worker-celery.yaml +++ b/deploy/kustomize/patches/worker-celery.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model-penalty.yaml b/deploy/kustomize/patches/worker-cost-model-penalty.yaml index d89cf88732..097229eb11 100644 --- a/deploy/kustomize/patches/worker-cost-model-penalty.yaml +++ b/deploy/kustomize/patches/worker-cost-model-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model-xl.yaml b/deploy/kustomize/patches/worker-cost-model-xl.yaml index 9981c39008..a4a188dbcc 100644 --- a/deploy/kustomize/patches/worker-cost-model-xl.yaml +++ b/deploy/kustomize/patches/worker-cost-model-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-cost-model.yaml b/deploy/kustomize/patches/worker-cost-model.yaml index 0c5364e2a8..093e3f3186 100644 --- a/deploy/kustomize/patches/worker-cost-model.yaml +++ b/deploy/kustomize/patches/worker-cost-model.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download-penalty.yaml b/deploy/kustomize/patches/worker-download-penalty.yaml index e8f9b9c728..1f24068381 100644 --- a/deploy/kustomize/patches/worker-download-penalty.yaml +++ b/deploy/kustomize/patches/worker-download-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download-xl.yaml b/deploy/kustomize/patches/worker-download-xl.yaml index 19c98fc607..4dcda30ce5 100644 --- a/deploy/kustomize/patches/worker-download-xl.yaml +++ b/deploy/kustomize/patches/worker-download-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-download.yaml b/deploy/kustomize/patches/worker-download.yaml index 2492985ec0..11a1ccbfd8 100644 --- a/deploy/kustomize/patches/worker-download.yaml +++ b/deploy/kustomize/patches/worker-download.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-hcs.yaml b/deploy/kustomize/patches/worker-hcs.yaml index e67a1d0905..45d7039b99 100644 --- a/deploy/kustomize/patches/worker-hcs.yaml +++ b/deploy/kustomize/patches/worker-hcs.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp-penalty.yaml b/deploy/kustomize/patches/worker-ocp-penalty.yaml index 30dc4c67cd..f22ddab309 100644 --- a/deploy/kustomize/patches/worker-ocp-penalty.yaml +++ b/deploy/kustomize/patches/worker-ocp-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp-xl.yaml b/deploy/kustomize/patches/worker-ocp-xl.yaml index fd748efb1f..b15ab6df68 100644 --- a/deploy/kustomize/patches/worker-ocp-xl.yaml +++ b/deploy/kustomize/patches/worker-ocp-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-ocp.yaml b/deploy/kustomize/patches/worker-ocp.yaml index 8cd798ddc0..fd90beca6f 100644 --- a/deploy/kustomize/patches/worker-ocp.yaml +++ b/deploy/kustomize/patches/worker-ocp.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority-penalty.yaml b/deploy/kustomize/patches/worker-priority-penalty.yaml index c4fc9314cd..42b4c889da 100644 --- a/deploy/kustomize/patches/worker-priority-penalty.yaml +++ b/deploy/kustomize/patches/worker-priority-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority-xl.yaml b/deploy/kustomize/patches/worker-priority-xl.yaml index ed4c9f319a..8786b15518 100644 --- a/deploy/kustomize/patches/worker-priority-xl.yaml +++ b/deploy/kustomize/patches/worker-priority-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-priority.yaml b/deploy/kustomize/patches/worker-priority.yaml index 05b64f3bc5..9a5eafee10 100644 --- a/deploy/kustomize/patches/worker-priority.yaml +++ b/deploy/kustomize/patches/worker-priority.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh-penalty.yaml b/deploy/kustomize/patches/worker-refresh-penalty.yaml index 7185748424..09b121737c 100644 --- a/deploy/kustomize/patches/worker-refresh-penalty.yaml +++ b/deploy/kustomize/patches/worker-refresh-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh-xl.yaml b/deploy/kustomize/patches/worker-refresh-xl.yaml index 30526c8825..a5ba56897b 100644 --- a/deploy/kustomize/patches/worker-refresh-xl.yaml +++ b/deploy/kustomize/patches/worker-refresh-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-refresh.yaml b/deploy/kustomize/patches/worker-refresh.yaml index c1b5979770..cf32f20568 100644 --- a/deploy/kustomize/patches/worker-refresh.yaml +++ b/deploy/kustomize/patches/worker-refresh.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-subs-extraction.yaml b/deploy/kustomize/patches/worker-subs-extraction.yaml index b3b213bc02..c8e6e4a352 100644 --- a/deploy/kustomize/patches/worker-subs-extraction.yaml +++ b/deploy/kustomize/patches/worker-subs-extraction.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-subs-transmission.yaml b/deploy/kustomize/patches/worker-subs-transmission.yaml index 23e89d85f9..ea1b32deb7 100644 --- a/deploy/kustomize/patches/worker-subs-transmission.yaml +++ b/deploy/kustomize/patches/worker-subs-transmission.yaml @@ -19,6 +19,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary-penalty.yaml b/deploy/kustomize/patches/worker-summary-penalty.yaml index f2c0c02c18..6069f46799 100644 --- a/deploy/kustomize/patches/worker-summary-penalty.yaml +++ b/deploy/kustomize/patches/worker-summary-penalty.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary-xl.yaml b/deploy/kustomize/patches/worker-summary-xl.yaml index 080247e158..57e079b708 100644 --- a/deploy/kustomize/patches/worker-summary-xl.yaml +++ b/deploy/kustomize/patches/worker-summary-xl.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/deploy/kustomize/patches/worker-summary.yaml b/deploy/kustomize/patches/worker-summary.yaml index a00a0487c9..f8126f6ab2 100644 --- a/deploy/kustomize/patches/worker-summary.yaml +++ b/deploy/kustomize/patches/worker-summary.yaml @@ -25,6 +25,12 @@ env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: SCHEMA_SUFFIX + value: ${SCHEMA_SUFFIX} + - name: TRINO_SCHEMA_PREFIX + value: ${TRINO_SCHEMA_PREFIX} + - name: TRINO_S3A_OR_S3 + value: ${TRINO_S3A_OR_S3} - name: AWS_SHARED_CREDENTIALS_FILE value: ${AWS_SHARED_CREDENTIALS_FILE} - name: GOOGLE_APPLICATION_CREDENTIALS diff --git a/koku/api/report/test/util/model_bakery_loader.py b/koku/api/report/test/util/model_bakery_loader.py index 1643cd7c79..fa8d01cf66 100644 --- a/koku/api/report/test/util/model_bakery_loader.py +++ b/koku/api/report/test/util/model_bakery_loader.py @@ -35,7 +35,6 @@ from reporting.models import AWSAccountAlias from reporting.models import AWSOrganizationalUnit - BILL_MODELS = { Provider.PROVIDER_AWS: "AWSCostEntryBill", Provider.PROVIDER_AWS_LOCAL: "AWSCostEntryBill", @@ -533,7 +532,7 @@ def load_openshift_on_cloud_data(self, provider_type, cluster_id, bills, report_ # update ui tables sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": self.first_start_date, "end_date": self.last_end_date, "source_uuid": provider.uuid, diff --git a/koku/koku/dev_middleware.py b/koku/koku/dev_middleware.py index a980072257..8e034371d2 100644 --- a/koku/koku/dev_middleware.py +++ b/koku/koku/dev_middleware.py @@ -60,6 +60,9 @@ def process_request(self, request): identity_header = settings.DEVELOPMENT_IDENTITY user_dict = identity_header.get("identity", {}).get("user") + org_id = identity_header.get("org_id") or "1234567" + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" user = Mock( spec=User, access=user_dict.get("access", {}), @@ -68,8 +71,8 @@ def process_request(self, request): admin=user_dict.get("is_org_admin", False), customer=Mock( account_id=identity_header.get("account_number", "10001"), - org_id=identity_header.get("org_id", "1234567"), - schema_name=f'org{identity_header.get("org_id", "1234567")}', + org_id=org_id, + schema_name=f"org{org_id}", ), req_id="DEVELOPMENT", ) diff --git a/koku/koku/middleware.py b/koku/koku/middleware.py index cc5c9ffeaf..4d14b1fae0 100644 --- a/koku/koku/middleware.py +++ b/koku/koku/middleware.py @@ -42,7 +42,6 @@ from koku.rbac import RbacConnectionError from koku.rbac import RbacService - MAX_CACHE_SIZE = 10000 USER_CACHE = TTLCache(maxsize=MAX_CACHE_SIZE, ttl=settings.MIDDLEWARE_TIME_TO_LIVE) @@ -331,7 +330,9 @@ def process_request(self, request): # noqa: C901 # Check for customer creation & user creation query_string = "" if request.META["QUERY_STRING"]: - query_string = "?{}".format(request.META["QUERY_STRING"]) + query_string = f"?{request.META['QUERY_STRING']}" + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" stmt = { "method": request.method, "path": request.path + query_string, diff --git a/koku/koku/settings.py b/koku/koku/settings.py index c2b1d61098..be070b5427 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -15,6 +15,7 @@ """ import logging import os +import re import sys from json import JSONDecodeError from zoneinfo import ZoneInfo @@ -137,7 +138,14 @@ MIDDLEWARE_TIME_TO_LIVE = ENVIRONMENT.int("MIDDLEWARE_TIME_TO_LIVE", default=900) # in seconds (default = 15 minutes) DEVELOPMENT = ENVIRONMENT.bool("DEVELOPMENT", default=False) +SCHEMA_SUFFIX = re.sub("[^a-zA-Z0-9_]", "_", ENVIRONMENT.get_value("SCHEMA_SUFFIX", default="")) +print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") if DEVELOPMENT: + if SCHEMA_SUFFIX == "": + SCHEMA_SUFFIX = f"_{ENVIRONMENT.get_value('USER', default='')}" + if not SCHEMA_SUFFIX.startswith("_"): + SCHEMA_SUFFIX = f"_{SCHEMA_SUFFIX}" + print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") DEFAULT_IDENTITY = { "identity": { "account_number": "10001", @@ -487,19 +495,29 @@ REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report") REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report") S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60) -S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +S3_DEFAULT_ENDPOINT = ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1") S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) S3_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET) +# HCS +S3_HCS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) +S3_HCS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) +S3_HCS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET) +S3_HCS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_BUCKET) +S3_HCS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +# ROS S3_ROS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_ROS_BUCKET) S3_ROS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_ROS_BUCKET) S3_ROS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_ROS_BUCKET) S3_ROS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_ROS_BUCKET) +S3_ROS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +# SUBS S3_SUBS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_SUBS_BUCKET) S3_SUBS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_SUBS_BUCKET) S3_SUBS_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_SUBS_BUCKET) S3_SUBS_REGION = CONFIGURATOR.get_object_store_region(REQUESTED_SUBS_BUCKET) +S3_SUBS_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() SKIP_MINIO_DATA_DELETION = ENVIRONMENT.bool("SKIP_MINIO_DATA_DELETION", default=False) PARQUET_PROCESSING_BATCH_SIZE = ENVIRONMENT.int("PARQUET_PROCESSING_BATCH_SIZE", default=200000) @@ -520,6 +538,9 @@ TRINO_HOST = ENVIRONMENT.get_value("TRINO_HOST", default=None) TRINO_PORT = ENVIRONMENT.get_value("TRINO_PORT", default=None) TRINO_DATE_STEP = ENVIRONMENT.int("TRINO_DATE_STEP", default=5) +TRINO_SCHEMA_PREFIX = re.sub("[^a-zA-Z0-9_]", "_", ENVIRONMENT.get_value("TRINO_SCHEMA_PREFIX", default="")) +TRINO_S3A_OR_S3 = ENVIRONMENT.get_value("TRINO_S3A_OR_S3", default="s3a") +TRINO_SCHEMA_PREFIX_KEY = "trino_schema_prefix" # IBM Settings IBM_SERVICE_URL = ENVIRONMENT.get_value("IBM_SERVICE_URL", default="https://enterprise.cloud.ibm.com") diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index 730705d3fd..838b74a075 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -9,6 +9,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -165,7 +166,7 @@ def populate_ocp_on_aws_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/aws/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -199,7 +200,7 @@ def delete_ocp_on_aws_hive_partition_by_day( else: column_name = "aws_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{aws_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py index ad14e936a7..79c4227804 100644 --- a/koku/masu/database/azure_report_db_accessor.py +++ b/koku/masu/database/azure_report_db_accessor.py @@ -9,6 +9,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -210,7 +211,7 @@ def populate_ocp_on_azure_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/azure/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -244,7 +245,7 @@ def delete_ocp_on_azure_hive_partition_by_day( else: column_name = "azure_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{az_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 1d3da7cae4..66fe65dcc7 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -12,6 +12,7 @@ from dateutil.parser import parse from dateutil.relativedelta import relativedelta +from django.conf import settings from django.db import connection from django.db.models import F from django.db.models import Q @@ -270,7 +271,7 @@ def get_gcp_topology_trino(self, source_uuid, start_date, end_date, invoice_mont service_id, service_description, location_region - FROM hive.{self.schema}.gcp_line_items as gcp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.gcp_line_items as gcp WHERE gcp.source = '{source_uuid}' AND gcp.year = '{invoice_month_date.strftime("%Y")}' AND gcp.month = '{invoice_month_date.strftime("%m")}' @@ -396,7 +397,7 @@ def populate_ocp_on_gcp_ui_summary_tables_trino( sql = pkgutil.get_data("masu.database", f"trino_sql/gcp/openshift/{table_name}.sql") sql = sql.decode("utf-8") sql_params = { - "schema_name": self.schema, + "schema": self.schema, "start_date": start_date, "end_date": end_date, "year": year, @@ -431,7 +432,7 @@ def delete_ocp_on_gcp_hive_partition_by_day( else: column_name = "gcp_source" sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {column_name} = '{gcp_source}' AND ocp_source = '{ocp_source}' AND year = '{year}' diff --git a/koku/masu/database/ocp_report_db_accessor.py b/koku/masu/database/ocp_report_db_accessor.py index 3f851ea989..aa596b1e5f 100644 --- a/koku/masu/database/ocp_report_db_accessor.py +++ b/koku/masu/database/ocp_report_db_accessor.py @@ -11,6 +11,7 @@ import uuid from dateutil.parser import parse +from django.conf import settings from django.db.models import DecimalField from django.db.models import F from django.db.models import Value @@ -226,7 +227,7 @@ def delete_ocp_hive_partition_by_day(self, days, source, year, month): ) for day in days: sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE source = '{source}' AND year = '{year}' AND (month = replace(ltrim(replace('{month}', '0', ' ')),' ', '0') OR month = '{month}') @@ -248,7 +249,7 @@ def delete_hive_partitions_by_source(self, table, partition_column, provider_uui } LOG.info(log_json(msg="deleting Hive partitions by source", context=ctx)) sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {partition_column} = '{provider_uuid}' """ self._execute_trino_raw_sql_query( @@ -920,8 +921,8 @@ def get_nodes_trino(self, source_uuid, start_date, end_date): WHEN any_match(array_agg(DISTINCT nl.node_labels), element -> element like '%"node_role_kubernetes_io": "infra"%') THEN 'infra' ELSE 'worker' END) as node_role - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp - LEFT JOIN hive.{self.schema}.openshift_node_labels_line_items_daily as nl + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp + LEFT JOIN hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_node_labels_line_items_daily as nl ON ocp.node = nl.node WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' @@ -947,7 +948,7 @@ def get_pvcs_trino(self, source_uuid, start_date, end_date): SELECT distinct persistentvolume, persistentvolumeclaim, csi_volume_handle - FROM hive.{self.schema}.openshift_storage_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_storage_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' @@ -961,7 +962,7 @@ def get_projects_trino(self, source_uuid, start_date, end_date): """Get the nodes from an OpenShift cluster.""" sql = f""" SELECT distinct namespace - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' @@ -1119,7 +1120,7 @@ def get_max_min_timestamp_from_parquet(self, source_uuid, start_date, end_date): sql = f""" SELECT min(interval_start) as min_timestamp, max(interval_start) as max_timestamp - FROM hive.{self.schema}.openshift_pod_usage_line_items_daily as ocp + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.openshift_pod_usage_line_items_daily as ocp WHERE ocp.source = '{source_uuid}' AND ocp.year = '{start_date.strftime("%Y")}' AND ocp.month = '{start_date.strftime("%m")}' diff --git a/koku/masu/database/report_db_accessor_base.py b/koku/masu/database/report_db_accessor_base.py index 0e64def15b..ac390b1d06 100644 --- a/koku/masu/database/report_db_accessor_base.py +++ b/koku/masu/database/report_db_accessor_base.py @@ -109,6 +109,8 @@ def _execute_raw_sql_query(self, table, sql, bind_params=None, operation="UPDATE def _execute_trino_raw_sql_query(self, sql, *, sql_params=None, context=None, log_ref=None): """Execute a single trino query returning only the fetchall results""" + if sql_params is None: + sql_params = {} results, _ = self._execute_trino_raw_sql_query_with_description( sql, sql_params=sql_params, context=context, log_ref=log_ref ) @@ -119,14 +121,12 @@ def _execute_trino_raw_sql_query_with_description( self, sql, *, - sql_params=None, + sql_params: dict, context=None, log_ref="Trino query", conn_params=None, ): """Execute a single trino query and return cur.fetchall and cur.description""" - if sql_params is None: - sql_params = {} if context is None: context = {} if conn_params is None: @@ -137,6 +137,8 @@ def _execute_trino_raw_sql_query_with_description( ctx = self.extract_context_from_sql_params(context) else: ctx = {} + if sql_params and settings.TRINO_SCHEMA_PREFIX_KEY not in sql_params: + sql_params[settings.TRINO_SCHEMA_PREFIX_KEY] = settings.TRINO_SCHEMA_PREFIX sql, bind_params = self.trino_prepare_query(sql, sql_params) t1 = time.time() trino_conn = trino_db.connect(schema=self.schema, **conn_params) @@ -169,12 +171,13 @@ def _execute_trino_raw_sql_query_with_description( def _execute_trino_multipart_sql_query(self, sql, *, bind_params=None): """Execute multiple related SQL queries in Trino.""" trino_conn = trino_db.connect(schema=self.schema) + if isinstance(bind_params, dict) and settings.TRINO_SCHEMA_PREFIX_KEY not in bind_params: + bind_params[settings.TRINO_SCHEMA_PREFIX_KEY] = settings.TRINO_SCHEMA_PREFIX return trino_db.executescript(trino_conn, sql, params=bind_params, preprocessor=self.trino_prepare_query) def delete_line_item_daily_summary_entries_for_date_range_raw( self, source_uuid, start_date, end_date, filters=None, null_filters=None, table=None ): - if table is None: table = self.line_item_daily_summary_table if not isinstance(table, str): @@ -243,7 +246,7 @@ def schema_exists_trino(self): cache_key = build_trino_schema_exists_key(self.schema) if result := get_value_from_cache(cache_key): return result - check_sql = f"SHOW SCHEMAS LIKE '{self.schema}'" + check_sql = f"SHOW SCHEMAS LIKE '{settings.TRINO_SCHEMA_PREFIX}{self.schema}'" exists = bool(self._execute_trino_raw_sql_query(check_sql, log_ref="schema_exists_trino")) set_value_in_cache(cache_key, exists) return exists @@ -264,7 +267,7 @@ def delete_hive_partition_by_month(self, table, source, year, month, source_colu for i in range(retries): try: sql = f""" - DELETE FROM hive.{self.schema}.{table} + DELETE FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {source_column} = '{source}' AND year = '{year}' AND (month = replace(ltrim(replace('{month}', '0', ' ')),' ', '0') OR month = '{month}') diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index b6b1e5ef8d..e1fec9af05 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 2b93eef4f0..c8f974c4c8 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -30,7 +30,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index dd57bb1bf7..59b59066c8 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index 18a9c07911..7fbe344f2b 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 6bbb23b5e2..6c0a7bbcea 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), {{source_uuid}}::uuid, max(cost_category_id) - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index b7ed3a240c..50ad4cee71 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_code IN ('AmazonRDS','AmazonDynamoDB','AmazonElastiCache','AmazonNeptune','AmazonRedshift','AmazonDocumentDB') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 03618d88cb..4c3b495e8b 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_code IN ('AmazonVPC','AmazonCloudFront','AmazonRoute53','AmazonAPIGateway') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index 9814e3e2e4..6c7133f9b6 100644 --- a/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( id, usage_start, usage_end, @@ -36,7 +36,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( sum(markup_cost_savingsplan), max(currency_code), {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE product_family LIKE '%%Storage%%' AND unit = 'GB-Mo' AND usage_start >= {{start_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index c0b3c9e1b2..6ab5002e06 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 67c43e15bd..120c90db8c 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index 1d483127fe..977e243b7e 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_locatio sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 534b15351c..7c86e20f54 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index aa48e80457..b32e0d03f9 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( max(currency) as currency, {{source_uuid}}::uuid as source_uuid, max(cost_category_id) as cost_category_id - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}} diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 275f68116c..782421d033 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_database_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name IN ('Cosmos DB','Cache for Redis') OR service_name ILIKE '%%database%%' AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 159bf3a433..25d509d62d 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name IN ('Virtual Network','VPN','DNS','Traffic Manager','ExpressRoute','Load Balancer','Application Gateway') AND usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 8cb55a800c..796536289a 100644 --- a/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( id, usage_start, usage_end, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( sum(markup_cost) as markup_cost, max(currency) as currency, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE service_name LIKE '%%Storage%%' AND unit_of_measure = 'GB-Mo' AND usage_start >= {{start_date}}::date diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index 79c8828470..5b8f6483f0 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( id, account_id, cluster_id, @@ -31,7 +31,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index 15eeadc463..efe2fe4fa0 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -1,6 +1,6 @@ -- Clear out old entries first -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( id, cluster_id, cluster_alias, @@ -26,7 +26,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index facde9c808..41f4f0d510 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_proje source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index 545368a1b3..7295acb5c3 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index 8a33d04d3f..ab141ce832 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -1,6 +1,6 @@ -- Clear out old entries first -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( id, cluster_id, cluster_alias, @@ -30,7 +30,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index b8a0694441..578aec8e4f 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index d174f19327..1c66fd17a3 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE (service_alias LIKE '%%SQL%%' OR service_alias LIKE '%%Spanner%%' diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 2fb32836e0..a0a0743263 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE (service_alias LIKE '%%Network%%' OR service_alias LIKE '%%VPC%%' diff --git a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 343cb479fa..61ab65ae24 100644 --- a/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( +INSERT INTO {{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO {{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p -- Get data for this month or last month WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql index b74ba3c3e5..ae9a6bd503 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_compute_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_compute_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_compute_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_compute_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -15,7 +15,7 @@ DELETE -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} -- TESTING!! INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_compute_summary_pt + INTO {{schema | sqlsafe}}.reporting_ocpall_compute_summary_pt ( source_type, usage_start, @@ -52,7 +52,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}} AND usage_end <= {{end_date}} AND source_type = {{source_type}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql index a98287f10a..7ef19ad777 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_account_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_account_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_account_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_account_pt ( source_type, usage_start, usage_end, @@ -39,7 +39,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_end <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql index a5f9629e1e..54c2489059 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_region_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_region_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_region_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_region_pt ( source_type, usage_start, usage_end, @@ -43,7 +43,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql index 6a39ffbd4e..0fd98f12c6 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_by_service_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_by_service_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -14,7 +14,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_by_service_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_by_service_pt ( source_type, usage_start, usage_end, @@ -44,7 +44,7 @@ SELECT {{source_type}}, max(currency_code) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql index 16ed7f5355..afbdcb1b42 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_cost_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_cost_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_cost_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_cost_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_cost_summary_pt + INTO {{schema | sqlsafe}}.reporting_ocpall_cost_summary_pt ( source_type, usage_start, @@ -36,7 +36,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql index ebb4fffc6d..799d70763b 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_database_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_database_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_database_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_database_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_database_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_database_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_database_summary_pt ( source_type, usage_start, usage_end, @@ -45,7 +45,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql index 63f79c4843..8b1f1ad689 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_network_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_network_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_network_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_network_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_network_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_network_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_network_summary_pt ( source_type, usage_start, usage_end, @@ -45,7 +45,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql b/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql index 664a7ae46c..e09511ed34 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpall_storage_summary_pt.sql @@ -2,7 +2,7 @@ -- CLEAR DATA FROM reporting_ocpall_storage_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpall_storage_summary_pt + FROM {{schema | sqlsafe}}.reporting_ocpall_storage_summary_pt WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -13,7 +13,7 @@ DELETE -- INSERT NEW DATA INTO reporting_ocpall_storage_summary_pt -- FOR {{start_date}} - {{end_date}}; source_type {{source_type}}; source {{source_uuid}}; cluster {{cluster_id}} INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpall_storage_summary_pt ( + INTO {{schema | sqlsafe}}.reporting_ocpall_storage_summary_pt ( source_type, usage_start, usage_end, @@ -47,7 +47,7 @@ SELECT {{source_type}}, max(currency_code), max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql index f53d2f3fe6..9ed29281fb 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_aws.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (AWS DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -61,7 +61,7 @@ SELECT 'AWS'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p AS aws + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p AS aws WHERE aws.usage_start >= {{start_date}}::date AND aws.usage_start <= {{end_date}}::date AND aws.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql index 85096de2ff..3a7a5c0f58 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_azure.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (AZURE DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -60,7 +60,7 @@ SELECT 'Azure'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p AS azure + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p AS azure WHERE azure.usage_start >= {{start_date}}::date AND azure.usage_start <= {{end_date}}::date AND azure.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql index aff598f70e..b649c9f226 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_daily_summary_gcp.sql @@ -1,7 +1,7 @@ -- OCP ON ALL DAILY SUMMARY PROCESSING (GCP DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -60,7 +60,7 @@ SELECT 'GCP'::text AS source_type, max(cost_category_id) as cost_category_id, cast(1 as decimal) as shared_projects, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p AS gcp + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p AS gcp WHERE gcp.usage_start >= {{start_date}}::date AND gcp.usage_start <= {{end_date}}::date AND gcp.cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql index 4a425dbc54..074db95b0c 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_aws.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (AWS DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -63,7 +63,7 @@ SELECT 'AWS' as source_type, max(currency_code) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql index b5b828d6a1..dbd6b3b4b3 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_azure.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (AZURE DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -62,7 +62,7 @@ SELECT 'Azure' as source_type, max(currency) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql index 88d1b1defd..3d52a21c46 100644 --- a/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql +++ b/koku/masu/database/sql/openshift/all/reporting_ocpallcostlineitem_project_daily_summary_gcp.sql @@ -1,7 +1,7 @@ -- OCP ON ALL PROJECT DAILY SUMMARY PROCESSING (GCP DATA) DELETE - FROM {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND source_uuid = {{source_uuid}}::uuid @@ -10,7 +10,7 @@ DELETE INSERT - INTO {{schema_name | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( + INTO {{schema | sqlsafe}}.reporting_ocpallcostlineitem_project_daily_summary_p ( source_type, cluster_id, cluster_alias, @@ -65,7 +65,7 @@ SELECT 'GCP' as source_type, max(currency) as currency_code, max(cost_category_id) as cost_category_id, {{source_uuid}}::uuid as source_uuid - FROM {{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p + FROM {{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary_p WHERE usage_start >= {{start_date}}::date AND usage_start <= {{end_date}}::date AND cluster_id = {{cluster_id}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql index cfd5c43177..3b20e001ac 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_compute_summary_p ( id, usage_start, usage_end, @@ -42,7 +42,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_compute_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql index 4e573a04eb..2cdc055dcd 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -34,7 +34,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql index 843138622e..352feacb63 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_region_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_region_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql index ec91d8592f..ef74f74521 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -38,7 +38,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_by_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql index 04342f54b4..1938331aff 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_cost_summary_p ( id, usage_start, usage_end, @@ -32,7 +32,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_cost_summary_p ( max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid, max(cost_category_id) - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql index 96dc30fb51..03abcbd246 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_database_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_database_summary sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql index 8e62f0244a..04e1f702d1 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_network_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_network_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql index 3606a46904..a376e83d33 100644 --- a/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/aws/openshift/reporting_ocpaws_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpaws_storage_summary_p ( id, usage_start, usage_end, @@ -40,7 +40,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpaws_storage_summary_ sum(markup_cost_amortized), max(currency_code), cast({{aws_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpawscostlineitem_project_daily_summary WHERE aws_source = {{aws_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql index 53f6f48e3c..bd323cc2c2 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_compute_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_compute_summary_p ( id, usage_start, usage_end, @@ -30,7 +30,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_compute_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql index 6ba4285d9d..7d6ff2fb50 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_account_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_account_p ( id, usage_start, usage_end, @@ -22,7 +22,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql index d91043b1a5..4df03bdbec 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_location_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_location_p ( id, usage_start, usage_end, @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql index 94ee3cb686..9943268054 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_by_service_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_by_service_p ( id, usage_start, usage_end, @@ -24,7 +24,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_b sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql index be2a7d438c..2233cc81df 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_cost_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_cost_summary_p ( id, usage_start, usage_end, @@ -20,7 +20,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_cost_summary_p max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid, max(cost_category_id) as cost_category_id - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql index 1e3d02a9aa..3ebe4d4dfe 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_database_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_database_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_database_summa sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql index 3e07ce82a6..150118b550 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_network_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_network_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_network_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql index 579b939b0c..29d6c87236 100644 --- a/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/azure/openshift/reporting_ocpazure_storage_summary_p.sql @@ -1,4 +1,4 @@ -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpazure_storage_summary_p ( id, usage_start, usage_end, @@ -28,7 +28,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpazure_storage_summar sum(markup_cost) as markup_cost, max(currency) as currency, cast({{azure_source_uuid}} as uuid) as source_uuid - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpazurecostlineitem_project_daily_summary WHERE azure_source = {{azure_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND year = {{year}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql index f7a727fce9..bb99732422 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_compute_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_compute_summary_p ( id, account_id, cluster_id, @@ -31,7 +31,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_compute_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql index ee64ca9491..9fdbe05d37 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_account_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_account_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql index 6e4162e37b..2fafd275ca 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_gcp_project_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_gcp_project_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql index ff9ccf4613..ef85f0a62c 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_region_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_region_p ( id, cluster_id, cluster_alias, @@ -27,7 +27,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql index 8702c72ef6..c135be4690 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_by_service_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_by_service_p ( id, cluster_id, cluster_alias, @@ -29,7 +29,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_by_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql index d5d53e5052..780ccac4c2 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_cost_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( id, cluster_id, cluster_alias, @@ -25,7 +25,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_cost_summary_p ( sum(credit_amount) as credit_amount, invoice_month, max(cost_category_id) - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} AND invoice_month = {{invoice_month}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql index ac0c9700f8..fafbd6ead9 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_database_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_database_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_database_summary cast({{gcp_source_uuid}} as uuid) as gcp_source, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql index 491a5126c2..2cd2b457b2 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_network_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_network_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_network_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql index 3c13580034..7231fd38e5 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcp_storage_summary_p.sql @@ -1,5 +1,5 @@ -- Populate the daily aggregate line item data -INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( +INSERT INTO postgres.{{schema | sqlsafe}}.reporting_ocpgcp_storage_summary_p ( id, cluster_id, cluster_alias, @@ -33,7 +33,7 @@ INSERT INTO postgres.{{schema_name | sqlsafe}}.reporting_ocpgcp_storage_summary_ cast({{gcp_source_uuid}} as uuid) as source_uuid, sum(credit_amount) as credit_amount, invoice_month - FROM hive.{{schema_name | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary + FROM hive.{{schema | sqlsafe}}.reporting_ocpgcpcostlineitem_project_daily_summary -- Get data for this month or last month WHERE gcp_source = {{gcp_source_uuid}} AND ocp_source = {{ocp_source_uuid}} diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql index 33f0b44690..f82528848b 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary.sql @@ -211,7 +211,7 @@ SELECT gcp.uuid as gcp_uuid, max(gcp.year) as year, max(gcp.month) as month FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND (strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_id | sqlsafe}}') != 0 -- THIS IS THE SPECIFIC TO OCP ON GCP TAG MATCH OR strpos(gcp.labels, 'kubernetes-io-cluster-{{cluster_alias | sqlsafe}}') != 0) @@ -329,7 +329,7 @@ SELECT gcp.uuid as gcp_uuid, max(gcp.year) as year, max(gcp.month) as month FROM hive.{{schema | sqlsafe}}.gcp_openshift_daily as gcp -JOIN hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +JOIN hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp ON date(gcp.usage_start_time) = ocp.usage_start AND ( json_query(gcp.labels, 'strict $.openshift_project' OMIT QUOTES) = ocp.namespace diff --git a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql index 23b06cd841..15f0fec8d6 100644 --- a/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql +++ b/koku/masu/database/trino_sql/gcp/openshift/reporting_ocpgcpcostlineitem_daily_summary_resource_id.sql @@ -430,7 +430,7 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( @@ -553,7 +553,7 @@ SELECT gcp.uuid as gcp_uuid, {{ocp_source_uuid}} as ocp_source, max(gcp.year) as year, max(gcp.month) as month -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_tag_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( @@ -806,7 +806,7 @@ SELECT gcp.uuid as gcp_uuid, cast(year(max(gcp.usage_start)) as varchar) as year, cast(month(max(gcp.usage_start)) as varchar) as month, cast(day(max(gcp.usage_start)) as varchar) as day -FROM hive.{{ schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp +FROM hive.{{schema | sqlsafe}}.reporting_ocpusagelineitem_daily_summary as ocp JOIN hive.{{schema | sqlsafe}}.gcp_openshift_daily_resource_matched_temp as gcp ON gcp.usage_start = ocp.usage_start AND ( diff --git a/koku/masu/processor/_tasks/data_validation.py b/koku/masu/processor/_tasks/data_validation.py index 33ec6ac36d..5dac103619 100644 --- a/koku/masu/processor/_tasks/data_validation.py +++ b/koku/masu/processor/_tasks/data_validation.py @@ -162,7 +162,7 @@ def execute_relevant_query(self, provider_type, cluster_id=None, trino=False): source = "source" if not self.ocp_on_cloud_type else "cluster_id" sql = f""" SELECT sum({query_filters.get("metric")}) as metric, {query_filters.get("date")} as date - FROM hive.{self.schema}.{table} + FROM hive.{settings.TRINO_SCHEMA_PREFIX}{self.schema}.{table} WHERE {source} = '{provider_filter}' AND {query_filters.get("date")} >= date('{start}') AND {query_filters.get("date")} <= date('{end}') diff --git a/koku/masu/processor/aws/aws_report_parquet_processor.py b/koku/masu/processor/aws/aws_report_parquet_processor.py index e1cdce25d7..97d9a66296 100644 --- a/koku/masu/processor/aws/aws_report_parquet_processor.py +++ b/koku/masu/processor/aws/aws_report_parquet_processor.py @@ -83,7 +83,7 @@ def create_bill(self, bill_date): AND month = '{bill_date.strftime("%m")}' """ - rows = self._execute_sql(sql, self._schema_name) + rows = self._execute_trino_sql(sql, self._schema_name) payer_account_id = None if rows: payer_account_id = rows[0][0] diff --git a/koku/masu/processor/oci/oci_report_parquet_processor.py b/koku/masu/processor/oci/oci_report_parquet_processor.py index af47589e86..0bce01dc04 100644 --- a/koku/masu/processor/oci/oci_report_parquet_processor.py +++ b/koku/masu/processor/oci/oci_report_parquet_processor.py @@ -67,7 +67,7 @@ def create_bill(self, bill_date): AND year = '{bill_date.strftime("%Y")}' AND month = '{bill_date.strftime("%m")}' """ - rows = self._execute_sql(sql, self._schema_name) + rows = self._execute_trino_sql(sql, self._schema_name) payer_tenant_id = None if rows: payer_tenant_id = rows[0][0] diff --git a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py index 7eb52f224c..904af56a0f 100644 --- a/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py +++ b/koku/masu/processor/ocp/ocp_cloud_parquet_summary_updater.py @@ -273,7 +273,7 @@ def update_aws_summary_tables(self, openshift_provider_uuid, aws_provider_uuid, # OpenShift on AWS sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": aws_provider_uuid, @@ -403,7 +403,7 @@ def update_azure_summary_tables(self, openshift_provider_uuid, azure_provider_uu # OpenShift on Azure sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": azure_provider_uuid, @@ -527,7 +527,7 @@ def update_gcp_summary_tables(self, openshift_provider_uuid, gcp_provider_uuid, # OpenShift on GCP sql_params = { - "schema_name": self._schema, + "schema": self._schema, "start_date": start_date, "end_date": end_date, "source_uuid": gcp_provider_uuid, diff --git a/koku/masu/processor/report_parquet_processor_base.py b/koku/masu/processor/report_parquet_processor_base.py index 73892e87af..cb05080127 100644 --- a/koku/masu/processor/report_parquet_processor_base.py +++ b/koku/masu/processor/report_parquet_processor_base.py @@ -53,9 +53,11 @@ def postgres_summary_table(self): """Return error if unimplemented in subclass.""" raise PostgresSummaryTableError("This must be a property on the sub class.") - def _execute_sql(self, sql, schema_name): # pragma: no cover + def _execute_trino_sql(self, sql, schema_name: str): # pragma: no cover """Execute Trino SQL.""" rows = [] + if not schema_name.startswith(settings.TRINO_SCHEMA_PREFIX): + schema_name = f"{settings.TRINO_SCHEMA_PREFIX}{schema_name}" try: with trino.dbapi.connect( host=settings.TRINO_HOST, port=settings.TRINO_PORT, user="admin", catalog="hive", schema=schema_name @@ -91,8 +93,8 @@ def schema_exists(self): cache_key = build_trino_schema_exists_key(self._schema_name) if result := get_value_from_cache(cache_key): return result - schema_check_sql = f"SHOW SCHEMAS LIKE '{self._schema_name}'" - exists = bool(self._execute_sql(schema_check_sql, "default")) + schema_check_sql = f"SHOW SCHEMAS LIKE '{settings.TRINO_SCHEMA_PREFIX}{self._schema_name}'" + exists = bool(self._execute_trino_sql(schema_check_sql, "default")) set_value_in_cache(cache_key, exists) return exists @@ -103,15 +105,15 @@ def table_exists(self): if result := get_value_from_cache(cache_key): return result table_check_sql = f"SHOW TABLES LIKE '{self._table_name}'" - exists = bool(self._execute_sql(table_check_sql, self._schema_name)) + exists = bool(self._execute_trino_sql(table_check_sql, self._schema_name)) set_value_in_cache(cache_key, exists) return exists def create_schema(self): """Create Trino schema.""" LOG.info(log_json(msg="create trino/hive schema sql", schema=self._schema_name)) - schema_create_sql = f"CREATE SCHEMA IF NOT EXISTS {self._schema_name}" - self._execute_sql(schema_create_sql, "default") + schema_create_sql = f"CREATE SCHEMA IF NOT EXISTS {settings.TRINO_SCHEMA_PREFIX}{self._schema_name}" + self._execute_trino_sql(schema_create_sql, "default") return self._schema_name def _generate_column_list(self): @@ -124,7 +126,7 @@ def _generate_create_table_sql(self, partition_map=None): parquet_columns = self._generate_column_list() s3_path = f"{settings.S3_BUCKET_NAME}/{self._s3_path}" - sql = f"CREATE TABLE IF NOT EXISTS {self._schema_name}.{self._table_name} (" + sql = f"CREATE TABLE IF NOT EXISTS {settings.TRINO_SCHEMA_PREFIX}{self._schema_name}.{self._table_name} (" for idx, col in enumerate(parquet_columns): norm_col = strip_characters_from_column_name(col) if norm_col in self._column_types["numeric_columns"]: @@ -145,7 +147,7 @@ def _generate_create_table_sql(self, partition_map=None): sql += ",".join([f"{item[0]} {item[1]} " for item in list(partition_map.items())]) partition_column_str = ", ".join([f"'{key}'" for key in partition_map.keys()]) sql += ( - f") WITH(external_location = 's3a://{s3_path}', format = 'PARQUET'," + f") WITH(external_location = '{settings.TRINO_S3A_OR_S3}://{s3_path}', format = 'PARQUET'," f" partitioned_by=ARRAY[{partition_column_str}])" ) else: @@ -153,7 +155,7 @@ def _generate_create_table_sql(self, partition_map=None): sql += ",source varchar, year varchar, month varchar" sql += ( - f") WITH(external_location = 's3a://{s3_path}', format = 'PARQUET'," + f") WITH(external_location = '{settings.TRINO_S3A_OR_S3}://{s3_path}', format = 'PARQUET'," " partitioned_by=ARRAY['source', 'year', 'month'])" ) return sql @@ -162,7 +164,7 @@ def create_table(self, partition_map=None): """Create Trino SQL table.""" sql = self._generate_create_table_sql(partition_map=partition_map) LOG.info(log_json(msg="attempting to create parquet table", table=self._table_name, schema=self._schema_name)) - self._execute_sql(sql, self._schema_name) + self._execute_trino_sql(sql, self._schema_name) LOG.info(log_json(msg="trino parquet table created", table=self._table_name, schema=self._schema_name)) def get_or_create_postgres_partition(self, bill_date, **kwargs): @@ -228,4 +230,4 @@ def sync_hive_partitions(self): ) sql = f"CALL system.sync_partition_metadata('{self._schema_name}', '{self._table_name}', 'FULL')" LOG.info(sql) - self._execute_sql(sql, self._schema_name) + self._execute_trino_sql(sql, self._schema_name) diff --git a/koku/masu/test/processor/test_report_parquet_processor_base.py b/koku/masu/test/processor/test_report_parquet_processor_base.py index 2fcb592d1d..920ff9873f 100644 --- a/koku/masu/test/processor/test_report_parquet_processor_base.py +++ b/koku/masu/test/processor/test_report_parquet_processor_base.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pandas as pd +from django.conf import settings from django.test.utils import override_settings from api.common import log_json @@ -97,7 +98,7 @@ def test_generate_create_table_sql(self, mock_execute): expected_start = f"CREATE TABLE IF NOT EXISTS {self.schema}.{self.table_name}" expected_end = ( - f"WITH(external_location = 's3a://test-bucket/{self.temp_dir}', " + f"WITH(external_location = '{settings.TRINO_S3A_OR_S3}://test-bucket/{self.temp_dir}', " "format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month'])" ) self.assertTrue(generated_sql.startswith(expected_start)) @@ -123,7 +124,7 @@ def test_generate_create_table_sql_with_provider_map(self, mock_execute): expected_start = f"CREATE TABLE IF NOT EXISTS {self.schema}.{self.table_name}" expected_end = ( - f"WITH(external_location = 's3a://test-bucket/{self.temp_dir}', " + f"WITH(external_location = '{settings.TRINO_S3A_OR_S3}://test-bucket/{self.temp_dir}', " "format = 'PARQUET', partitioned_by=ARRAY['source', 'year', 'month', 'day'])" ) self.assertTrue(generated_sql.startswith(expected_start)) diff --git a/koku/sources/api/serializers.py b/koku/sources/api/serializers.py index 097a27bbcb..0cc9356ecd 100644 --- a/koku/sources/api/serializers.py +++ b/koku/sources/api/serializers.py @@ -6,6 +6,7 @@ import logging from uuid import uuid4 +from django.conf import settings from django.db import transaction from rest_framework import serializers @@ -18,7 +19,6 @@ from sources.api import get_auth_header from sources.api import get_param_from_header - LOG = logging.getLogger(__name__) ALLOWED_BILLING_SOURCE_PROVIDERS = ( @@ -104,7 +104,10 @@ def _validate_account_id(self, account_id): return get_param_from_header(self.context.get("request"), "account_number") def _validate_org_id(self, account_id): - return get_param_from_header(self.context.get("request"), "org_id") + org_id = get_param_from_header(self.context.get("request"), "org_id") + if not org_id.endswith(settings.SCHEMA_SUFFIX): + org_id = f"{org_id}{settings.SCHEMA_SUFFIX}" + return org_id def validate(self, data): data["source_id"] = self._validate_source_id(data.get("id")) diff --git a/koku/sources/kafka_message_processor.py b/koku/sources/kafka_message_processor.py index 173b9557b7..92afcb30dd 100644 --- a/koku/sources/kafka_message_processor.py +++ b/koku/sources/kafka_message_processor.py @@ -5,6 +5,7 @@ import json import logging +from django.conf import settings from rest_framework.exceptions import ValidationError from api.provider.models import Provider @@ -17,7 +18,6 @@ from sources.sources_http_client import SourcesHTTPClient from sources.sources_http_client import SourcesHTTPClientError - LOG = logging.getLogger(__name__) KAFKA_APPLICATION_CREATE = "Application.create" @@ -101,6 +101,8 @@ def __init__(self, msg, event_type, cost_mgmt_id): msg = f"[KafkaMessageProcessor] missing `{KAFKA_HDR_RH_IDENTITY}` or org_id: {msg.headers()}" LOG.warning(msg) raise SourcesMessageError(msg) + if isinstance(self.org_id, str) and not self.org_id.endswith(settings.SCHEMA_SUFFIX): + self.org_id = f"{self.org_id}{settings.SCHEMA_SUFFIX}" self.source_id = None self.application_type_id = None diff --git a/koku/subs/subs_data_extractor.py b/koku/subs/subs_data_extractor.py index 2c93b48970..7fa447bde0 100644 --- a/koku/subs/subs_data_extractor.py +++ b/koku/subs/subs_data_extractor.py @@ -22,7 +22,6 @@ from reporting.models import SubsIDMap from reporting.models import SubsLastProcessed - LOG = logging.getLogger(__name__) @@ -40,7 +39,10 @@ def __init__(self, tracing_id, context): self.creation_processing_time = self.creation_processing_time - timedelta(days=1) self.tracing_id = tracing_id self.s3_resource = get_s3_resource( - settings.S3_SUBS_ACCESS_KEY, settings.S3_SUBS_SECRET, settings.S3_SUBS_REGION + access_key=settings.S3_SUBS_ACCESS_KEY, + secret_key=settings.S3_SUBS_SECRET, + region=settings.S3_SUBS_REGION, + endpoint_url=settings.S3_SUBS_ENDPOINT, ) self.context = context diff --git a/koku/subs/subs_data_messenger.py b/koku/subs/subs_data_messenger.py index 32c78ca042..58f7d1cec6 100644 --- a/koku/subs/subs_data_messenger.py +++ b/koku/subs/subs_data_messenger.py @@ -25,7 +25,6 @@ from masu.prometheus_stats import RHEL_ELS_VCPU_HOURS from masu.util.aws.common import get_s3_resource - LOG = logging.getLogger(__name__) HPC_ROLE = "Red Hat Enterprise Linux Compute Node" @@ -51,7 +50,10 @@ def __init__(self, context, schema_name, tracing_id): self.tracing_id = tracing_id self.schema_name = schema_name self.s3_resource = get_s3_resource( - settings.S3_SUBS_ACCESS_KEY, settings.S3_SUBS_SECRET, settings.S3_SUBS_REGION + access_key=settings.S3_SUBS_ACCESS_KEY, + secret_key=settings.S3_SUBS_SECRET, + region=settings.S3_SUBS_REGION, + endpoint_url=settings.S3_SUBS_ENDPOINT, ) subs_cust = Customer.objects.get(schema_name=schema_name) self.account_id = subs_cust.account_id