diff --git a/.env.example b/.env.example index ee2a3d3e99..0827fb1a00 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,8 @@ DATABASE_NAME=postgres PGADMIN_EMAIL=postgres@local.dev PGADMIN_PASSWORD=postgres PGADMIN_PORT=8432 -POSTGRES_SQL_SERVICE_HOST=localhost -POSTGRES_SQL_SERVICE_PORT=15432 +POSTGRES_SQL_SERVICE_HOST=db +POSTGRES_SQL_SERVICE_PORT=5432 DATABASE_USER=postgres DATABASE_ADMIN=postgres DATABASE_PASSWORD=postgres @@ -32,18 +32,23 @@ TAG_ENABLED_LIMIT=200 # Set the max amount of tags per account DELAYED_TASK_TIME=30 # Set the seconds before a delayed summary task should expire DELAYED_TASK_POLLING_MINUTES=5 # Set the time before the delayed task kick off. -ENABLE_S3_ARCHIVING=True -S3_BUCKET_NAME=koku-bucket -S3_BUCKET_PATH=data -S3_ENDPOINT=http://koku-minio:9000 -S3_ACCESS_KEY=kokuminioaccess -S3_SECRET=kokuminiosecret SKIP_MINIO_DATA_DELETION=False # AWS AWS_SHARED_CREDENTIALS_FILE=/etc/credentials/aws AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN +# Glue +SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique + +AWS_CATALOG_ID=589173575009 +S3_ENDPOINT=https://s3.us-east-1.amazonaws.com + +S3_BUCKET_NAME=hccm-local-s3 +S3_ACCESS_KEY=CHANGEME +S3_SECRET=CHANGEME +S3_REGION=us-east-1 + # GCP GOOGLE_APPLICATION_CREDENTIALS=/etc/credentials/gcp GCP_DATASET='dataset_example' diff --git a/dev/containers/hive-metastore/metastore-site.xml b/dev/containers/hive-metastore/metastore-site.xml new file mode 100644 index 0000000000..2712961920 --- /dev/null +++ b/dev/containers/hive-metastore/metastore-site.xml @@ -0,0 +1,116 @@ + + + + metastore.thrift.port + 8000 + Hive metastore listener port + + + metastore.thrift.uris + thrift://${env.HOSTNAME}:8000 + Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore. + + + metastore.metrics.enabled + true + Enable metrics on the metastore. + + + metastore.metrics.reporters + jmx + A comma separated list of metrics reporters to start + + + datanucleus.autoStartMechanismMode + ignored + Autostart mechanism for datanucleus. Currently ignored is the only option supported. + + + datanucleus.schema.autoCreateAll + false + Auto creates necessary schema on a startup if one doesn't exist. Set this to false, after creating it once.To enable auto create also set hive.metastore.schema.verification=false. Auto creation is not recommended for production use cases, run schematool command instead. + + + metastore.schema.verification + true + + Enforce metastore schema version consistency. + True: Verify that version information stored in is compatible with one from Hive jars. Also disable automatic + schema migration attempt. Users are required to manually migrate schema after Hive upgrade which ensures + proper metastore schema migration. (Default) + False: Warn if the version information stored in metastore doesn't match with one from in Hive jars. + + + + hive.default.fileformat + Parquet + + + fs.s3a.endpoint + AWS S3 endpoint to connect to. + ${env.S3_ENDPOINT} + + + fs.s3a.access.key + AWS access key ID. + ${env.S3_ACCESS_KEY} + + + fs.s3a.secret.key + AWS secret key. + ${env.S3_SECRET} + + + fs.s3a.path.style.access + true + Enable S3 path style access. + + + metastore.warehouse.dir + s3a://${env.S3_BUCKET_NAME}/${env.S3_BUCKET_PATH}/ + + + hive.metastore.db.type + postgres + + Expects one of [derby, oracle, mysql, mssql, postgres]. + Type of database used by the metastore. Information schema & JDBCStorageHandler depend on it. + + + + javax.jdo.option.ConnectionUserName + ${env.DATABASE_USER} + Username to use against metastore database + + + javax.jdo.option.ConnectionPassword + ${env.DATABASE_PASSWORD} + password to use against metastore database + + + javax.jdo.option.ConnectionURL + jdbc:postgresql://${env.POSTGRES_SQL_SERVICE_HOST}:${env.POSTGRES_SQL_SERVICE_PORT}/${env.DATABASE_NAME}?sslmode=prefer + + JDBC connect string for a JDBC metastore. + To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL. + For example, jdbc:postgresql://myhost/db?ssl=true for postgres database. + + + + javax.jdo.option.ConnectionDriverName + org.postgresql.Driver + Driver class name for a JDBC metastore + + + hive.cluster.delegation.token.store.class + org.apache.hadoop.hive.thrift.DBTokenStore + + + metastore.task.threads.always + org.apache.hadoop.hive.metastore.events.EventCleanerTask + + + metastore.expression.proxy + org.apache.hadoop.hive.metastore.DefaultPartitionExpressionProxy + + diff --git a/dev/containers/trino/etc/catalog/glue.properties b/dev/containers/trino/etc/catalog/glue.properties new file mode 100644 index 0000000000..2965c0c776 --- /dev/null +++ b/dev/containers/trino/etc/catalog/glue.properties @@ -0,0 +1,21 @@ +connector.name=hive +hive.metastore=glue +hive.storage-format=Parquet +hive.compression-codec=SNAPPY +hive.collect-column-statistics-on-write=true +hive.recursive-directories=true +hive.non-managed-table-writes-enabled=true + +fs.hadoop.enabled=false +fs.native-s3.enabled=true +s3.region=${ENV:S3_REGION} +s3.endpoint=${ENV:S3_ENDPOINT} +s3.aws-access-key=${ENV:S3_ACCESS_KEY} +s3.aws-secret-key=${ENV:S3_SECRET} +s3.path-style-access=true + +hive.metastore.glue.default-warehouse-dir=s3://${ENV:S3_BUCKET_NAME}/data +hive.metastore.glue.region=${ENV:S3_REGION} +hive.metastore.glue.aws-access-key=${ENV:S3_ACCESS_KEY} +hive.metastore.glue.aws-secret-key=${ENV:S3_SECRET} +hive.metastore.glue.catalogid=${ENV:AWS_CATALOG_ID} diff --git a/dev/containers/trino/etc/catalog/hive.properties b/dev/containers/trino/etc/catalog/hive.properties index 856b3b7aee..b1dd26b916 100644 --- a/dev/containers/trino/etc/catalog/hive.properties +++ b/dev/containers/trino/etc/catalog/hive.properties @@ -9,8 +9,8 @@ hive.parquet.use-column-names=true fs.native-s3.enabled=true -s3.aws-access-key=${ENV:AWS_ACCESS_KEY_ID} -s3.aws-secret-key=${ENV:AWS_SECRET_ACCESS_KEY} +s3.aws-access-key=${ENV:S3_ACCESS_KEY} +s3.aws-secret-key=${ENV:S3_SECRET} s3.endpoint=${ENV:S3_ENDPOINT} s3.path-style-access=true s3.region=us-east-1 diff --git a/dev/containers/trino/etc/catalog/postgres.properties b/dev/containers/trino/etc/catalog/postgres.properties index ffa2dff6a1..0a96fef7fe 100644 --- a/dev/containers/trino/etc/catalog/postgres.properties +++ b/dev/containers/trino/etc/catalog/postgres.properties @@ -1,5 +1,5 @@ connector.name=postgresql -connection-url=jdbc:postgresql://db:5432/${ENV:DATABASE_NAME} +connection-url=jdbc:postgresql://${ENV:POSTGRES_SQL_SERVICE_HOST}:${ENV:POSTGRES_SQL_SERVICE_PORT}/${ENV:DATABASE_NAME} connection-user=${ENV:DATABASE_USER} connection-password=${ENV:DATABASE_PASSWORD} postgresql.array-mapping=AS_ARRAY diff --git a/dev/scripts/load_test_customer_data.sh b/dev/scripts/load_test_customer_data.sh index b14bce2e57..22ef9de337 100755 --- a/dev/scripts/load_test_customer_data.sh +++ b/dev/scripts/load_test_customer_data.sh @@ -241,7 +241,7 @@ enable_ocp_tags() { log-info "Enabling OCP tags..." RESPONSE=$(curl -s -w "%{http_code}\n" --header "Content-Type: application/json" \ --request POST \ - --data '{"schema": "org1234567","action": "create","tag_keys": ["environment", "app", "version", "storageclass", "application", "instance-type"], "provider_type": "ocp"}' \ + --data '{"schema": "org1234567'"${SCHEMA_SUFFIX}"'","action": "create","tag_keys": ["environment", "app", "version", "storageclass", "application", "instance-type"], "provider_type": "ocp"}' \ "${MASU_URL_PREFIX}"/v1/enabled_tags/) STATUS_CODE=${RESPONSE: -3} DATA=${RESPONSE:: -3} @@ -278,8 +278,8 @@ build_aws_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report aws --static-report-file "$YAML_PATH/ocp_on_aws/rendered_aws_static_data.yml" --aws-s3-report-name None --aws-s3-bucket-name "$NISE_DATA_PATH/local_providers/aws_local" log-info "Cleanup ${_source_name} rendered YAML files..." @@ -314,8 +314,8 @@ build_azure_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report azure --static-report-file "$YAML_PATH/ocp_on_azure/rendered_azure_static_data.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report nise_report azure --static-report-file "$YAML_PATH/rendered_azure_v2.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report-v2 --resource-group @@ -350,8 +350,8 @@ build_gcp_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report gcp --static-report-file "$YAML_PATH/gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local" nise_report gcp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local_0" -r @@ -379,8 +379,8 @@ build_onprem_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" log-info "Cleanup ${_source_name} rendered YAML files..." cleanup_rendered_files "${_rendered_yaml_files[@]}" diff --git a/docker-compose.yml b/docker-compose.yml index 5c420c9af0..567625d4d0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,14 +25,17 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_SOURCES_CLIENT_HOST=${KOKU_SOURCES_CLIENT_HOST-sources-client} - KOKU_SOURCES_CLIENT_PORT=${KOKU_SOURCES_CLIENT_PORT-9000} - DJANGO_READ_DOT_ENV_FILE=True - DEVELOPMENT=${DEVELOPMENT-True} + - SCHEMA_SUFFIX=${SCHEMA_SUFFIX-} + - TRINO_SCHEMA_PREFIX=${TRINO_SCHEMA_PREFIX-} + - TRINO_S3A_OR_S3=${TRINO_S3A_OR_S3-s3a} - RBAC_SERVICE_HOST=${RBAC_SERVICE_HOST-rbac-server} - RBAC_SERVICE_PORT=${RBAC_SERVICE_PORT-9000} - RBAC_SERVICE_PATH=${RBAC_SERVICE_PATH-/r/insights/platform/rbac/v1/access/} @@ -89,8 +92,8 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - S3_BUCKET_NAME_OCP_INGRESS=${S3_BUCKET_NAME_OCP_INGRESS-ocp-ingress} @@ -100,6 +103,9 @@ services: - S3_SECRET - DJANGO_READ_DOT_ENV_FILE=True - DEVELOPMENT=${DEVELOPMENT-True} + - SCHEMA_SUFFIX=${SCHEMA_SUFFIX-} + - TRINO_SCHEMA_PREFIX=${TRINO_SCHEMA_PREFIX-} + - TRINO_S3A_OR_S3=${TRINO_S3A_OR_S3-s3a} - RBAC_SERVICE_HOST=${RBAC_SERVICE_HOST-rbac-server} - RBAC_SERVICE_PORT=${RBAC_SERVICE_PORT-9000} - RBAC_SERVICE_PATH=${RBAC_SERVICE_PATH-/r/insights/platform/rbac/v1/access/} @@ -159,14 +165,17 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - REDIS_HOST=${REDIS_HOST-redis} - REDIS_PORT=${REDIS_PORT-6379} - DEBUG_ATTACH=${DEBUG_ATTACH-False} - DEVELOPMENT=${DEVELOPMENT-True} + - SCHEMA_SUFFIX=${SCHEMA_SUFFIX-} + - TRINO_SCHEMA_PREFIX=${TRINO_SCHEMA_PREFIX-} + - TRINO_S3A_OR_S3=${TRINO_S3A_OR_S3-s3a} - KOKU_LOG_LEVEL=${KOKU_LOG_LEVEL-DEBUG} - CELERY_LOG_LEVEL=${CELERY_LOG_LEVEL-INFO} - DJANGO_LOG_LEVEL=${DJANGO_LOG_LEVEL-INFO} @@ -252,8 +261,8 @@ services: environment: - DJANGO_READ_DOT_ENV_FILE=True - DATABASE_SERVICE_NAME=POSTGRES_SQL - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - DATABASE_HOST=db @@ -303,8 +312,8 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_API_HOST=${KOKU_API_HOST-koku-server} @@ -358,8 +367,8 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - KOKU_API_HOST=${KOKU_API_HOST-koku-server} @@ -416,8 +425,8 @@ services: - DATABASE_SERVICE_NAME=POSTGRES_SQL - DATABASE_ENGINE=postgresql - DATABASE_NAME=${DATABASE_NAME-postgres} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - REDIS_HOST=${REDIS_HOST-redis} @@ -572,8 +581,8 @@ services: - CHECK_VERSION=false - DATABASE_HOST=db - DATABASE_NAME=unleash - - DATABASE_USERNAME=${DATABASE_USER-postgres} - - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} + - DATABASE_USERNAME=${UNLEASH_DATABASE_USER-postgres} + - DATABASE_PASSWORD=${UNLEASH_DATABASE_PASSWORD-postgres} - DATABASE_SSL=false - IMPORT_DROP_BEFORE_IMPORT=true - IMPORT_FILE=/.unleash/flags.json @@ -629,16 +638,18 @@ services: - 9083:8000 environment: - HIVE_LOGLEVEL=INFO - - S3_HOST=koku-minio - - S3_PORT=9000 + - S3_ENDPOINT=${S3_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - - POSTGRES_SQL_SERVICE_HOST=db - - POSTGRES_SQL_SERVICE_PORT=5432 + - S3_BUCKET_PATH=data + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_NAME=${DATABASE_NAME-hive} - DATABASE_USER=${DATABASE_USER-hive} - DATABASE_PASSWORD=${DATABASE_PASSWORD-hive} + volumes: + - ./dev/containers/hive-metastore/metastore-site.xml:/opt/hive-metastore-bin/conf/metastore-site.xml depends_on: - db - minio @@ -646,7 +657,7 @@ services: trino: container_name: trino - image: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/ubi-trino:461-002 + image: quay.io/redhat-services-prod/cost-mgmt-dev-tenant/ubi-trino:461-007 user: root ports: - 8080:8080 @@ -655,12 +666,14 @@ services: - MY_NODE_ID=${MY_NODE_ID-localhost} - LOCAL=TRUE - TRINO_LOG_LEVEL=${LOG_LEVEL-INFO} - - AWS_ACCESS_KEY_ID=${S3_ACCESS_KEY-kokuminioaccess} - - AWS_SECRET_ACCESS_KEY=${S3_SECRET-kokuminiosecret} - S3_ENDPOINT=${S3_ENDPOINT-http://koku-minio:9000} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} + - S3_REGION=${S3_REGION-us-east-1} + - AWS_CATALOG_ID=${AWS_CATALOG_ID-} + - POSTGRES_SQL_SERVICE_HOST=${POSTGRES_SQL_SERVICE_HOST-db} + - POSTGRES_SQL_SERVICE_PORT=${POSTGRES_SQL_SERVICE_PORT-5432} - DATABASE_NAME=${DATABASE_NAME-postgres} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} @@ -668,14 +681,14 @@ services: - ./dev/containers/trino/etc/config.properties:/etc/trino/config.properties - ./dev/containers/trino/etc/jvm.config:/etc/trino/jvm.config - ./dev/containers/trino/etc/log.properties:/etc/trino/log.properties - - ./dev/containers/trino/etc/catalog/hive.properties:/etc/trino/catalog/hive.properties + - ./dev/containers/trino/etc/catalog/glue.properties:/etc/trino/catalog/hive.properties - ./dev/containers/trino/etc/catalog/postgres.properties:/etc/trino/catalog/postgres.properties - ./dev/containers/trino/data:/data/trino/data - ./dev/containers/trino/logs:/data/trino/logs - links: - - hive-metastore - depends_on: - - hive-metastore + # links: + # - hive-metastore + # depends_on: + # - hive-metastore ulimits: nofile: soft: 4096 diff --git a/koku/koku/settings.py b/koku/koku/settings.py index 3be93221fa..446bb4edf2 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -143,9 +143,7 @@ if DEVELOPMENT: # if SCHEMA_SUFFIX == "": # SCHEMA_SUFFIX = f"_{ENVIRONMENT.get_value('USER', default='')}" - # if not SCHEMA_SUFFIX.startswith("_"): - # SCHEMA_SUFFIX = f"_{SCHEMA_SUFFIX}" - # print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") + print(f"ORG ID SUFFIX: '{SCHEMA_SUFFIX}'") DEFAULT_IDENTITY = { "identity": { "account_number": "10001", diff --git a/koku/masu/processor/report_parquet_processor_base.py b/koku/masu/processor/report_parquet_processor_base.py index ec77863a0b..7f1b315ba6 100644 --- a/koku/masu/processor/report_parquet_processor_base.py +++ b/koku/masu/processor/report_parquet_processor_base.py @@ -228,6 +228,11 @@ def sync_hive_partitions(self): table=self._table_name, ) ) - sql = f"CALL system.sync_partition_metadata('{self._schema_name}', '{self._table_name}', 'FULL')" + sql = ( + "CALL system.sync_partition_metadata('" + f"{settings.TRINO_SCHEMA_PREFIX}{self._schema_name}', " + f"'{self._table_name}', " + "'FULL')" + ) LOG.info(sql) self._execute_trino_sql(sql, self._schema_name)