From 0d123cdaf7c2ec0f318657ec5c75537174211b34 Mon Sep 17 00:00:00 2001 From: myersCody Date: Wed, 22 Jan 2025 13:29:41 -0500 Subject: [PATCH 1/3] Fix load test customer --- dev/scripts/load_test_customer_data.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dev/scripts/load_test_customer_data.sh b/dev/scripts/load_test_customer_data.sh index 22ef9de337..46f921b23e 100755 --- a/dev/scripts/load_test_customer_data.sh +++ b/dev/scripts/load_test_customer_data.sh @@ -278,8 +278,8 @@ build_aws_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" nise_report aws --static-report-file "$YAML_PATH/ocp_on_aws/rendered_aws_static_data.yml" --aws-s3-report-name None --aws-s3-bucket-name "$NISE_DATA_PATH/local_providers/aws_local" log-info "Cleanup ${_source_name} rendered YAML files..." @@ -314,8 +314,8 @@ build_azure_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" nise_report azure --static-report-file "$YAML_PATH/ocp_on_azure/rendered_azure_static_data.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report nise_report azure --static-report-file "$YAML_PATH/rendered_azure_v2.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report-v2 --resource-group @@ -350,8 +350,8 @@ build_gcp_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" nise_report gcp --static-report-file "$YAML_PATH/gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local" nise_report gcp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local_0" -r @@ -379,8 +379,8 @@ build_onprem_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" log-info "Cleanup ${_source_name} rendered YAML files..." cleanup_rendered_files "${_rendered_yaml_files[@]}" From 054e31a41baf3650c5e36cda7c0237ad0acc95d0 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 22 Jan 2025 13:44:04 -0500 Subject: [PATCH 2/3] add MINIO_ENDPOINT --- dev/scripts/load_test_customer_data.sh | 16 ++++++++-------- docker-compose.yml | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/dev/scripts/load_test_customer_data.sh b/dev/scripts/load_test_customer_data.sh index 46f921b23e..22ef9de337 100755 --- a/dev/scripts/load_test_customer_data.sh +++ b/dev/scripts/load_test_customer_data.sh @@ -278,8 +278,8 @@ build_aws_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_aws/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-1 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report aws --static-report-file "$YAML_PATH/ocp_on_aws/rendered_aws_static_data.yml" --aws-s3-report-name None --aws-s3-bucket-name "$NISE_DATA_PATH/local_providers/aws_local" log-info "Cleanup ${_source_name} rendered YAML files..." @@ -314,8 +314,8 @@ build_azure_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_azure/rendered_ocp_static_data.yml" --ocp-cluster-id my-ocp-cluster-2 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report azure --static-report-file "$YAML_PATH/ocp_on_azure/rendered_azure_static_data.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report nise_report azure --static-report-file "$YAML_PATH/rendered_azure_v2.yml" --azure-container-name "$NISE_DATA_PATH/local_providers/azure_local" --azure-report-name azure-report-v2 --resource-group @@ -350,8 +350,8 @@ build_gcp_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_ocp_static_data.yml" --ocp-cluster-id test-ocp-gcp-cluster --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" nise_report gcp --static-report-file "$YAML_PATH/gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local" nise_report gcp --static-report-file "$YAML_PATH/ocp_on_gcp/rendered_gcp_static_data.yml" --gcp-bucket-name "$NISE_DATA_PATH/local_providers/gcp_local_0" -r @@ -379,8 +379,8 @@ build_onprem_data() { render_yaml_files "${_yaml_files[@]}" log-info "Building OpenShift on ${_source_name} report data..." - nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --daily-reports --payload-name "$_ocp_payload" - # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload http://localhost:9000 --payload-name "$_ocp_payload" + nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --daily-reports --payload-name "$_ocp_payload" + # nise_report ocp --static-report-file "$YAML_PATH/ocp/rendered_ocp_on_premise.yml" --ocp-cluster-id my-ocp-cluster-3 --minio-upload "${S3_ENDPOINT}" --payload-name "$_ocp_payload" log-info "Cleanup ${_source_name} rendered YAML files..." cleanup_rendered_files "${_rendered_yaml_files[@]}" diff --git a/docker-compose.yml b/docker-compose.yml index fd8c50bc00..871c09b4e4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,7 +98,7 @@ services: - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} - S3_BUCKET_NAME_OCP_INGRESS=${S3_BUCKET_NAME_OCP_INGRESS-ocp-ingress} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - - S3_ENDPOINT + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY - S3_SECRET - DJANGO_READ_DOT_ENV_FILE=True @@ -188,7 +188,7 @@ services: - ENABLE_SUBS_PROVIDER_TYPES=${ENABLE_SUBS_PROVIDER_TYPES} - ENABLE_ROS_DEBUG=${ENABLE_ROS_DEBUG-False} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - - S3_ENDPOINT + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY - S3_SECRET - DATA_DIR=${DATA_DIR-/testing/data} @@ -282,7 +282,7 @@ services: - INITIAL_INGEST_NUM_MONTHS=1 - PYTHONPATH=/koku/koku - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - - S3_ENDPOINT + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY - S3_SECRET - TRINO_HOST=${TRINO_HOST-trino} @@ -322,7 +322,7 @@ services: - ENABLE_SUBS_DEBUG=${ENABLE_SUBS_DEBUG-False} - ENABLE_SUBS_PROVIDER_TYPES=${ENABLE_SUBS_PROVIDER_TYPES} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - - S3_ENDPOINT + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY - S3_SECRET - AWS_SHARED_CREDENTIALS_FILE=${AWS_SHARED_CREDENTIALS_FILE-} @@ -619,7 +619,7 @@ services: entrypoint: > /bin/sh -c " /usr/bin/mc config host rm local; - /usr/bin/mc config host add --quiet --api s3v4 local $S3_ENDPOINT $S3_ACCESS_KEY $S3_SECRET; + /usr/bin/mc config host add --quiet --api s3v4 local $MINIO_ENDPOINT $S3_ACCESS_KEY $S3_SECRET; /usr/bin/mc rb --force local/koku-bucket/; /usr/bin/mc mb --quiet local/koku-bucket/; /usr/bin/mc anonymous set public local/koku-bucket; @@ -638,7 +638,7 @@ services: - 9083:8000 environment: - HIVE_LOGLEVEL=INFO - - S3_ENDPOINT=${S3_ENDPOINT-http://koku-minio:9000} + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} @@ -666,7 +666,7 @@ services: - MY_NODE_ID=${MY_NODE_ID-localhost} - LOCAL=TRUE - TRINO_LOG_LEVEL=${LOG_LEVEL-INFO} - - S3_ENDPOINT=${S3_ENDPOINT-http://koku-minio:9000} + - S3_ENDPOINT=${MINIO_ENDPOINT-http://koku-minio:9000} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_ACCESS_KEY=${S3_ACCESS_KEY-kokuminioaccess} - S3_SECRET=${S3_SECRET-kokuminiosecret} From f9c716720398b1f3a8c9f361e94555dc1fbe540e Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 22 Jan 2025 13:52:47 -0500 Subject: [PATCH 3/3] update env example --- .env.example | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 3a121a1a75..c41c27c942 100644 --- a/.env.example +++ b/.env.example @@ -42,11 +42,12 @@ AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique AWS_CATALOG_ID=589173575009 -S3_ENDPOINT=https://s3.us-east-1.amazonaws.com +MINIO_ENDPOINT=http://koku-minio:9000 +S3_ENDPOINT=http://localhost:9000 +S3_ACCESS_KEY=kokuminioaccess +S3_SECRET=kokuminiosecret -S3_BUCKET_NAME=hccm-local-s3 -S3_ACCESS_KEY=CHANGEME -S3_SECRET=CHANGEME +S3_BUCKET_NAME=koku-bucket S3_REGION=us-east-1 # GCP