diff --git a/.env.example b/.env.example index c41c27c942..174885eed9 100644 --- a/.env.example +++ b/.env.example @@ -40,6 +40,8 @@ AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN # Glue SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique +TRINO_SCHEMA_PREFIX="" +TRINO_S3A_OR_S3=s3 AWS_CATALOG_ID=589173575009 MINIO_ENDPOINT=http://koku-minio:9000 @@ -48,6 +50,7 @@ S3_ACCESS_KEY=kokuminioaccess S3_SECRET=kokuminiosecret S3_BUCKET_NAME=koku-bucket +S3_BUCKET_NAME_OCP_INGRESS=koku-bucket S3_REGION=us-east-1 # GCP diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7dcaf41c73..56f225d479 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -70,6 +70,8 @@ jobs: echo Dockerfile >> docker-files.txt; echo pr_check.sh >> docker-files.txt; echo deploy/clowdapp.yaml >> docker-files.txt; + echo Jenkinsfile >> docker-files.txt; + echo ci/functions.sh >> docker-files.txt; - name: Show Dockerfiles run: cat docker-files.txt @@ -334,11 +336,11 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: - file: ./coverage.xml + files: ./coverage.xml flags: unittests name: Python-${{ matrix.python-version}} fail_ci_if_error: true - plugin: pycoverage # Only run one plugin even though we don't want any to run. + plugins: noop - name: Set Codecov job status on skipped tests if: needs.changed-files.outputs.run_tests != 'true' diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index f453a28c3c..ba650e2417 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -5614,6 +5614,8 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: AWS_SHARED_CREDENTIALS_FILE + value: ${AWS_SHARED_CREDENTIALS_FILE} - name: SCHEMA_SUFFIX value: ${SCHEMA_SUFFIX} - name: TRINO_SCHEMA_PREFIX @@ -5644,6 +5646,17 @@ objects: requests: cpu: ${CPU_REQUEST_KOKU_MIGRATIONS} memory: ${MEMORY_REQUEST_KOKU_MIGRATIONS} + volumeMounts: + - mountPath: /etc/aws + name: aws-credentials + readOnly: true + volumes: + - name: aws-credentials + secret: + items: + - key: aws-credentials + path: aws-credentials + secretName: koku-aws - name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} podSpec: args: @@ -5653,6 +5666,8 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: AWS_SHARED_CREDENTIALS_FILE + value: ${AWS_SHARED_CREDENTIALS_FILE} - name: SCHEMA_SUFFIX value: ${SCHEMA_SUFFIX} - name: TRINO_SCHEMA_PREFIX @@ -5677,6 +5692,17 @@ objects: requests: cpu: ${CPU_REQUEST_KOKU_MGMT} memory: ${MEMORY_REQUEST_KOKU_MGMT} + volumeMounts: + - mountPath: /etc/aws + name: aws-credentials + readOnly: true + volumes: + - name: aws-credentials + secret: + items: + - key: aws-credentials + path: aws-credentials + secretName: koku-aws kafkaTopics: - topicName: platform.sources.event-stream - topicName: platform.upload.announce diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml index 19aae2e90b..0453a974ae 100644 --- a/deploy/kustomize/base/base.yaml +++ b/deploy/kustomize/base/base.yaml @@ -73,6 +73,8 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: AWS_SHARED_CREDENTIALS_FILE + value: ${AWS_SHARED_CREDENTIALS_FILE} - name: SCHEMA_SUFFIX value: ${SCHEMA_SUFFIX} - name: TRINO_SCHEMA_PREFIX @@ -95,6 +97,17 @@ objects: value: ${TENANT_MULTIPROCESSING_MAX_PROCESSES} - name: TENANT_MULTIPROCESSING_CHUNKS value: ${TENANT_MULTIPROCESSING_CHUNKS} + volumeMounts: + - mountPath: /etc/aws + name: aws-credentials + readOnly: true + volumes: + - name: aws-credentials + secret: + items: + - key: aws-credentials + path: aws-credentials + secretName: koku-aws # ==================================================== # koku Management Command Job # ==================================================== @@ -115,6 +128,8 @@ objects: env: - name: CLOWDER_ENABLED value: ${CLOWDER_ENABLED} + - name: AWS_SHARED_CREDENTIALS_FILE + value: ${AWS_SHARED_CREDENTIALS_FILE} - name: SCHEMA_SUFFIX value: ${SCHEMA_SUFFIX} - name: TRINO_SCHEMA_PREFIX @@ -131,6 +146,17 @@ objects: value: ${TRINO_HOST} - name: TRINO_PORT value: ${TRINO_PORT} + volumeMounts: + - mountPath: /etc/aws + name: aws-credentials + readOnly: true + volumes: + - name: aws-credentials + secret: + items: + - key: aws-credentials + path: aws-credentials + secretName: koku-aws # The bulk of your App. This is where your running apps will live deployments: - diff --git a/dev/scripts/load_test_customer_data.sh b/dev/scripts/load_test_customer_data.sh index 4602a2ed38..0f853beb25 100755 --- a/dev/scripts/load_test_customer_data.sh +++ b/dev/scripts/load_test_customer_data.sh @@ -75,7 +75,7 @@ export OS export S3_ACCESS_KEY="${S3_ACCESS_KEY}" export S3_SECRET_KEY="${S3_SECRET}" -export S3_BUCKET_NAME="ocp-ingress" +export S3_BUCKET_NAME="${S3_BUCKET_NAME_OCP_INGRESS}" log-info "Calculating dates..." diff --git a/koku/koku/configurator.py b/koku/koku/configurator.py index 12208969fb..51406fb260 100644 --- a/koku/koku/configurator.py +++ b/koku/koku/configurator.py @@ -7,12 +7,17 @@ """ import pathlib -from .env import ENVIRONMENT +import boto3 +from .env import ENVIRONMENT CLOWDER_ENABLED = ENVIRONMENT.bool("CLOWDER_ENABLED", default=False) if CLOWDER_ENABLED: - from app_common_python import ObjectBuckets, LoadedConfig, KafkaTopics, KafkaServers, DependencyEndpoints + from app_common_python import DependencyEndpoints + from app_common_python import KafkaServers + from app_common_python import KafkaTopics + from app_common_python import LoadedConfig + from app_common_python import ObjectBuckets class Configurator: @@ -78,6 +83,9 @@ def get_cloudwatch_log_group(): """Obtain cloudwatch log group.""" pass + def get_object_store_endpoint_default(self): + pass + @staticmethod def get_object_store_endpoint(): """Obtain object store endpoint.""" @@ -98,11 +106,19 @@ def get_object_store_tls(): """Obtain object store secret key.""" pass + def get_object_store_access_key_default(self, requested_name: str = ""): + """Obtain object store access key.""" + pass + @staticmethod def get_object_store_access_key(requested_name: str = ""): """Obtain object store access key.""" pass + def get_object_store_secret_key_default(self, requested_name: str = ""): + """Obtain object store secret key.""" + pass + @staticmethod def get_object_store_secret_key(requested_name: str = ""): """Obtain object store secret key.""" @@ -250,6 +266,9 @@ def get_cloudwatch_log_group(): """Obtain cloudwatch log group.""" return ENVIRONMENT.get_value("CW_LOG_GROUP", default="platform-dev") + def get_object_store_endpoint_default(self): + return self.get_object_store_endpoint() + @staticmethod def get_object_store_endpoint(): """Obtain object store endpoint.""" @@ -276,11 +295,19 @@ def get_object_store_tls(): # return ENVIRONMENT.bool("S3_SECURE", default=False) pass + def get_object_store_access_key_default(self, requested_name: str = ""): + """Obtain object store access key.""" + return self.get_object_store_access_key(requested_name) + @staticmethod def get_object_store_access_key(requested_name: str = ""): """Obtain object store access key.""" return ENVIRONMENT.get_value("S3_ACCESS_KEY", default=None) + def get_object_store_secret_key_default(self, requested_name: str = ""): + """Obtain object store secret key.""" + return self.get_object_store_secret_key(requested_name) + @staticmethod def get_object_store_secret_key(requested_name: str = ""): """Obtain object store secret key.""" @@ -359,6 +386,12 @@ def get_endpoint_port(app, name, default): class ClowderConfigurator(Configurator): """Obtain configuration based on using Clowder and app-common.""" + def __init__(self): + session = boto3.Session() + credentials = session.get_credentials() + self.credentials = credentials.get_frozen_credentials() + self.is_minio = ENVIRONMENT.get_value("TRINO_S3A_OR_S3", default="s3a") == "s3a" + @staticmethod def get_feature_flag_host(): """Obtain feature flag (Unleash) host.""" @@ -431,6 +464,11 @@ def get_cloudwatch_log_group(): """Obtain cloudwatch log group.""" return LoadedConfig.logging.cloudwatch.logGroup + def get_object_store_endpoint_default(self): + if self.is_minio: + return self.get_object_store_endpoint() + return ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com") + @staticmethod def get_object_store_endpoint(): """Obtain object store endpoint.""" @@ -465,6 +503,11 @@ def get_object_store_tls(): else: return False + def get_object_store_access_key_default(self, requested_name: str = ""): + if self.is_minio: + return self.get_object_store_access_key(requested_name) + return self.credentials.access_key + @staticmethod def get_object_store_access_key(requested_name: str = ""): """Obtain object store access key.""" @@ -475,6 +518,11 @@ def get_object_store_access_key(requested_name: str = ""): if LoadedConfig.objectStore.accessKey: return LoadedConfig.objectStore.accessKey + def get_object_store_secret_key_default(self, requested_name: str = ""): + if self.is_minio: + return self.get_object_store_secret_key(requested_name) + return self.credentials.secret_key + @staticmethod def get_object_store_secret_key(requested_name: str = ""): """Obtain object store secret key.""" @@ -610,7 +658,7 @@ class ConfigFactory: @staticmethod def get_configurator(): """Returns configurator based on mode from env variable.""" - return ClowderConfigurator if CLOWDER_ENABLED else EnvConfigurator + return ClowderConfigurator() if CLOWDER_ENABLED else EnvConfigurator() CONFIGURATOR = ConfigFactory.get_configurator() diff --git a/koku/koku/settings.py b/koku/koku/settings.py index 446bb4edf2..c80970ca47 100644 --- a/koku/koku/settings.py +++ b/koku/koku/settings.py @@ -493,11 +493,11 @@ REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report") REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report") S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60) -S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint() +S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint_default() S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1") S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) -S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) -S3_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET) +S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key_default(REQUESTED_BUCKET) +S3_SECRET = CONFIGURATOR.get_object_store_secret_key_default(REQUESTED_BUCKET) # HCS S3_HCS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET) S3_HCS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET) diff --git a/koku/masu/test/api/test_db_performance.py b/koku/masu/test/api/test_db_performance.py index c6f43093ec..a509db03b0 100644 --- a/koku/masu/test/api/test_db_performance.py +++ b/koku/masu/test/api/test_db_performance.py @@ -13,8 +13,7 @@ from koku.configurator import CONFIGURATOR from masu.api.db_performance.db_performance import DBPerformanceStats - -TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", CONFIGURATOR.__bases__, dict(CONFIGURATOR.__dict__)) +TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", (CONFIGURATOR.__class__,), dict(CONFIGURATOR.__dict__)) def _get_database_name():