more prep for glue (#5465)

.env.example

@@ -40,6 +40,8 @@ AWS_RESOURCE_NAME=YOUR_COST_MANAGEMENT_AWS_ARN
 
 # Glue
 SCHEMA_SUFFIX="" # if DEVELOPMENT=True, this can be left empty and will default to $USER; otherwise, set this value to something unique
+TRINO_SCHEMA_PREFIX=""
+TRINO_S3A_OR_S3=s3
 
 AWS_CATALOG_ID=589173575009
 MINIO_ENDPOINT=http://koku-minio:9000
@@ -48,6 +50,7 @@ S3_ACCESS_KEY=kokuminioaccess
 S3_SECRET=kokuminiosecret
 
 S3_BUCKET_NAME=koku-bucket
+S3_BUCKET_NAME_OCP_INGRESS=koku-bucket
 S3_REGION=us-east-1
 
 # GCP

.github/workflows/ci.yml

@@ -70,6 +70,8 @@ jobs:
           echo Dockerfile >> docker-files.txt;
           echo pr_check.sh >> docker-files.txt;
           echo deploy/clowdapp.yaml >> docker-files.txt;
+          echo Jenkinsfile >> docker-files.txt;
+          echo ci/functions.sh >> docker-files.txt;
 
       - name: Show Dockerfiles
         run: cat docker-files.txt
@@ -334,11 +336,11 @@ jobs:
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
-          file: ./coverage.xml
+          files: ./coverage.xml
           flags: unittests
           name: Python-${{ matrix.python-version}}
           fail_ci_if_error: true
-          plugin: pycoverage  # Only run one plugin even though we don't want any to run.
+          plugins: noop
 
       - name: Set Codecov job status on skipped tests
         if: needs.changed-files.outputs.run_tests != 'true'

deploy/clowdapp.yaml

@@ -5614,6 +5614,8 @@ objects:
         env:
         - name: CLOWDER_ENABLED
           value: ${CLOWDER_ENABLED}
+        - name: AWS_SHARED_CREDENTIALS_FILE
+          value: ${AWS_SHARED_CREDENTIALS_FILE}
         - name: SCHEMA_SUFFIX
           value: ${SCHEMA_SUFFIX}
         - name: TRINO_SCHEMA_PREFIX
@@ -5644,6 +5646,17 @@ objects:
           requests:
             cpu: ${CPU_REQUEST_KOKU_MIGRATIONS}
             memory: ${MEMORY_REQUEST_KOKU_MIGRATIONS}
+        volumeMounts:
+        - mountPath: /etc/aws
+          name: aws-credentials
+          readOnly: true
+        volumes:
+        - name: aws-credentials
+          secret:
+            items:
+            - key: aws-credentials
+              path: aws-credentials
+            secretName: koku-aws
     - name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION}
       podSpec:
         args:
@@ -5653,6 +5666,8 @@ objects:
         env:
         - name: CLOWDER_ENABLED
           value: ${CLOWDER_ENABLED}
+        - name: AWS_SHARED_CREDENTIALS_FILE
+          value: ${AWS_SHARED_CREDENTIALS_FILE}
         - name: SCHEMA_SUFFIX
           value: ${SCHEMA_SUFFIX}
         - name: TRINO_SCHEMA_PREFIX
@@ -5677,6 +5692,17 @@ objects:
           requests:
             cpu: ${CPU_REQUEST_KOKU_MGMT}
             memory: ${MEMORY_REQUEST_KOKU_MGMT}
+        volumeMounts:
+        - mountPath: /etc/aws
+          name: aws-credentials
+          readOnly: true
+        volumes:
+        - name: aws-credentials
+          secret:
+            items:
+            - key: aws-credentials
+              path: aws-credentials
+            secretName: koku-aws
     kafkaTopics:
     - topicName: platform.sources.event-stream
     - topicName: platform.upload.announce

deploy/kustomize/base/base.yaml

@@ -73,6 +73,8 @@ objects:
         env:
         - name: CLOWDER_ENABLED
           value: ${CLOWDER_ENABLED}
+        - name: AWS_SHARED_CREDENTIALS_FILE
+          value: ${AWS_SHARED_CREDENTIALS_FILE}
         - name: SCHEMA_SUFFIX
           value: ${SCHEMA_SUFFIX}
         - name: TRINO_SCHEMA_PREFIX
@@ -95,6 +97,17 @@ objects:
           value: ${TENANT_MULTIPROCESSING_MAX_PROCESSES}
         - name: TENANT_MULTIPROCESSING_CHUNKS
           value: ${TENANT_MULTIPROCESSING_CHUNKS}
+        volumeMounts:
+        - mountPath: /etc/aws
+          name: aws-credentials
+          readOnly: true
+        volumes:
+        - name: aws-credentials
+          secret:
+            items:
+            - key: aws-credentials
+              path: aws-credentials
+            secretName: koku-aws
     # ====================================================
     #      koku Management Command Job
     # ====================================================
@@ -115,6 +128,8 @@ objects:
         env:
         - name: CLOWDER_ENABLED
           value: ${CLOWDER_ENABLED}
+        - name: AWS_SHARED_CREDENTIALS_FILE
+          value: ${AWS_SHARED_CREDENTIALS_FILE}
         - name: SCHEMA_SUFFIX
           value: ${SCHEMA_SUFFIX}
         - name: TRINO_SCHEMA_PREFIX
@@ -131,6 +146,17 @@ objects:
           value: ${TRINO_HOST}
         - name: TRINO_PORT
           value: ${TRINO_PORT}
+        volumeMounts:
+        - mountPath: /etc/aws
+          name: aws-credentials
+          readOnly: true
+        volumes:
+        - name: aws-credentials
+          secret:
+            items:
+            - key: aws-credentials
+              path: aws-credentials
+            secretName: koku-aws
     # The bulk of your App. This is where your running apps will live
     deployments:
     -

dev/scripts/load_test_customer_data.sh

@@ -75,7 +75,7 @@ export OS
 
 export S3_ACCESS_KEY="${S3_ACCESS_KEY}"
 export S3_SECRET_KEY="${S3_SECRET}"
-export S3_BUCKET_NAME="ocp-ingress"
+export S3_BUCKET_NAME="${S3_BUCKET_NAME_OCP_INGRESS}"
 
 
 log-info "Calculating dates..."

koku/koku/configurator.py

@@ -7,12 +7,17 @@
 """
 import pathlib
 
-from .env import ENVIRONMENT
+import boto3
 
+from .env import ENVIRONMENT
 
 CLOWDER_ENABLED = ENVIRONMENT.bool("CLOWDER_ENABLED", default=False)
 if CLOWDER_ENABLED:
-    from app_common_python import ObjectBuckets, LoadedConfig, KafkaTopics, KafkaServers, DependencyEndpoints
+    from app_common_python import DependencyEndpoints
+    from app_common_python import KafkaServers
+    from app_common_python import KafkaTopics
+    from app_common_python import LoadedConfig
+    from app_common_python import ObjectBuckets
 
 
 class Configurator:
@@ -78,6 +83,9 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         pass
 
+    def get_object_store_endpoint_default(self):
+        pass
+
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -98,11 +106,19 @@ def get_object_store_tls():
         """Obtain object store secret key."""
         pass
 
+    def get_object_store_access_key_default(self, requested_name: str = ""):
+        """Obtain object store access key."""
+        pass
+
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
         pass
 
+    def get_object_store_secret_key_default(self, requested_name: str = ""):
+        """Obtain object store secret key."""
+        pass
+
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -250,6 +266,9 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         return ENVIRONMENT.get_value("CW_LOG_GROUP", default="platform-dev")
 
+    def get_object_store_endpoint_default(self):
+        return self.get_object_store_endpoint()
+
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -276,11 +295,19 @@ def get_object_store_tls():
         # return ENVIRONMENT.bool("S3_SECURE", default=False)
         pass
 
+    def get_object_store_access_key_default(self, requested_name: str = ""):
+        """Obtain object store access key."""
+        return self.get_object_store_access_key(requested_name)
+
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
         return ENVIRONMENT.get_value("S3_ACCESS_KEY", default=None)
 
+    def get_object_store_secret_key_default(self, requested_name: str = ""):
+        """Obtain object store secret key."""
+        return self.get_object_store_secret_key(requested_name)
+
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -359,6 +386,12 @@ def get_endpoint_port(app, name, default):
 class ClowderConfigurator(Configurator):
     """Obtain configuration based on using Clowder and app-common."""
 
+    def __init__(self):
+        session = boto3.Session()
+        credentials = session.get_credentials()
+        self.credentials = credentials.get_frozen_credentials()
+        self.is_minio = ENVIRONMENT.get_value("TRINO_S3A_OR_S3", default="s3a") == "s3a"
+
     @staticmethod
     def get_feature_flag_host():
         """Obtain feature flag (Unleash) host."""
@@ -431,6 +464,11 @@ def get_cloudwatch_log_group():
         """Obtain cloudwatch log group."""
         return LoadedConfig.logging.cloudwatch.logGroup
 
+    def get_object_store_endpoint_default(self):
+        if self.is_minio:
+            return self.get_object_store_endpoint()
+        return ENVIRONMENT.get_value("S3_ENDPOINT", default="https://s3.amazonaws.com")
+
     @staticmethod
     def get_object_store_endpoint():
         """Obtain object store endpoint."""
@@ -465,6 +503,11 @@ def get_object_store_tls():
         else:
             return False
 
+    def get_object_store_access_key_default(self, requested_name: str = ""):
+        if self.is_minio:
+            return self.get_object_store_access_key(requested_name)
+        return self.credentials.access_key
+
     @staticmethod
     def get_object_store_access_key(requested_name: str = ""):
         """Obtain object store access key."""
@@ -475,6 +518,11 @@ def get_object_store_access_key(requested_name: str = ""):
         if LoadedConfig.objectStore.accessKey:
             return LoadedConfig.objectStore.accessKey
 
+    def get_object_store_secret_key_default(self, requested_name: str = ""):
+        if self.is_minio:
+            return self.get_object_store_secret_key(requested_name)
+        return self.credentials.secret_key
+
     @staticmethod
     def get_object_store_secret_key(requested_name: str = ""):
         """Obtain object store secret key."""
@@ -610,7 +658,7 @@ class ConfigFactory:
     @staticmethod
     def get_configurator():
         """Returns configurator based on mode from env variable."""
-        return ClowderConfigurator if CLOWDER_ENABLED else EnvConfigurator
+        return ClowderConfigurator() if CLOWDER_ENABLED else EnvConfigurator()
 
 
 CONFIGURATOR = ConfigFactory.get_configurator()

koku/koku/settings.py

@@ -493,11 +493,11 @@
 REQUESTED_ROS_BUCKET = ENVIRONMENT.get_value("REQUESTED_ROS_BUCKET", default="ros-report")
 REQUESTED_SUBS_BUCKET = ENVIRONMENT.get_value("REQUESTED_SUBS_BUCKET", default="subs-report")
 S3_TIMEOUT = ENVIRONMENT.int("S3_CONNECTION_TIMEOUT", default=60)
-S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint()
+S3_ENDPOINT = CONFIGURATOR.get_object_store_endpoint_default()
 S3_REGION = ENVIRONMENT.get_value("S3_REGION", default="us-east-1")
 S3_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET)
-S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET)
-S3_SECRET = CONFIGURATOR.get_object_store_secret_key(REQUESTED_BUCKET)
+S3_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key_default(REQUESTED_BUCKET)
+S3_SECRET = CONFIGURATOR.get_object_store_secret_key_default(REQUESTED_BUCKET)
 # HCS
 S3_HCS_BUCKET_NAME = CONFIGURATOR.get_object_store_bucket(REQUESTED_BUCKET)
 S3_HCS_ACCESS_KEY = CONFIGURATOR.get_object_store_access_key(REQUESTED_BUCKET)

koku/masu/test/api/test_db_performance.py

@@ -13,8 +13,7 @@
 from koku.configurator import CONFIGURATOR
 from masu.api.db_performance.db_performance import DBPerformanceStats
 
-
-TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", CONFIGURATOR.__bases__, dict(CONFIGURATOR.__dict__))
+TEST_CONFIGURATOR = type("TEST_CONFIGURATOR", (CONFIGURATOR.__class__,), dict(CONFIGURATOR.__dict__))
 
 
 def _get_database_name():