diff --git a/.env.example b/.env.example index 6b41b1d22a..36f3e3f670 100644 --- a/.env.example +++ b/.env.example @@ -32,7 +32,6 @@ TAG_ENABLED_LIMIT=200 # Set the max amount of tags per account DELAYED_TASK_TIME=30 # Set the seconds before a delayed summary task should expire DELAYED_TASK_POLLING_MINUTES=5 # Set the time before the delayed task kick off. -ENABLE_S3_ARCHIVING=True SKIP_MINIO_DATA_DELETION=False # AWS @@ -82,12 +81,10 @@ DEVELOPMENT_IDENTITY='{"identity": {"account_number": "10001", "org_id": "123456 # Glue SCHEMA_SUFFIX="" -S3_ENDPOINT=https://s3.us-east-1.amazonaws.com -WAREHOUSE_PATH=data AWS_CATALOG_ID=589173575009 -S3_BUCKET_NAME=CHANGEME +S3_ENDPOINT=https://s3.us-east-1.amazonaws.com +S3_BUCKET_NAME=CHANGEME S3_ACCESS_KEY=CHANGEME S3_SECRET=CHANGEME -S3_BUCKET_PATH=data S3_REGION=us-east-1 diff --git a/dev/containers/trino/etc/catalog/glue.properties b/dev/containers/trino/etc/catalog/glue.properties index db57348a19..65d2f0d96d 100644 --- a/dev/containers/trino/etc/catalog/glue.properties +++ b/dev/containers/trino/etc/catalog/glue.properties @@ -14,7 +14,7 @@ s3.aws-access-key=${ENV:AWS_ACCESS_KEY_ID} s3.aws-secret-key=${ENV:AWS_SECRET_ACCESS_KEY} s3.path-style-access=true -hive.metastore.glue.default-warehouse-dir=s3://${ENV:S3_BUCKET_NAME}/${ENV:WAREHOUSE_PATH} +hive.metastore.glue.default-warehouse-dir=s3://${ENV:S3_BUCKET_NAME}/data hive.metastore.glue.region=${ENV:S3_REGION} hive.metastore.glue.aws-access-key=${ENV:AWS_ACCESS_KEY_ID} hive.metastore.glue.aws-secret-key=${ENV:AWS_SECRET_ACCESS_KEY} diff --git a/docker-compose.yml b/docker-compose.yml index fac247be27..022c896b5d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -681,7 +681,6 @@ services: - S3_ENDPOINT=${S3_ENDPOINT-http://koku-minio:9000} - S3_BUCKET_NAME=${S3_BUCKET_NAME-koku-bucket} - S3_REGION=${S3_REGION-us-east-1} - - WAREHOUSE_PATH=${WAREHOUSE_PATH} - DATABASE_NAME=${DATABASE_NAME-postgres} - DATABASE_USER=${DATABASE_USER-postgres} - DATABASE_PASSWORD=${DATABASE_PASSWORD-postgres} diff --git a/koku/masu/test/util/aws/test_common.py b/koku/masu/test/util/aws/test_common.py index fadc43aa92..6f3e687261 100644 --- a/koku/masu/test/util/aws/test_common.py +++ b/koku/masu/test/util/aws/test_common.py @@ -262,8 +262,9 @@ def test_update_account_aliases_no_aliases(self): mock_account_id = self.account_id - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization"), ): mock_get.return_value = (mock_account_id, mock_account_id) utils.update_account_aliases(self.aws_provider) @@ -279,8 +280,9 @@ def test_update_account_aliases_with_aliases(self): mock_account_id = self.account_id mock_alias = "mock_alias" - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization"), ): mock_get.return_value = (mock_account_id, mock_alias) utils.update_account_aliases(self.aws_provider) @@ -298,9 +300,10 @@ def test_update_account_aliases_with_aliases_and_orgs(self): mock_alias = "mock_alias" mock_alias2 = "mock_alias2" - with patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, patch( - "masu.util.aws.common.get_account_names_by_organization" - ) as mock_get_orgs: + with ( + patch("masu.util.aws.common.get_account_alias_from_role_arn") as mock_get, + patch("masu.util.aws.common.get_account_names_by_organization") as mock_get_orgs, + ): mock_get.return_value = (mock_account_id, mock_alias) mock_get_orgs.return_value = [{"id": mock_account_id2, "name": mock_alias2}] utils.update_account_aliases(self.aws_provider) @@ -497,10 +500,12 @@ def test_remove_s3_objects_not_matching_metadata(self): """Test remove_s3_objects_not_matching_metadata.""" metadata_key = "manifestid" metadata_value = "manifest_id" - removed = utils.delete_s3_objects_not_matching_metadata( - "request_id", None, metadata_key=metadata_key, metadata_value_check=metadata_value - ) - self.assertEqual(removed, []) + with patch("masu.util.aws.common.get_s3_resource") as mock_s3: + mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [] + removed = utils.delete_s3_objects_not_matching_metadata( + "request_id", None, metadata_key=metadata_key, metadata_value_check=metadata_value + ) + self.assertEqual(removed, []) start_date = self.dh.this_month_start s3_csv_path = get_path_prefix( @@ -536,9 +541,10 @@ def test_remove_s3_objects_not_matching_metadata(self): ) self.assertListEqual(removed, []) - with patch("masu.util.aws.common.get_s3_objects_not_matching_metadata") as mock_get_objects, patch( - "masu.util.aws.common.get_s3_resource" - ) as mock_s3: + with ( + patch("masu.util.aws.common.get_s3_objects_not_matching_metadata") as mock_get_objects, + patch("masu.util.aws.common.get_s3_resource") as mock_s3, + ): mock_s3.return_value.Object.return_value.delete.side_effect = ClientError({}, "Error") mock_get_objects.return_value = [] removed = utils.delete_s3_objects_not_matching_metadata( @@ -604,10 +610,12 @@ def test_remove_s3_objects_matching_metadata(self): """Test remove_s3_objects_matching_metadata.""" metadata_key = "manifestid" metadata_value = "manifest_id" - removed = utils.delete_s3_objects_matching_metadata( - "request_id", None, metadata_key=metadata_key, metadata_value_check=metadata_value - ) - self.assertEqual(removed, []) + with patch("masu.util.aws.common.get_s3_resource") as mock_s3: + mock_s3.return_value.Bucket.return_value.objects.filter.return_value = [] + removed = utils.delete_s3_objects_matching_metadata( + "request_id", None, metadata_key=metadata_key, metadata_value_check=metadata_value + ) + self.assertEqual(removed, []) start_date = self.dh.this_month_start s3_csv_path = get_path_prefix( @@ -644,9 +652,10 @@ def test_remove_s3_objects_matching_metadata(self): ) self.assertListEqual(removed, []) - with patch("masu.util.aws.common.get_s3_objects_matching_metadata") as mock_get_objects, patch( - "masu.util.aws.common.get_s3_resource" - ) as mock_s3: + with ( + patch("masu.util.aws.common.get_s3_objects_matching_metadata") as mock_get_objects, + patch("masu.util.aws.common.get_s3_resource") as mock_s3, + ): mock_s3.return_value.Object.return_value.delete.side_effect = ClientError({}, "Error") mock_get_objects.return_value = [] removed = utils.delete_s3_objects_matching_metadata( @@ -665,8 +674,9 @@ def test_copy_data_to_s3_bucket(self): with self.assertRaises(utils.UploadError): utils.copy_data_to_s3_bucket("request_id", "path", "filename", "data", "manifest_id") + @patch("masu.util.aws.common.get_s3_resource", return_value="fake resource") @patch("masu.util.aws.common.copy_data_to_s3_bucket") - def test_copy_local_hcs_report_file_to_s3_bucket_with_finalize(self, mock_copy): + def test_copy_local_hcs_report_file_to_s3_bucket_with_finalize(self, mock_copy, mock_resource): """Test that the proper metadata is used when a finalized date is passed in with the finalize option""" fake_request_id = "fake_id" fake_s3_path = "fake_path" @@ -679,7 +689,13 @@ def test_copy_local_hcs_report_file_to_s3_bucket_with_finalize(self, mock_copy): fake_request_id, fake_s3_path, fake_filename, fake_filename, True, "2023-08-15", expected_context ) mock_copy.assert_called_once_with( - fake_request_id, fake_s3_path, fake_filename, mock_op(), expected_metadata, expected_context + fake_request_id, + fake_s3_path, + fake_filename, + mock_op(), + expected_metadata, + expected_context, + "fake resource", ) def test_match_openshift_resources_and_labels(self):