some fixes

project-koku · Jan 16, 2025 · b9c4154 · b9c4154
1 parent 8f5771f
commit b9c4154
Show file tree

Hide file tree

Showing 6 changed files with 20 additions and 20 deletions.
diff --git a/koku/masu/processor/report_parquet_processor_base.py b/koku/masu/processor/report_parquet_processor_base.py
@@ -65,7 +65,7 @@ def _execute_trino_sql(self, sql, schema_name: str):  # pragma: no cover
                 cur = conn.cursor()
                 cur.execute(sql)
                 rows = cur.fetchall()
-                LOG.debug(f"_execute_sql rows: {str(rows)}. Type: {type(rows)}")
+                LOG.debug(f"_execute_trino_sql rows: {str(rows)}. Type: {type(rows)}")
         except TrinoUserError as err:
             LOG.warning(err)
         except TrinoExternalError as err:

diff --git a/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py b/koku/masu/test/processor/aws/test_aws_report_parquet_processor.py
@@ -54,7 +54,7 @@ def test_postgres_summary_table(self):
         """Test that the correct table is returned."""
         self.assertEqual(self.processor.postgres_summary_table, AWSCostEntryLineItemDailySummary)
 
-    @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_sql")
+    @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_trino_sql")
     def test_create_bill(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start
@@ -73,7 +73,7 @@ def test_create_bill(self, mock_execute_sql):
             )
             self.assertIsNotNone(bill.first())
 
-    @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_sql")
+    @patch("masu.processor.aws.aws_report_parquet_processor.AWSReportParquetProcessor._execute_trino_sql")
     def test_create_bill_with_string_arg(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start

diff --git a/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py b/koku/masu/test/processor/gcp/test_gcp_report_parquet_processor.py
@@ -54,7 +54,7 @@ def test_postgres_summary_table(self):
         """Test that the correct table is returned."""
         self.assertEqual(self.processor.postgres_summary_table, GCPCostEntryLineItemDailySummary)
 
-    @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_sql")
+    @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_trino_sql")
     def test_create_bill(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start
@@ -72,7 +72,7 @@ def test_create_bill(self, mock_execute_sql):
             )
             self.assertIsNotNone(bill.first())
 
-    @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_sql")
+    @patch("masu.processor.gcp.gcp_report_parquet_processor.GCPReportParquetProcessor._execute_trino_sql")
     def test_create_bill_with_string_arg(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start

diff --git a/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py b/koku/masu/test/processor/oci/test_oci_report_parquet_processor.py
@@ -47,7 +47,7 @@ def test_postgres_summary_table(self):
         """Test that the correct table is returned."""
         self.assertEqual(self.processor.postgres_summary_table, OCICostEntryLineItemDailySummary)
 
-    @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_sql")
+    @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_trino_sql")
     def test_create_bill(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start
@@ -66,7 +66,7 @@ def test_create_bill(self, mock_execute_sql):
             )
             self.assertIsNotNone(bill.first())
 
-    @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_sql")
+    @patch("masu.processor.oci.oci_report_parquet_processor.OCIReportParquetProcessor._execute_trino_sql")
     def test_create_bill_with_string_arg(self, mock_execute_sql):
         """Test that a bill is created in the Postgres database."""
         bill_date = DateHelper().this_month_start

diff --git a/koku/masu/test/processor/test_report_parquet_processor_base.py b/koku/masu/test/processor/test_report_parquet_processor_base.py
@@ -91,7 +91,7 @@ def test_postgres_summary_table(self):
             self.processor.postgres_summary_table
 
     @override_settings(S3_BUCKET_NAME="test-bucket")
-    @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_sql")
+    @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_trino_sql")
     def test_generate_create_table_sql(self, mock_execute):
         """Test the generate parquet table sql."""
         generated_sql = self.processor._generate_create_table_sql()
@@ -111,7 +111,7 @@ def test_generate_create_table_sql(self, mock_execute):
         self.assertTrue(generated_sql.endswith(expected_end))
 
     @override_settings(S3_BUCKET_NAME="test-bucket")
-    @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_sql")
+    @patch("masu.processor.aws.aws_report_parquet_processor.ReportParquetProcessorBase._execute_trino_sql")
     def test_generate_create_table_sql_with_provider_map(self, mock_execute):
         """Test the generate parquet table sql."""
         partition_map = {
@@ -136,7 +136,7 @@ def test_generate_create_table_sql_with_provider_map(self, mock_execute):
             self.assertIn(f"{other_col} varchar", generated_sql)
         self.assertTrue(generated_sql.endswith(expected_end))
 
-    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql")
+    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql")
     def test_create_table(self, mock_execute):
         """Test the Trino/Hive create table method."""
         expected_logs = []
@@ -150,7 +150,7 @@ def test_create_table(self, mock_execute):
             for expected_log in expected_logs:
                 self.assertIn(expected_log, logger.output)
 
-    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql")
+    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql")
     def test_sync_hive_partitions(self, mock_execute):
         """Test that hive partitions are synced."""
         expected_log = self.log_output_info + str(
@@ -160,7 +160,7 @@ def test_sync_hive_partitions(self, mock_execute):
             self.processor.sync_hive_partitions()
             self.assertIn(expected_log, logger.output)
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_schema_exists_cache_value_in_cache(self, trino_mock):
         with patch(
             "masu.processor.report_parquet_processor_base.get_value_from_cache",
@@ -169,23 +169,23 @@ def test_schema_exists_cache_value_in_cache(self, trino_mock):
             self.assertTrue(self.processor.schema_exists())
             trino_mock.assert_not_called()
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_schema_exists_cache_value_not_in_cache(self, trino_mock):
         trino_mock.return_value = True
         key = build_trino_schema_exists_key(self.account)
         with patch("masu.processor.report_parquet_processor_base.set_value_in_cache") as mock_cache_set:
             self.assertTrue(self.processor.schema_exists())
             mock_cache_set.assert_called_with(key, True)
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_schema_exists_cache_value_not_in_cache_not_exists(self, trino_mock):
         trino_mock.return_value = False
         key = build_trino_schema_exists_key(self.account)
         with patch("masu.processor.report_parquet_processor_base.set_value_in_cache") as mock_cache_set:
             self.assertFalse(self.processor.schema_exists())
             mock_cache_set.assert_called_with(key, False)
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_table_exists_cache_value_in_cache(self, trino_mock):
         with patch(
             "masu.processor.report_parquet_processor_base.get_value_from_cache",
@@ -194,23 +194,23 @@ def test_table_exists_cache_value_in_cache(self, trino_mock):
             self.assertTrue(self.processor.table_exists())
             trino_mock.assert_not_called()
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_table_exists_cache_value_not_in_cache(self, trino_mock):
         trino_mock.return_value = True
         key = build_trino_table_exists_key(self.account, self.table_name)
         with patch("masu.processor.report_parquet_processor_base.set_value_in_cache") as mock_cache_set:
             self.assertTrue(self.processor.table_exists())
             mock_cache_set.assert_called_with(key, True)
 
-    @patch.object(ReportParquetProcessorBase, "_execute_sql")
+    @patch.object(ReportParquetProcessorBase, "_execute_trino_sql")
     def test_table_exists_cache_value_not_in_cache_not_exists(self, trino_mock):
         trino_mock.return_value = False
         key = build_trino_table_exists_key(self.account, self.table_name)
         with patch("masu.processor.report_parquet_processor_base.set_value_in_cache") as mock_cache_set:
             self.assertFalse(self.processor.table_exists())
             mock_cache_set.assert_called_with(key, False)
 
-    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_sql")
+    @patch("masu.processor.report_parquet_processor_base.ReportParquetProcessorBase._execute_trino_sql")
     def test_create_schema(self, mock_execute):
         """Test that hive partitions are synced."""
         expected_log = self.log_output_info + str(

diff --git a/koku/masu/util/aws/common.py b/koku/masu/util/aws/common.py
@@ -511,7 +511,7 @@ def get_bills_from_provider(
     return bills
 
 
-def get_s3_resource(access_key, secret_key, region):  # pragma: no cover
+def get_s3_resource(access_key, secret_key, region, endpoint_url=settings.S3_ENDPOINT):  # pragma: no cover
     """
     Obtain the s3 session client
     """
@@ -521,7 +521,7 @@ def get_s3_resource(access_key, secret_key, region):  # pragma: no cover
         aws_secret_access_key=secret_key,
         region_name=region,
     )
-    return aws_session.resource("s3", endpoint_url=settings.S3_ENDPOINT, config=config)
+    return aws_session.resource("s3", endpoint_url=endpoint_url, config=config)
 
 
 def copy_data_to_s3_bucket(request_id, path, filename, data, metadata=None, context=None):