Fix unittests

koku/masu/processor/parquet/managed_flow_params.py

@@ -5,6 +5,9 @@
 from typing import Dict
 from typing import List
 
+from dateutil.parser import parse
+from django.conf import settings
+
 from api.utils import DateHelper
 
 
@@ -22,6 +25,7 @@ class ManagedSqlMetadata:
     tmp_id: str = field(init=False)
 
     def __post_init__(self):
+        self._check_date_parameters_format()
         if not self.ocp_source_uuids:
             raise ValueError("ocp_source_uuids must not be empty.")
         if len(self.cloud_provider_uuid) == 0:
@@ -30,6 +34,13 @@ def __post_init__(self):
             raise ValueError("start_date cannot be after end_date.")
         self._generate_sql_params()
 
+    def _check_date_parameters_format(self):
+        """Checks to make sure the date parameters are in the correct format"""
+        if type(self.start_date) == str:
+            self.start_date = parse(self.start_date).astimezone(tz=settings.UTC)
+        if type(self.end_date) == str:
+            self.end_date = parse(self.end_date).astimezone(tz=settings.UTC)
+
     def _generate_sql_params(self):
         """Populates additional SQL parameters options"""
         days = DateHelper().list_days(self.start_date, self.end_date)

koku/masu/processor/parquet/ocp_cloud_parquet_report_processor.py

@@ -10,8 +10,6 @@
 from functools import cached_property
 
 import pandas as pd
-from dateutil.parser import parse
-from django.conf import settings
 from django_tenants.utils import schema_context
 
 from api.common import log_json
@@ -313,10 +311,6 @@ def process_ocp_cloud_trino(self, start_date, end_date):
         """Populate cloud_openshift_daily trino table via SQL."""
         if not (ocp_provider_uuids := self.get_ocp_provider_uuids_tuple()):
             return
-        if type(start_date) == str:
-            start_date = parse(start_date).astimezone(tz=settings.UTC)
-        if type(end_date) == str:
-            end_date = parse(end_date).astimezone(tz=settings.UTC)
         matched_tags = self.get_matched_tags(ocp_provider_uuids)
         matched_tag_strs = []
         if matched_tags:

koku/masu/test/database/test_aws_report_db_accessor.py

@@ -7,11 +7,11 @@
 import decimal
 import pkgutil
 from decimal import Decimal
+from unittest.mock import ANY
 from unittest.mock import Mock
 from unittest.mock import patch
 
 from dateutil import relativedelta
-from dateutil.parser import parse
 from django.conf import settings
 from django.db.models import F
 from django.db.models import Max
@@ -30,6 +30,7 @@
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.database.ocp_report_db_accessor import OCPReportDBAccessor
 from masu.database.report_manifest_db_accessor import ReportManifestDBAccessor
+from masu.processor.parquet.managed_flow_params import ManagedSqlMetadata
 from masu.test import MasuTestCase
 from reporting.models import OCPAWSCostLineItemProjectDailySummaryP
 from reporting.provider.all.models import EnabledTagKeys
@@ -38,7 +39,6 @@
 from reporting.provider.aws.models import AWSCostEntryLineItemDailySummary
 from reporting.provider.aws.models import AWSCostEntryLineItemSummaryByEC2ComputeP
 from reporting.provider.aws.models import TRINO_MANAGED_OCP_AWS_DAILY_TABLE
-from reporting.provider.aws.models import TRINO_OCP_ON_AWS_DAILY_TABLE
 
 
 class AWSReportDBAccessorTest(MasuTestCase):
@@ -523,22 +523,22 @@ def test_populate_ocp_on_cloud_daily_trino(self, mock_trino, mock_partition_dele
         """
         Test that calling ocp on cloud populate triggers the deletes and summary sql.
         """
-        start_date = parse("2024-08-01").astimezone(tz=settings.UTC)
-        end_date = parse("2024-08-05").astimezone(tz=settings.UTC)
-        year = "2024"
-        month = "08"
         matched_tags = "fake-tags"
-        expected_days = ("1", "2", "3", "4", "5")
-
-        self.accessor.populate_ocp_on_cloud_daily_trino(
-            self.aws_provider_uuid, self.ocp_provider_uuid, start_date, end_date, matched_tags
+        params = ManagedSqlMetadata(
+            self.schema_name,
+            [self.ocp_provider_uuid],
+            self.aws_provider_uuid,
+            "2024-08-01",
+            "2024-08-01",
+            matched_tags,
         )
+        self.accessor.populate_ocp_on_cloud_daily_trino(params)
         mock_partition_delete.assert_called_with(
-            expected_days,
+            params.days_tup,
             self.aws_provider_uuid,
             self.ocp_provider_uuid,
-            year,
-            month,
+            params.year,
+            params.month,
             TRINO_MANAGED_OCP_AWS_DAILY_TABLE,
         )
         mock_trino.assert_called()
@@ -548,21 +548,14 @@ def test_verify_populate_ocp_on_cloud_daily_trino(self, mock_trino):
         """
         Test validating trino tables.
         """
-        verification_params = {
-            "schema": self.schema,
-            "cloud_source_uuid": self.aws_provider_uuid,
-            "year": "2024",
-            "month": "08",
-            "managed_table": TRINO_MANAGED_OCP_AWS_DAILY_TABLE,
-            "parquet_table": TRINO_OCP_ON_AWS_DAILY_TABLE,
-        }
+        params = ManagedSqlMetadata(self.schema_name, ANY, self.aws_provider_uuid, "2024-08-01", "2024-08-01", ANY)
         with self.assertLogs("masu.database.aws_report_db_accessor", level="INFO") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], params)
             assert any(
                 "Verification successful" in log for log in logger.output
             ), "Verification successful not found in logs"
 
         mock_trino.side_effect = [[[False]]]
         with self.assertLogs("masu.database.aws_report_db_accessor", level="ERROR") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], params)
             assert any("Verification failed" in log for log in logger.output), "Verification failed not found in logs"

koku/masu/test/database/test_azure_report_db_accessor.py

@@ -4,10 +4,10 @@
 #
 """Test the AzureReportDBAccessor utility object."""
 import decimal
+from unittest.mock import ANY
 from unittest.mock import Mock
 from unittest.mock import patch
 
-from dateutil.parser import parse
 from django.conf import settings
 from django.db.models import F
 from django.db.models import Max
@@ -24,14 +24,14 @@
 from masu.database.azure_report_db_accessor import AzureReportDBAccessor
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.database.report_manifest_db_accessor import ReportManifestDBAccessor
+from masu.processor.parquet.managed_flow_params import ManagedSqlMetadata
 from masu.test import MasuTestCase
 from reporting.models import OCPAzureCostLineItemProjectDailySummaryP
 from reporting.provider.all.models import EnabledTagKeys
 from reporting.provider.all.models import TagMapping
 from reporting.provider.azure.models import AzureCostEntryLineItemDailySummary
 from reporting.provider.azure.models import AzureTagsSummary
 from reporting.provider.azure.models import TRINO_MANAGED_OCP_AZURE_DAILY_TABLE
-from reporting.provider.azure.models import TRINO_OCP_ON_AZURE_DAILY_TABLE
 
 
 class AzureReportDBAccessorTest(MasuTestCase):
@@ -426,22 +426,18 @@ def test_populate_ocp_on_cloud_daily_trino(self, mock_trino, mock_partition_dele
         """
         Test that calling ocp on cloud populate triggers the deletes and summary sql.
         """
-        start_date = parse("2024-08-01").astimezone(tz=settings.UTC)
-        end_date = parse("2024-08-05").astimezone(tz=settings.UTC)
-        year = "2024"
-        month = "08"
         matched_tags = "fake-tags"
-        expected_days = ("1", "2", "3", "4", "5")
-
-        self.accessor.populate_ocp_on_cloud_daily_trino(
-            self.azure_provider_uuid, self.ocp_provider_uuid, start_date, end_date, matched_tags
+        params = ManagedSqlMetadata(
+            ANY, [self.ocp_provider_uuid], self.azure_provider_uuid, "2024-08-01", "2024-08-05", matched_tags
         )
+
+        self.accessor.populate_ocp_on_cloud_daily_trino(params)
         mock_partition_delete.assert_called_with(
-            expected_days,
+            params.days_tup,
             self.azure_provider_uuid,
             self.ocp_provider_uuid,
-            year,
-            month,
+            params.year,
+            params.month,
             TRINO_MANAGED_OCP_AZURE_DAILY_TABLE,
         )
         mock_trino.assert_called()
@@ -451,21 +447,14 @@ def test_verify_populate_ocp_on_cloud_daily_trino(self, mock_trino):
         """
         Test validating trino tables.
         """
-        verification_params = {
-            "schema": self.schema,
-            "cloud_source_uuid": self.azure_provider_uuid,
-            "year": "2024",
-            "month": "08",
-            "managed_table": TRINO_MANAGED_OCP_AZURE_DAILY_TABLE,
-            "parquet_table": TRINO_OCP_ON_AZURE_DAILY_TABLE,
-        }
+        params = ManagedSqlMetadata(self.schema_name, ANY, self.azure_provider_uuid, "2024-08-01", "2024-08-01", ANY)
         with self.assertLogs("masu.database.azure_report_db_accessor", level="INFO") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], params)
             assert any(
                 "Verification successful" in log for log in logger.output
             ), "Verification successful not found in logs"
 
         mock_trino.side_effect = [[[False]]]
         with self.assertLogs("masu.database.azure_report_db_accessor", level="ERROR") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], params)
             assert any("Verification failed" in log for log in logger.output), "Verification failed not found in logs"

koku/masu/test/database/test_gcp_report_db_accessor.py

@@ -5,12 +5,12 @@
 """Test the GCPReportDBAccessor utility object."""
 import datetime
 import decimal
+from unittest.mock import ANY
 from unittest.mock import Mock
 from unittest.mock import patch
 from uuid import uuid4
 
 from dateutil import relativedelta
-from dateutil.parser import parse
 from django.conf import settings
 from django.db.models import F
 from django.db.models import Max
@@ -25,6 +25,7 @@
 from masu.database import GCP_REPORT_TABLE_MAP
 from masu.database.cost_model_db_accessor import CostModelDBAccessor
 from masu.database.gcp_report_db_accessor import GCPReportDBAccessor
+from masu.processor.parquet.managed_flow_params import ManagedSqlMetadata
 from masu.test import MasuTestCase
 from reporting.models import OCPGCPCostLineItemProjectDailySummaryP
 from reporting.provider.all.models import EnabledTagKeys
@@ -34,7 +35,6 @@
 from reporting.provider.gcp.models import GCPTagsSummary
 from reporting.provider.gcp.models import GCPTopology
 from reporting.provider.gcp.models import TRINO_MANAGED_OCP_GCP_DAILY_TABLE
-from reporting.provider.gcp.models import TRINO_OCP_ON_GCP_DAILY_TABLE
 from reporting_common.models import CostUsageReportManifest
 from reporting_common.models import CostUsageReportStatus
 
@@ -566,28 +566,29 @@ def test_populate_ocp_on_gcp_tag_information(self):
             ).count()
             self.assertEqual(0, actual_child_count)
 
+    @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor._clean_up_managed_temp_tables")
     @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor.delete_ocp_on_gcp_hive_partition_by_day")
     @patch("masu.database.gcp_report_db_accessor.GCPReportDBAccessor._execute_trino_multipart_sql_query")
-    def test_populate_ocp_on_cloud_daily_trino(self, mock_trino, mock_partition_delete):
+    def test_populate_ocp_on_cloud_daily_trino(self, mock_trino, mock_partition_delete, _):
         """
         Test that calling ocp on cloud populate triggers the deletes and summary sql.
         """
-        start_date = parse("2024-08-01").astimezone(tz=settings.UTC)
-        end_date = parse("2024-08-05").astimezone(tz=settings.UTC)
-        year = "2024"
-        month = "08"
         matched_tags = "fake-tags"
-        expected_days = ("1", "2", "3", "4", "5")
-
-        self.accessor.populate_ocp_on_cloud_daily_trino(
-            self.gcp_provider_uuid, self.ocp_provider_uuid, start_date, end_date, matched_tags
+        mparams = ManagedSqlMetadata(
+            self.schema_name,
+            [self.ocp_provider_uuid],
+            self.gcp_provider_uuid,
+            "2024-08-01",
+            "2024-08-05",
+            matched_tags,
         )
+        self.accessor.populate_ocp_on_cloud_daily_trino(mparams)
         mock_partition_delete.assert_called_with(
-            expected_days,
+            mparams.days_tup,
             self.gcp_provider_uuid,
             self.ocp_provider_uuid,
-            year,
-            month,
+            mparams.year,
+            mparams.month,
             TRINO_MANAGED_OCP_GCP_DAILY_TABLE,
         )
         mock_trino.assert_called()
@@ -597,21 +598,14 @@ def test_verify_populate_ocp_on_cloud_daily_trino(self, mock_trino):
         """
         Test validating trino tables.
         """
-        verification_params = {
-            "schema": self.schema,
-            "cloud_source_uuid": self.gcp_provider_uuid,
-            "year": "2024",
-            "month": "08",
-            "managed_table": TRINO_MANAGED_OCP_GCP_DAILY_TABLE,
-            "parquet_table": TRINO_OCP_ON_GCP_DAILY_TABLE,
-        }
+        mparams = ManagedSqlMetadata(self.schema_name, ANY, self.gcp_provider_uuid, "2024-08-01", "2024-08-01", ANY)
         with self.assertLogs("masu.database.gcp_report_db_accessor", level="INFO") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], mparams)
             assert any(
                 "Verification successful" in log for log in logger.output
             ), "Verification successful not found in logs"
 
         mock_trino.side_effect = [[[False]], [["fail"]]]
         with self.assertLogs("masu.database.gcp_report_db_accessor", level="ERROR") as logger:
-            self.accessor.verify_populate_ocp_on_cloud_daily_trino(verification_params)
+            self.accessor.verify_populate_ocp_on_cloud_daily_trino([], mparams)
             assert any("Verification failed" in log for log in logger.output), "Verification failed not found in logs"

koku/masu/test/processor/parquet/test_ocp_cloud_parquet_report_processor.py

@@ -7,8 +7,6 @@
 from unittest.mock import patch
 
 import pandas as pd
-from dateutil.parser import parse
-from django.conf import settings
 from django_tenants.utils import schema_context
 
 from api.models import Provider
@@ -18,6 +16,7 @@
 from masu.database.gcp_report_db_accessor import GCPReportDBAccessor
 from masu.database.ocp_report_db_accessor import OCPReportDBAccessor
 from masu.processor.ocp.ocp_cloud_updater_base import OCPCloudUpdaterBase
+from masu.processor.parquet.managed_flow_params import ManagedSqlMetadata
 from masu.processor.parquet.ocp_cloud_parquet_report_processor import OCPCloudParquetReportProcessor
 from masu.processor.parquet.parquet_report_processor import OPENSHIFT_REPORT_TYPE
 from masu.processor.parquet.parquet_report_processor import PARQUET_EXT
@@ -467,10 +466,13 @@ def test_instantiating_processor_with_manifest_id(self):
 
     def test_process_ocp_cloud_trino(self):
         """Test that processing ocp on cloud via trino calls the expected functions."""
+        ocp_uuids = [self.ocp_provider_uuid]
         start_date = "2024-08-01"
         end_date = "2024-08-05"
-        ocp_uuids = (self.ocp_provider_uuid,)
         matched_tags = []
+        managed_sql_params = ManagedSqlMetadata(
+            ANY, ocp_uuids, self.aws_provider_uuid, start_date, end_date, matched_tags
+        )
         with patch(
             (
                 "masu.processor.parquet.ocp_cloud_parquet_report_processor"
@@ -492,10 +494,4 @@ def test_process_ocp_cloud_trino(self):
                 context={"request_id": self.request_id, "start_date": self.start_date, "create_table": True},
             )
             rp.process_ocp_cloud_trino(start_date, end_date)
-            accessor.populate_ocp_on_cloud_daily_trino.assert_called_with(
-                self.aws_provider_uuid,
-                self.ocp_provider_uuid,
-                parse(start_date).astimezone(tz=settings.UTC),
-                parse(end_date).astimezone(tz=settings.UTC),
-                ANY,
-            )
+            accessor.populate_ocp_on_cloud_daily_trino.assert_called_with(managed_sql_params)