[COST-5135] Update Masu endpoint to handle new ocp on cloud flow (#5277)

* [COST-5135] Update Masu endpoint to handle new ocp on cloud flow * limit to managed providers too * Fix gcp trigger to prevent 500 * Add unittest --------- Co-authored-by: Cody Myers <[email protected]> Co-authored-by: myersCody <[email protected]>
project-koku · Nov 26, 2024 · 2758a31 · 2758a31
1 parent b61c403
commit 2758a31
Show file tree

Hide file tree

Showing 6 changed files with 59 additions and 6 deletions.
diff --git a/koku/masu/api/process_openshift_on_cloud.py b/koku/masu/api/process_openshift_on_cloud.py
@@ -23,8 +23,11 @@
 from common.queues import DownloadQueue
 from common.queues import get_customer_queue
 from common.queues import QUEUE_LIST
+from common.queues import SummaryQueue
+from masu.processor import is_managed_ocp_cloud_processing_enabled
 from masu.processor.tasks import process_daily_openshift_on_cloud as process_daily_openshift_on_cloud_task
 from masu.processor.tasks import process_openshift_on_cloud as process_openshift_on_cloud_task
+from masu.processor.tasks import process_openshift_on_cloud_trino
 
 LOG = logging.getLogger(__name__)
 REPORT_DATA_KEY = "process_openshift_on_cloud Task IDs"
@@ -68,7 +71,33 @@ def process_openshift_on_cloud(request):
         errmsg = f"You must provide a cloud provider UUID from {Provider.OPENSHIFT_ON_CLOUD_PROVIDER_LIST}."
         return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST)
 
-    months = get_months_in_date_range(start=start_date, end=end_date)
+    if provider.type in [Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL]:
+        invoice_month = DateHelper().invoice_month_from_bill_date(start_date)
+        months = get_months_in_date_range(start=start_date, end=end_date, invoice_month=invoice_month)
+    else:
+        months = get_months_in_date_range(start=start_date, end=end_date)
+
+    if provider.type in Provider.MANAGED_OPENSHIFT_ON_CLOUD_PROVIDER_LIST and is_managed_ocp_cloud_processing_enabled(
+        schema_name
+    ):
+        fallback_queue = get_customer_queue(schema_name, SummaryQueue)
+        for month in months:
+            tracing_id = str(uuid4())
+            report = {
+                "schema_name": schema_name,
+                "provider_type": provider.type,
+                "provider_uuid": cloud_provider_uuid,
+                "tracing_id": tracing_id,
+                "start": month[0],
+                "end": month[1],
+                "invoice_month": month[2],
+            }
+            ocp_async = process_openshift_on_cloud_trino.s(
+                [report], provider.type, schema_name, cloud_provider_uuid, tracing_id, masu_api_trigger=True
+            ).apply_async(queue=queue_name or fallback_queue)
+            async_results.append({f"Managed OCP on Cloud {str(month)}": str(ocp_async)})
+        return Response({REPORT_DATA_KEY: async_results})
+
     bill_dates = [ciso8601.parse_datetime(month[0]).replace(day=1).strftime("%Y-%m-%d") for month in months]
 
     for bill_date in bill_dates:

diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py
@@ -491,7 +491,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
     def populate_ocp_on_cloud_daily_trino(
         self, aws_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
     ):
-        """Populate the aws_openshift_daily trino table for OCP on AWS.
+        """Populate the managed_aws_openshift_daily trino table for OCP on AWS.
         Args:
             aws_provider_uuid (UUID) AWS source UUID.
             ocp_provider_uuid (UUID) OCP source UUID.

diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py
@@ -447,7 +447,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
     def populate_ocp_on_cloud_daily_trino(
         self, azure_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
     ):
-        """Populate the azure_openshift_daily trino table for OCP on Azure.
+        """Populate the managed_azure_openshift_daily trino table for OCP on Azure.
         Args:
             azure_provider_uuid (UUID) GCP source UUID.
             ocp_provider_uuid (UUID) OCP source UUID.

diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py
@@ -589,7 +589,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params):
     def populate_ocp_on_cloud_daily_trino(
         self, gcp_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags
     ):
-        """Populate the gcp_openshift_daily trino table for OCP on Azure.
+        """Populate the managed_gcp_openshift_daily trino table for OCP on GCP.
         Args:
             gcp_provider_uuid (UUID) GCP source UUID.
             ocp_provider_uuid (UUID) OCP source UUID.

diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py
@@ -1255,7 +1255,7 @@ def validate_daily_data(schema, start_date, end_date, provider_uuid, ocp_on_clou
 
 @celery_app.task(name="masu.processor.tasks.process_openshift_on_cloud_trino", queue=SummaryQueue.DEFAULT, bind=True)
 def process_openshift_on_cloud_trino(
-    self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id
+    self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id, masu_api_trigger=False
 ):
     """Process OCP on Cloud data into managed tables for summary"""
     reports_deduplicated = deduplicate_summary_reports(reports_to_summarize, manifest_list=[])
@@ -1275,7 +1275,7 @@ def process_openshift_on_cloud_trino(
         with ReportManifestDBAccessor() as manifest_accesor:
             tracing_id = report.get("tracing_id", report.get("manifest_uuid", "no-tracing-id"))
 
-            if not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")):
+            if not masu_api_trigger and not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")):
                 LOG.info(log_json(tracing_id, msg="manifest not ready for summary", context=report))
                 continue
 

diff --git a/koku/masu/test/api/test_process_openshift_on_cloud.py b/koku/masu/test/api/test_process_openshift_on_cloud.py
@@ -139,3 +139,27 @@ def test_get_process_daily_openshift_on_cloud(self, mock_process, _):
         self.assertEqual(response.status_code, 200)
         self.assertIn(expected_key, body)
         mock_process.s.return_value.apply_async.assert_called()
+
+    @patch("masu.api.process_openshift_on_cloud.process_openshift_on_cloud_trino")
+    @patch("masu.api.process_openshift_on_cloud.is_managed_ocp_cloud_processing_enabled")
+    @patch("koku.middleware.MASU")
+    def test_get_process_daily_openshift_on_cloud_trino(self, middleware, unleash, mock_task):
+        """Test the GET report_data endpoint for a provider that is daily partitioned."""
+        unleash.return_value = True
+        middleware.return_value = True
+        dh = DateHelper()
+        start_date = str(dh.this_month_start.date())
+        end_date = str(dh.this_month_end.date())
+        params = {
+            "schema": self.schema,
+            "provider_uuid": self.gcp_provider_uuid,
+            "start_date": start_date,
+            "end_date": end_date,
+        }
+        expected_key = "process_openshift_on_cloud Task IDs"
+        response = self.client.get(reverse("process_openshift_on_cloud"), params)
+        body = response.json()
+
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(expected_key, body)
+        mock_task.s.return_value.apply_async.assert_called()