From 2758a315edec42ee7aec44496f82512c0d773b7a Mon Sep 17 00:00:00 2001 From: Corey Goodfred Date: Tue, 26 Nov 2024 10:01:47 -0500 Subject: [PATCH] [COST-5135] Update Masu endpoint to handle new ocp on cloud flow (#5277) * [COST-5135] Update Masu endpoint to handle new ocp on cloud flow * limit to managed providers too * Fix gcp trigger to prevent 500 * Add unittest --------- Co-authored-by: Cody Myers Co-authored-by: myersCody --- koku/masu/api/process_openshift_on_cloud.py | 31 ++++++++++++++++++- koku/masu/database/aws_report_db_accessor.py | 2 +- .../masu/database/azure_report_db_accessor.py | 2 +- koku/masu/database/gcp_report_db_accessor.py | 2 +- koku/masu/processor/tasks.py | 4 +-- .../api/test_process_openshift_on_cloud.py | 24 ++++++++++++++ 6 files changed, 59 insertions(+), 6 deletions(-) diff --git a/koku/masu/api/process_openshift_on_cloud.py b/koku/masu/api/process_openshift_on_cloud.py index e98ece4dc5..8dcf8dfb27 100644 --- a/koku/masu/api/process_openshift_on_cloud.py +++ b/koku/masu/api/process_openshift_on_cloud.py @@ -23,8 +23,11 @@ from common.queues import DownloadQueue from common.queues import get_customer_queue from common.queues import QUEUE_LIST +from common.queues import SummaryQueue +from masu.processor import is_managed_ocp_cloud_processing_enabled from masu.processor.tasks import process_daily_openshift_on_cloud as process_daily_openshift_on_cloud_task from masu.processor.tasks import process_openshift_on_cloud as process_openshift_on_cloud_task +from masu.processor.tasks import process_openshift_on_cloud_trino LOG = logging.getLogger(__name__) REPORT_DATA_KEY = "process_openshift_on_cloud Task IDs" @@ -68,7 +71,33 @@ def process_openshift_on_cloud(request): errmsg = f"You must provide a cloud provider UUID from {Provider.OPENSHIFT_ON_CLOUD_PROVIDER_LIST}." return Response({"Error": errmsg}, status=status.HTTP_400_BAD_REQUEST) - months = get_months_in_date_range(start=start_date, end=end_date) + if provider.type in [Provider.PROVIDER_GCP, Provider.PROVIDER_GCP_LOCAL]: + invoice_month = DateHelper().invoice_month_from_bill_date(start_date) + months = get_months_in_date_range(start=start_date, end=end_date, invoice_month=invoice_month) + else: + months = get_months_in_date_range(start=start_date, end=end_date) + + if provider.type in Provider.MANAGED_OPENSHIFT_ON_CLOUD_PROVIDER_LIST and is_managed_ocp_cloud_processing_enabled( + schema_name + ): + fallback_queue = get_customer_queue(schema_name, SummaryQueue) + for month in months: + tracing_id = str(uuid4()) + report = { + "schema_name": schema_name, + "provider_type": provider.type, + "provider_uuid": cloud_provider_uuid, + "tracing_id": tracing_id, + "start": month[0], + "end": month[1], + "invoice_month": month[2], + } + ocp_async = process_openshift_on_cloud_trino.s( + [report], provider.type, schema_name, cloud_provider_uuid, tracing_id, masu_api_trigger=True + ).apply_async(queue=queue_name or fallback_queue) + async_results.append({f"Managed OCP on Cloud {str(month)}": str(ocp_async)}) + return Response({REPORT_DATA_KEY: async_results}) + bill_dates = [ciso8601.parse_datetime(month[0]).replace(day=1).strftime("%Y-%m-%d") for month in months] for bill_date in bill_dates: diff --git a/koku/masu/database/aws_report_db_accessor.py b/koku/masu/database/aws_report_db_accessor.py index 5a6abd2003..7c99ae634c 100644 --- a/koku/masu/database/aws_report_db_accessor.py +++ b/koku/masu/database/aws_report_db_accessor.py @@ -491,7 +491,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params): def populate_ocp_on_cloud_daily_trino( self, aws_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags ): - """Populate the aws_openshift_daily trino table for OCP on AWS. + """Populate the managed_aws_openshift_daily trino table for OCP on AWS. Args: aws_provider_uuid (UUID) AWS source UUID. ocp_provider_uuid (UUID) OCP source UUID. diff --git a/koku/masu/database/azure_report_db_accessor.py b/koku/masu/database/azure_report_db_accessor.py index 969c13675c..7aef190775 100644 --- a/koku/masu/database/azure_report_db_accessor.py +++ b/koku/masu/database/azure_report_db_accessor.py @@ -447,7 +447,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params): def populate_ocp_on_cloud_daily_trino( self, azure_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags ): - """Populate the azure_openshift_daily trino table for OCP on Azure. + """Populate the managed_azure_openshift_daily trino table for OCP on Azure. Args: azure_provider_uuid (UUID) GCP source UUID. ocp_provider_uuid (UUID) OCP source UUID. diff --git a/koku/masu/database/gcp_report_db_accessor.py b/koku/masu/database/gcp_report_db_accessor.py index 97ae993bcb..a428ebfbde 100644 --- a/koku/masu/database/gcp_report_db_accessor.py +++ b/koku/masu/database/gcp_report_db_accessor.py @@ -589,7 +589,7 @@ def verify_populate_ocp_on_cloud_daily_trino(self, verification_params): def populate_ocp_on_cloud_daily_trino( self, gcp_provider_uuid, openshift_provider_uuid, start_date, end_date, matched_tags ): - """Populate the gcp_openshift_daily trino table for OCP on Azure. + """Populate the managed_gcp_openshift_daily trino table for OCP on GCP. Args: gcp_provider_uuid (UUID) GCP source UUID. ocp_provider_uuid (UUID) OCP source UUID. diff --git a/koku/masu/processor/tasks.py b/koku/masu/processor/tasks.py index ccd3c432cc..8f344d2e87 100644 --- a/koku/masu/processor/tasks.py +++ b/koku/masu/processor/tasks.py @@ -1255,7 +1255,7 @@ def validate_daily_data(schema, start_date, end_date, provider_uuid, ocp_on_clou @celery_app.task(name="masu.processor.tasks.process_openshift_on_cloud_trino", queue=SummaryQueue.DEFAULT, bind=True) def process_openshift_on_cloud_trino( - self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id + self, reports_to_summarize, provider_type, schema_name, provider_uuid, tracing_id, masu_api_trigger=False ): """Process OCP on Cloud data into managed tables for summary""" reports_deduplicated = deduplicate_summary_reports(reports_to_summarize, manifest_list=[]) @@ -1275,7 +1275,7 @@ def process_openshift_on_cloud_trino( with ReportManifestDBAccessor() as manifest_accesor: tracing_id = report.get("tracing_id", report.get("manifest_uuid", "no-tracing-id")) - if not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")): + if not masu_api_trigger and not manifest_accesor.manifest_ready_for_summary(report.get("manifest_id")): LOG.info(log_json(tracing_id, msg="manifest not ready for summary", context=report)) continue diff --git a/koku/masu/test/api/test_process_openshift_on_cloud.py b/koku/masu/test/api/test_process_openshift_on_cloud.py index b4d2edf35c..41b11174c9 100644 --- a/koku/masu/test/api/test_process_openshift_on_cloud.py +++ b/koku/masu/test/api/test_process_openshift_on_cloud.py @@ -139,3 +139,27 @@ def test_get_process_daily_openshift_on_cloud(self, mock_process, _): self.assertEqual(response.status_code, 200) self.assertIn(expected_key, body) mock_process.s.return_value.apply_async.assert_called() + + @patch("masu.api.process_openshift_on_cloud.process_openshift_on_cloud_trino") + @patch("masu.api.process_openshift_on_cloud.is_managed_ocp_cloud_processing_enabled") + @patch("koku.middleware.MASU") + def test_get_process_daily_openshift_on_cloud_trino(self, middleware, unleash, mock_task): + """Test the GET report_data endpoint for a provider that is daily partitioned.""" + unleash.return_value = True + middleware.return_value = True + dh = DateHelper() + start_date = str(dh.this_month_start.date()) + end_date = str(dh.this_month_end.date()) + params = { + "schema": self.schema, + "provider_uuid": self.gcp_provider_uuid, + "start_date": start_date, + "end_date": end_date, + } + expected_key = "process_openshift_on_cloud Task IDs" + response = self.client.get(reverse("process_openshift_on_cloud"), params) + body = response.json() + + self.assertEqual(response.status_code, 200) + self.assertIn(expected_key, body) + mock_task.s.return_value.apply_async.assert_called()