From 1cb61b83b076ab1b8b6732fd2eb4dea22ce39d44 Mon Sep 17 00:00:00 2001 From: Corey Goodfred Date: Tue, 30 Jan 2024 14:02:19 -0500 Subject: [PATCH] [COST-4572] Add Django management command to cleanup AWS bills and a job for running it in the clowdapp (#4875) * use cascade_delete to cleanup null payer account bills * change bill cleanup to a management command, add new job in clowdapp for running management commands * cleanup and iterate over dates first then providers * make whole command configurable with a base command of an echo * initialize unleash client --------- Co-authored-by: maskarb --- deploy/clowdapp.yaml | 62 +++++++++++ deploy/kustomize/base/base.yaml | 71 +++++++++++- .../commands/aws_null_bill_cleanup.py | 101 ++++++++++++++++++ 3 files changed, 233 insertions(+), 1 deletion(-) create mode 100644 koku/masu/management/commands/aws_null_bill_cleanup.py diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index efb412c3ad..52a70cd04e 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -4207,6 +4207,29 @@ objects: requests: cpu: ${KOKU_MIGRATIONS_CPU_REQUEST} memory: ${KOKU_MIGRATIONS_MEMORY_REQUEST} + - name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} + podSpec: + args: + - /bin/bash + - -c + - ${MGMT_COMMAND} + env: + - name: CLOWDER_ENABLED + value: ${CLOWDER_ENABLED} + - name: DEVELOPMENT + value: ${DEVELOPMENT} + - name: MGMT_IMAGE + value: ${MGMT_IMAGE} + - name: MGMT_IMAGE_TAG + value: ${MGMT_IMAGE_TAG} + image: ${MGMT_IMAGE}:${MGMT_IMAGE_TAG} + resources: + limits: + cpu: ${KOKU_MGMT_CPU_LIMIT} + memory: ${KOKU_MGMT_MEMORY_LIMIT} + requests: + cpu: ${KOKU_MGMT_CPU_REQUEST} + memory: ${KOKU_MGMT_MEMORY_REQUEST} kafkaTopics: - topicName: platform.sources.event-stream - topicName: platform.upload.announce @@ -4229,6 +4252,14 @@ objects: appName: koku jobs: - db-migrate-cji-${DBM_IMAGE_TAG}-${DBM_INVOCATION} +- apiVersion: cloud.redhat.com/v1alpha1 + kind: ClowdJobInvocation + metadata: + name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} + spec: + appName: koku + jobs: + - management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} - apiVersion: v1 data: django-secret-key: ${SECRET_KEY} @@ -4394,6 +4425,37 @@ parameters: - name: TENANT_MULTIPROCESSING_CHUNKS required: true value: "2" +- description: Management Command Image Tag + name: MGMT_IMAGE_TAG + required: true + value: latest +- description: Management Command Image + name: MGMT_IMAGE + required: true + value: quay.io/cloudservices/koku +- description: Management Command Invocation Iterator + name: MGMT_INVOCATION + required: true + value: "00" +- description: Management Command + name: MGMT_COMMAND + value: echo 'No Command to Run' +- displayName: Memory Request + name: KOKU_MGMT_MEMORY_REQUEST + required: true + value: 20Mi +- displayName: Memory Limit + name: KOKU_MGMT_MEMORY_LIMIT + required: true + value: 20Mi +- displayName: CPU Request + name: KOKU_MGMT_CPU_REQUEST + required: true + value: 10m +- displayName: CPU Limit + name: KOKU_MGMT_CPU_LIMIT + required: true + value: 10m - displayName: App domain name: APP_DOMAIN value: project-koku.com diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml index aa9cf1d1f3..99018da298 100644 --- a/deploy/kustomize/base/base.yaml +++ b/deploy/kustomize/base/base.yaml @@ -89,7 +89,32 @@ objects: value: ${TENANT_MULTIPROCESSING_MAX_PROCESSES} - name: TENANT_MULTIPROCESSING_CHUNKS value: ${TENANT_MULTIPROCESSING_CHUNKS} - + # ==================================================== + # koku Management Command Job + # ==================================================== + - name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} + podSpec: + image: ${MGMT_IMAGE}:${MGMT_IMAGE_TAG} + resources: + limits: + cpu: ${KOKU_MGMT_CPU_LIMIT} + memory: ${KOKU_MGMT_MEMORY_LIMIT} + requests: + cpu: ${KOKU_MGMT_CPU_REQUEST} + memory: ${KOKU_MGMT_MEMORY_REQUEST} + args: + - /bin/bash + - -c + - ${MGMT_COMMAND} + env: + - name: CLOWDER_ENABLED + value: ${CLOWDER_ENABLED} + - name: DEVELOPMENT + value: ${DEVELOPMENT} + - name: MGMT_IMAGE + value: ${MGMT_IMAGE} + - name: MGMT_IMAGE_TAG + value: ${MGMT_IMAGE_TAG} # The bulk of your App. This is where your running apps will live deployments: - @@ -106,6 +131,18 @@ objects: jobs: - db-migrate-cji-${DBM_IMAGE_TAG}-${DBM_INVOCATION} +# ==================================================== +# koku Management CJI +# ==================================================== +- apiVersion: cloud.redhat.com/v1alpha1 + kind: ClowdJobInvocation + metadata: + name: management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} + spec: + appName: koku + jobs: + - management-command-cji-${MGMT_IMAGE_TAG}-${MGMT_INVOCATION} + - apiVersion: v1 kind: Secret # For ephemeral/local environment only metadata: @@ -279,6 +316,38 @@ parameters: required: True value: "2" +- description: Management Command Image Tag + name: MGMT_IMAGE_TAG + required: true + value: latest +- description: Management Command Image + name: MGMT_IMAGE + required: true + value: quay.io/cloudservices/koku +- description: Management Command Invocation Iterator + name: MGMT_INVOCATION + required: true + value: "00" +- description: Management Command + name: MGMT_COMMAND + value: "echo 'No Command to Run'" +- displayName: Memory Request + name: KOKU_MGMT_MEMORY_REQUEST + required: true + value: 20Mi +- displayName: Memory Limit + name: KOKU_MGMT_MEMORY_LIMIT + required: true + value: 20Mi +- displayName: CPU Request + name: KOKU_MGMT_CPU_REQUEST + required: true + value: "10m" +- displayName: CPU Limit + name: KOKU_MGMT_CPU_LIMIT + required: true + value: "10m" + - displayName: App domain name: APP_DOMAIN value: project-koku.com diff --git a/koku/masu/management/commands/aws_null_bill_cleanup.py b/koku/masu/management/commands/aws_null_bill_cleanup.py new file mode 100644 index 0000000000..de6fd425a6 --- /dev/null +++ b/koku/masu/management/commands/aws_null_bill_cleanup.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import argparse +import logging +from datetime import datetime +from typing import Any + +from dateutil.relativedelta import relativedelta +from django.conf import settings +from django.core.management.base import BaseCommand +from django_tenants.utils import schema_context + +from api.provider.models import Provider +from koku.database import cascade_delete +from koku.feature_flags import UNLEASH_CLIENT +from masu.processor import is_customer_large +from masu.processor.tasks import PRIORITY_QUEUE +from masu.processor.tasks import PRIORITY_QUEUE_XL +from masu.processor.tasks import update_summary_tables +from reporting.models import AWSCostEntryBill + +LOG = logging.getLogger(__name__) +DATE_FORMAT = "%Y-%m-%d" +DATETIMES = ( + datetime(2024, 1, 1, tzinfo=settings.UTC), + datetime(2023, 12, 1, tzinfo=settings.UTC), + datetime(2023, 11, 1, tzinfo=settings.UTC), + datetime(2023, 10, 1, tzinfo=settings.UTC), + datetime(2023, 9, 1, tzinfo=settings.UTC), +) + + +class Command(BaseCommand): + help = "Delete AWS Bills and with a NULL payer_account_id and all FK references for a given month." + + def add_arguments(self, parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--delete", + action="store_true", + default=False, + help="Actually delete the cost entry bills.", + ) + + def handle(self, *args: Any, delete: bool, **kwargs: Any) -> None: + LOG.info("Initializing UNLEASH_CLIENT for bill cleanup.") + UNLEASH_CLIENT.initialize_client() + if delete: + LOG.info(msg="DELETING BILLS (--delete passed)") + else: + LOG.info(msg="In dry run mode (--delete not passed)") + + total_cleaned_bills = cleanup_aws_bills(delete) + + if delete: + LOG.info(f"{total_cleaned_bills} bills deleted.") + else: + LOG.info(f"DRY RUN: {total_cleaned_bills} bills would be deleted.") + + +def cleanup_aws_bills(delete: bool) -> int: + """Deletes AWS Bills with a null payer account ID.""" + total_cleaned_bills = 0 + providers = Provider.objects.filter(type__in=[Provider.PROVIDER_AWS, Provider.PROVIDER_AWS_LOCAL]) + for start_date in DATETIMES: + end_date = start_date + relativedelta(day=31) + + for prov in providers: + schema = prov.customer.schema_name + provider_uuid = prov.uuid + + with schema_context(schema): + if bills := AWSCostEntryBill.objects.filter( + provider_id=provider_uuid, + payer_account_id=None, + billing_period_start=start_date, + ): + queue_name = PRIORITY_QUEUE_XL if is_customer_large(schema) else PRIORITY_QUEUE + total_cleaned_bills += len(bills) + if delete: + formatted_start = start_date.strftime(DATE_FORMAT) + formatted_end = end_date.strftime(DATE_FORMAT) + cascade_delete(bills.query.model, bills) + async_result = update_summary_tables.s( + schema, + prov.type, + provider_uuid, + formatted_start, + end_date=formatted_end, + queue_name=queue_name, + ocp_on_cloud=True, + ).apply_async(queue=queue_name) + LOG.info( + f"Deletes completed and summary triggered for provider {provider_uuid} " + f"with start {formatted_start} and end {formatted_end}, task_id: {async_result.id}" + ) + + else: + bill_ids = [bill.id for bill in bills] + LOG.info(f"bills {bill_ids} would be deleted for provider: {provider_uuid}") + + return total_cleaned_bills