diff --git a/bin/produce_report.sh b/bin/produce_report.sh index dfc5d91..dbe7af1 100755 --- a/bin/produce_report.sh +++ b/bin/produce_report.sh @@ -3,4 +3,5 @@ python -m openshift_metrics.merge /data/*.json \ --invoice-file /tmp/invoice.csv \ --pod-report-file /tmp/pod-report.csv \ - --upload-to-s3 + --upload-to-s3 \ + --use-nerc-rates diff --git a/openshift_metrics/merge.py b/openshift_metrics/merge.py index 8bc4e8c..389924f 100644 --- a/openshift_metrics/merge.py +++ b/openshift_metrics/merge.py @@ -6,8 +6,10 @@ from datetime import datetime, UTC import json from typing import Tuple +from decimal import Decimal +from nerc_rates import load_from_url -from openshift_metrics import utils +from openshift_metrics import utils, invoice from openshift_metrics.metrics_processor import MetricsProcessor def compare_dates(date_str1, date_str2): @@ -53,6 +55,15 @@ def main(): nargs="*", help="List of timestamp ranges in UTC to ignore in the format 'YYYY-MM-DDTHH:MM:SS,YYYY-MM-DDTHH:MM:SS'" ) + parser.add_argument( + "--use-nerc-rates", + action="store_true", + help="Use rates from the nerc-rates repo", + ) + parser.add_argument("--rate-cpu-su", type=Decimal) + parser.add_argument("--rate-gpu-v100-su", type=Decimal) + parser.add_argument("--rate-gpu-a100sxm4-su", type=Decimal) + parser.add_argument("--rate-gpu-a100-su", type=Decimal) args = parser.parse_args() files = args.files @@ -91,6 +102,22 @@ def main(): report_month = datetime.strftime(report_start_date, "%Y-%m") + if args.use_nerc_rates: + nerc_rates = load_from_url() + rates = invoice.Rates( + cpu=Decimal(nerc_rates.get_value_at("CPU SU Rate", report_month)), + gpu_a100=Decimal(nerc_rates.get_value_at("GPUA100 SU Rate", report_month)), + gpu_a100sxm4=Decimal(nerc_rates.get_value_at("GPUA100SXM4 SU Rate", report_month)), + gpu_v100=Decimal(nerc_rates.get_value_at("GPUV100 SU Rate", report_month)), + ) + else: + rates = invoice.Rates( + cpu=Decimal(args.rate_cpu_su), + gpu_a100=Decimal(args.rate_gpu_a100_su), + gpu_a100sxm4=Decimal(args.rate_gpu_a100sxm4_su), + gpu_v100=Decimal(args.rate_gpu_v100_su) + ) + if args.invoice_file: invoice_file = args.invoice_file else: @@ -109,10 +136,11 @@ def main(): ["cpu_request", "memory_request", "gpu_request", "gpu_type"] ) utils.write_metrics_by_namespace( - condensed_metrics_dict, - invoice_file, - report_month, - ignore_hours, + condensed_metrics_dict=condensed_metrics_dict, + file_name=invoice_file, + report_month=report_month, + rates=rates, + ignore_hours=ignore_hours, ) utils.write_metrics_by_pod(condensed_metrics_dict, pod_report_file, ignore_hours) diff --git a/openshift_metrics/tests/test_utils.py b/openshift_metrics/tests/test_utils.py index 095c795..e4fc2fb 100644 --- a/openshift_metrics/tests/test_utils.py +++ b/openshift_metrics/tests/test_utils.py @@ -13,11 +13,19 @@ from requests.exceptions import ConnectionError import tempfile from unittest import TestCase, mock +from decimal import Decimal from openshift_metrics import utils, invoice import os from datetime import datetime, UTC +RATES = invoice.Rates( + cpu = Decimal("0.013"), + gpu_a100sxm4 = Decimal("2.078"), + gpu_a100 = Decimal("1.803"), + gpu_v100 = Decimal("1.214") + ) + class TestGetNamespaceAnnotations(TestCase): @mock.patch('openshift_metrics.utils.requests.post') @@ -241,7 +249,12 @@ def test_write_metrics_log(self, mock_gna): "2023-01,namespace2,namespace2,PI2,,,,,48,OpenShift GPUA100SXM4,2.078,99.74\n") with tempfile.NamedTemporaryFile(mode="w+") as tmp: - utils.write_metrics_by_namespace(test_metrics_dict, tmp.name, "2023-01") + utils.write_metrics_by_namespace( + condensed_metrics_dict=test_metrics_dict, + file_name=tmp.name, + report_month="2023-01", + rates=RATES + ) self.assertEqual(tmp.read(), expected_output) @@ -286,7 +299,12 @@ def test_write_metrics_by_namespace_decimal(self, mock_gna): "2023-01,namespace1,namespace1,PI1,,,,76,35,OpenShift CPU,0.013,0.46\n") with tempfile.NamedTemporaryFile(mode="w+") as tmp: - utils.write_metrics_by_namespace(test_metrics_dict, tmp.name, "2023-01") + utils.write_metrics_by_namespace( + condensed_metrics_dict=test_metrics_dict, + file_name=tmp.name, + report_month="2023-01", + rates=RATES + ) self.assertEqual(tmp.read(), expected_output) @@ -294,6 +312,9 @@ class TestWriteMetricsWithIgnoreHours(TestCase): def setUp(self): """Creates a test dictionary with condensed data that can be used to test WriteMetricsByPod and WriteMetricsByNamespace""" start_dt = int(datetime.fromisoformat("2024-04-10T11:00:00Z").timestamp()) + + + self.ignore_times = [ ( datetime(2024, 4, 9, 11, 0, 0, tzinfo=UTC), @@ -371,7 +392,11 @@ def test_write_metrics_by_namespace_with_ignore_hours(self, mock_gna): with tempfile.NamedTemporaryFile(mode="w+") as tmp: utils.write_metrics_by_namespace( - self.test_metrics_dict, tmp.name, "2023-01", self.ignore_times + condensed_metrics_dict=self.test_metrics_dict, + file_name=tmp.name, + report_month="2023-01", + rates=RATES, + ignore_hours=self.ignore_times ) self.assertEqual(tmp.read(), expected_output) diff --git a/openshift_metrics/utils.py b/openshift_metrics/utils.py index 6a10c8d..58e2cb2 100755 --- a/openshift_metrics/utils.py +++ b/openshift_metrics/utils.py @@ -110,7 +110,7 @@ def csv_writer(rows, file_name): csvwriter.writerows(rows) -def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month, ignore_hours=None): +def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month, rates, ignore_hours=None): """ Process metrics dictionary to aggregate usage by namespace and then write that to a file """ @@ -134,14 +134,6 @@ def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month, rows.append(headers) - # TODO: the caller will pass in the rates as an argument - rates = invoice.Rates( - cpu = Decimal("0.013"), - gpu_a100 = Decimal("1.803"), - gpu_a100sxm4 = Decimal("2.078"), - gpu_v100 = Decimal("1.214") - ) - for namespace, pods in condensed_metrics_dict.items(): namespace_annotation_dict = namespace_annotations.get(namespace, {}) cf_pi = namespace_annotation_dict.get("cf_pi") diff --git a/requirements.txt b/requirements.txt index 835cb27..71d4546 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ requests>=2.18.4 boto3>=1.34.40 +git+https://github.com/CCI-MOC/nerc-rates#egg=nerc_rates