Use nerc-rates for billing

This add the necessary code to get rates from the nerc-rates repo or specify
them from the command line. The produce report script is updated to use
nerc-rates.

Addtionaly the tests were updated since the function `write_metrics_by_namespace`
now takes an additional argument. All the calls to that function have been
updated to keep things more readable.

bin/produce_report.sh

@@ -3,4 +3,5 @@
 python -m openshift_metrics.merge /data/*.json \
     --invoice-file /tmp/invoice.csv \
     --pod-report-file /tmp/pod-report.csv \
-    --upload-to-s3
+    --upload-to-s3 \
+    --use-nerc-rates

openshift_metrics/merge.py

@@ -6,8 +6,10 @@
 from datetime import datetime, UTC
 import json
 from typing import Tuple
+from decimal import Decimal
+from nerc_rates import load_from_url
 
-from openshift_metrics import utils
+from openshift_metrics import utils, invoice
 from openshift_metrics.metrics_processor import MetricsProcessor
 
 def compare_dates(date_str1, date_str2):
@@ -53,6 +55,15 @@ def main():
         nargs="*",
         help="List of timestamp ranges in UTC to ignore in the format 'YYYY-MM-DDTHH:MM:SS,YYYY-MM-DDTHH:MM:SS'"
     )
+    parser.add_argument(
+        "--use-nerc-rates",
+        action="store_true",
+        help="Use rates from the nerc-rates repo",
+    )
+    parser.add_argument("--rate-cpu-su", type=Decimal)
+    parser.add_argument("--rate-gpu-v100-su", type=Decimal)
+    parser.add_argument("--rate-gpu-a100sxm4-su", type=Decimal)
+    parser.add_argument("--rate-gpu-a100-su", type=Decimal)
 
     args = parser.parse_args()
     files = args.files
@@ -91,6 +102,22 @@ def main():
 
     report_month = datetime.strftime(report_start_date, "%Y-%m")
 
+    if args.use_nerc_rates:
+        nerc_rates = load_from_url()
+        rates = invoice.Rates(
+            cpu=Decimal(nerc_rates.get_value_at("CPU SU Rate", report_month)),
+            gpu_a100=Decimal(nerc_rates.get_value_at("GPUA100 SU Rate", report_month)),
+            gpu_a100sxm4=Decimal(nerc_rates.get_value_at("GPUA100SXM4 SU Rate", report_month)),
+            gpu_v100=Decimal(nerc_rates.get_value_at("GPUV100 SU Rate", report_month)),
+        )
+    else:
+        rates = invoice.Rates(
+            cpu=Decimal(args.rate_cpu_su),
+            gpu_a100=Decimal(args.rate_gpu_a100_su),
+            gpu_a100sxm4=Decimal(args.rate_gpu_a100sxm4_su),
+            gpu_v100=Decimal(args.rate_gpu_v100_su)
+        )
+
     if args.invoice_file:
         invoice_file = args.invoice_file
     else:
@@ -109,10 +136,11 @@ def main():
         ["cpu_request", "memory_request", "gpu_request", "gpu_type"]
     )
     utils.write_metrics_by_namespace(
-        condensed_metrics_dict,
-        invoice_file,
-        report_month,
-        ignore_hours,
+        condensed_metrics_dict=condensed_metrics_dict,
+        file_name=invoice_file,
+        report_month=report_month,
+        rates=rates,
+        ignore_hours=ignore_hours,
     )
     utils.write_metrics_by_pod(condensed_metrics_dict, pod_report_file, ignore_hours)
 

openshift_metrics/tests/test_utils.py

@@ -13,11 +13,19 @@
 from requests.exceptions import ConnectionError
 import tempfile
 from unittest import TestCase, mock
+from decimal import Decimal
 
 from openshift_metrics import utils, invoice
 import os
 from datetime import datetime, UTC
 
+RATES = invoice.Rates(
+    cpu = Decimal("0.013"),
+    gpu_a100sxm4 = Decimal("2.078"),
+    gpu_a100 = Decimal("1.803"),
+    gpu_v100 = Decimal("1.214")
+    )
+
 class TestGetNamespaceAnnotations(TestCase):
 
     @mock.patch('openshift_metrics.utils.requests.post')
@@ -241,7 +249,12 @@ def test_write_metrics_log(self, mock_gna):
                             "2023-01,namespace2,namespace2,PI2,,,,,48,OpenShift GPUA100SXM4,2.078,99.74\n")
 
         with tempfile.NamedTemporaryFile(mode="w+") as tmp:
-            utils.write_metrics_by_namespace(test_metrics_dict, tmp.name, "2023-01")
+            utils.write_metrics_by_namespace(
+                condensed_metrics_dict=test_metrics_dict,
+                file_name=tmp.name,
+                report_month="2023-01",
+                rates=RATES
+                )
             self.assertEqual(tmp.read(), expected_output)
 
 
@@ -286,14 +299,22 @@ def test_write_metrics_by_namespace_decimal(self, mock_gna):
                             "2023-01,namespace1,namespace1,PI1,,,,76,35,OpenShift CPU,0.013,0.46\n")
 
         with tempfile.NamedTemporaryFile(mode="w+") as tmp:
-            utils.write_metrics_by_namespace(test_metrics_dict, tmp.name, "2023-01")
+            utils.write_metrics_by_namespace(
+                condensed_metrics_dict=test_metrics_dict,
+                file_name=tmp.name,
+                report_month="2023-01",
+                rates=RATES
+                )
             self.assertEqual(tmp.read(), expected_output)
 
 
 class TestWriteMetricsWithIgnoreHours(TestCase):
     def setUp(self):
         """Creates a test dictionary with condensed data that can be used to test WriteMetricsByPod and WriteMetricsByNamespace"""
         start_dt = int(datetime.fromisoformat("2024-04-10T11:00:00Z").timestamp())
+
+
+
         self.ignore_times = [
             (
                 datetime(2024, 4, 9, 11, 0, 0, tzinfo=UTC),
@@ -371,7 +392,11 @@ def test_write_metrics_by_namespace_with_ignore_hours(self, mock_gna):
 
         with tempfile.NamedTemporaryFile(mode="w+") as tmp:
             utils.write_metrics_by_namespace(
-                self.test_metrics_dict, tmp.name, "2023-01", self.ignore_times
+                condensed_metrics_dict=self.test_metrics_dict,
+                file_name=tmp.name,
+                report_month="2023-01",
+                rates=RATES,
+                ignore_hours=self.ignore_times
             )
             self.assertEqual(tmp.read(), expected_output)
 

openshift_metrics/utils.py

@@ -110,7 +110,7 @@ def csv_writer(rows, file_name):
         csvwriter.writerows(rows)
 
 
-def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month, ignore_hours=None):
+def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month, rates, ignore_hours=None):
     """
     Process metrics dictionary to aggregate usage by namespace and then write that to a file
     """
@@ -134,14 +134,6 @@ def write_metrics_by_namespace(condensed_metrics_dict, file_name, report_month,
 
     rows.append(headers)
 
-    # TODO: the caller will pass in the rates as an argument
-    rates = invoice.Rates(
-        cpu = Decimal("0.013"),
-        gpu_a100 = Decimal("1.803"),
-        gpu_a100sxm4 = Decimal("2.078"),
-        gpu_v100 = Decimal("1.214")
-    )
-
     for namespace, pods in condensed_metrics_dict.items():
         namespace_annotation_dict = namespace_annotations.get(namespace, {})
         cf_pi = namespace_annotation_dict.get("cf_pi")

requirements.txt

@@ -1,2 +1,3 @@
 requests>=2.18.4
 boto3>=1.34.40
+git+https://github.com/CCI-MOC/nerc-rates#egg=nerc_rates