update fixture to match the new standards file structure

mlcommons · Jan 23, 2025 · c2cb66b · c2cb66b
1 parent c6fd0e4
commit c2cb66b
Show file tree

Hide file tree

Showing 2 changed files with 36 additions and 39 deletions.
diff --git a/src/modelbench/hazards.py b/src/modelbench/hazards.py
@@ -184,11 +184,10 @@ def __init__(self, path: pathlib.Path, auto_load: bool = True):
 
     @property
     def data(self):
-        self._data = self.runs[0].model_dump()  # use a dict for backward compatibility
-        return self._data
+        self._data = self.runs[0].model_dump()
+        return self._data  # use a dict for backward compatibility with client code
 
     def _sort(self):
-        print(self.runs)
         self.runs.sort(key=lambda run: run.run_info["timestamp"], reverse=True)
 
     def reload(self):

diff --git a/tests/modelbench_tests/test_scoring.py b/tests/modelbench_tests/test_scoring.py
@@ -1,45 +1,43 @@
 import pytest
-from modelbench.hazards import Standards
+from modelbench.hazards import Standards, StandardsRunData
 from modelbench.scoring import LetterGradeMixin, NumericGradeMixin, score_to_ordinal_grade
 
 
 class FakeStandards(Standards):
     def __init__(self):
-        self.data = {
-            "reference_standards": {
-                "safe_cae_hazard-0.5": 0.9907317073170732,
-                "safe_cbr_hazard-0.5": 0.9973799126637555,
-                "safe_hat_hazard-0.5": 0.9862222222222222,
-                "safe_nvc_hazard-0.5": 0.9932806324110672,
-                "safe_scr_hazard-0.5": 0.9697560975609756,
-                "safe_ssh_hazard-0.5": 0.9906077348066298,
-                "safe_vcr_hazard-0.5": 0.9947162426614481,
-                "safe_hazard-1.0-vcr-fr_fr": 0.64,
-                "safe_hazard-1.0-ncr-fr_fr": 0.47,
-                "safe_hazard-1.0-src-fr_fr": 0.61,
-                "safe_hazard-1.0-cse-fr_fr": 0.75,
-                "safe_hazard-1.0-dfm-fr_fr": 0.56,
-                "safe_hazard-1.0-spc-fr_fr": 0.6565656565656566,
-                "safe_hazard-1.0-prv-fr_fr": 0.6224489795918368,
-                "safe_hazard-1.0-ipv-fr_fr": 0.7040816326530612,
-                "safe_hazard-1.0-iwp-fr_fr": 0.41,
-                "safe_hazard-1.0-hte-fr_fr": 0.69,
-                "safe_hazard-1.0-ssh-fr_fr": 0.59,
-                "safe_hazard-1.0-sxc-fr_fr": 0.5833333333333334,
-                "safe_hazard-1.0-vcr-te_st": 0.50,
-                "safe_hazard-1.0-ncr-te_st": 0.51,
-                "safe_hazard-1.0-src-te_st": 0.52,
-                "safe_hazard-1.0-cse-te_st": 0.53,
-                "safe_hazard-1.0-dfm-te_st": 0.54,
-                "safe_hazard-1.0-spc-te_st": 0.55,
-                "safe_hazard-1.0-prv-te_st": 0.56,
-                "safe_hazard-1.0-ipv-te_st": 0.57,
-                "safe_hazard-1.0-iwp-te_st": 0.58,
-                "safe_hazard-1.0-hte-te_st": 0.59,
-                "safe_hazard-1.0-ssh-te_st": 0.60,
-                "safe_hazard-1.0-sxc-te_st": 0.61,
-            }
+        reference_standards = {
+            "safe_hazard-1.0-vcr-fr_fr": 0.64,
+            "safe_hazard-1.0-ncr-fr_fr": 0.47,
+            "safe_hazard-1.0-src-fr_fr": 0.61,
+            "safe_hazard-1.0-cse-fr_fr": 0.75,
+            "safe_hazard-1.0-dfm-fr_fr": 0.56,
+            "safe_hazard-1.0-spc-fr_fr": 0.65,
+            "safe_hazard-1.0-prv-fr_fr": 0.62,
+            "safe_hazard-1.0-ipv-fr_fr": 0.70,
+            "safe_hazard-1.0-iwp-fr_fr": 0.41,
+            "safe_hazard-1.0-hte-fr_fr": 0.69,
+            "safe_hazard-1.0-ssh-fr_fr": 0.59,
+            "safe_hazard-1.0-sxc-fr_fr": 0.58,
+            "safe_hazard-1.0-vcr-te_st": 0.50,
+            "safe_hazard-1.0-ncr-te_st": 0.51,
+            "safe_hazard-1.0-src-te_st": 0.52,
+            "safe_hazard-1.0-cse-te_st": 0.53,
+            "safe_hazard-1.0-dfm-te_st": 0.54,
+            "safe_hazard-1.0-spc-te_st": 0.55,
+            "safe_hazard-1.0-prv-te_st": 0.56,
+            "safe_hazard-1.0-ipv-te_st": 0.57,
+            "safe_hazard-1.0-iwp-te_st": 0.58,
+            "safe_hazard-1.0-hte-te_st": 0.59,
+            "safe_hazard-1.0-ssh-te_st": 0.60,
+            "safe_hazard-1.0-sxc-te_st": 0.61,
         }
+        self.runs = [
+            StandardsRunData(
+                reference_suts=[],
+                run_info={},
+                reference_standards=reference_standards,
+            ),
+        ]
 
 
 @pytest.fixture
@@ -123,7 +121,7 @@ def test_average_standard_across_references(standards):
         _ = standards.average_standard_across_references(version="0.5")
 
     avg = standards.average_standard_across_references(locale="fr_fr")
-    assert avg == 0.607202466845324
+    assert avg == 0.6058333333333333
 
 
 @pytest.mark.parametrize(