From 5b2945dc1c6aeef5a496398e044a5d8c291bca92 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Wed, 20 Nov 2024 14:10:51 +0100 Subject: [PATCH 1/5] Implement safeguard against problematic meta keys --- ixmp4/data/db/meta/repository.py | 5 +++++ tests/data/test_meta.py | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ixmp4/data/db/meta/repository.py b/ixmp4/data/db/meta/repository.py index 01b160b8..344cbd79 100644 --- a/ixmp4/data/db/meta/repository.py +++ b/ixmp4/data/db/meta/repository.py @@ -7,6 +7,7 @@ from ixmp4 import db from ixmp4.core.decorators import check_types +from ixmp4.core.exceptions import InvalidRunMeta from ixmp4.data import abstract from ixmp4.data.auth.decorators import guard from ixmp4.data.db.model import Model @@ -16,6 +17,8 @@ from .. import base from .model import RunMetaEntry +ILLEGAL_META_KEYS = {"model", "scenario", "id", "version", "is_default"} + class RemoveRunMetaEntryFrameSchema(pa.DataFrameModel): key: Series[pa.String] = pa.Field(coerce=True) @@ -60,6 +63,8 @@ def add( default_only=False, ) + if key in ILLEGAL_META_KEYS: + raise InvalidRunMeta("Illegal meta key: " + key) entry = RunMetaEntry(run__id=run__id, key=key, value=value) self.session.add(entry) return entry diff --git a/tests/data/test_meta.py b/tests/data/test_meta.py index 89ee08a6..f753af36 100644 --- a/tests/data/test_meta.py +++ b/tests/data/test_meta.py @@ -2,7 +2,7 @@ import pytest import ixmp4 -from ixmp4.core.exceptions import SchemaError +from ixmp4.core.exceptions import InvalidRunMeta, SchemaError from ixmp4.data.abstract.meta import RunMetaEntry from ..utils import assert_unordered_equality @@ -37,6 +37,11 @@ def test_create_get_entry(self, platform: ixmp4.Platform): assert entry.value == value assert entry.type == type + def test_illegal_key(self, platform: ixmp4.Platform): + run = platform.runs.create("Model", "Scenario") + with pytest.raises(InvalidRunMeta): + platform.backend.meta.create(run.id, "version", "foo") + def test_entry_unique(self, platform: ixmp4.Platform): run = platform.runs.create("Model", "Scenario") platform.backend.meta.create(run.id, "Key", "Value") From 6b139ded2322d3b12161ab48e303de08ccfda9a9 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 22 Nov 2024 13:59:20 +0100 Subject: [PATCH 2/5] Add check for illegal meta-keys to `bulk_upsert()` --- ixmp4/data/db/meta/repository.py | 4 ++++ tests/data/test_meta.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ixmp4/data/db/meta/repository.py b/ixmp4/data/db/meta/repository.py index 344cbd79..c64fee08 100644 --- a/ixmp4/data/db/meta/repository.py +++ b/ixmp4/data/db/meta/repository.py @@ -1,5 +1,6 @@ from typing import Union +import numpy as np import pandas as pd import pandera as pa from pandera.typing import DataFrame, Series @@ -210,6 +211,9 @@ def map_value_column(df: pd.DataFrame): @check_types @guard("edit") def bulk_upsert(self, df: DataFrame[AddRunMetaEntryFrameSchema]) -> None: + if illegal_keys:= ( set(np.unique(df.key.values)) & ILLEGAL_META_KEYS ): + raise InvalidRunMeta("Illegal meta key(s): " + ", ".join(illegal_keys)) + self.check_df_access(df) df["type"] = df["value"].map(type).map(RunMetaEntry.Type.from_pytype) diff --git a/tests/data/test_meta.py b/tests/data/test_meta.py index f753af36..099f499a 100644 --- a/tests/data/test_meta.py +++ b/tests/data/test_meta.py @@ -19,6 +19,8 @@ columns=["id", "key", "type", "value"], ) +TEST_ILLEGAL_META_KEYS = {"model", "scenario", "id", "version", "is_default"} + class TestDataMeta: def test_create_get_entry(self, platform: ixmp4.Platform): @@ -39,8 +41,15 @@ def test_create_get_entry(self, platform: ixmp4.Platform): def test_illegal_key(self, platform: ixmp4.Platform): run = platform.runs.create("Model", "Scenario") - with pytest.raises(InvalidRunMeta): - platform.backend.meta.create(run.id, "version", "foo") + for key in TEST_ILLEGAL_META_KEYS: + with pytest.raises(InvalidRunMeta, match="Illegal meta key: " + key): + platform.backend.meta.create(run.id, key, "foo") + + df = pd.DataFrame( + {"run__id": [run.id] * 2, "key": [key, "foo"], "value": ["bar", "baz"]}, + ) + with pytest.raises(InvalidRunMeta, match=r"Illegal meta key\(s\): " + key): + platform.backend.meta.bulk_upsert(df) def test_entry_unique(self, platform: ixmp4.Platform): run = platform.runs.create("Model", "Scenario") From f0a74d94fedeb15fbe9e0d49bd143d7b50fc3459 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 22 Nov 2024 14:02:41 +0100 Subject: [PATCH 3/5] Make ruff --- ixmp4/data/db/meta/repository.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ixmp4/data/db/meta/repository.py b/ixmp4/data/db/meta/repository.py index c64fee08..47a7bc16 100644 --- a/ixmp4/data/db/meta/repository.py +++ b/ixmp4/data/db/meta/repository.py @@ -211,7 +211,7 @@ def map_value_column(df: pd.DataFrame): @check_types @guard("edit") def bulk_upsert(self, df: DataFrame[AddRunMetaEntryFrameSchema]) -> None: - if illegal_keys:= ( set(np.unique(df.key.values)) & ILLEGAL_META_KEYS ): + if illegal_keys := (set(np.unique(df.key.values)) & ILLEGAL_META_KEYS): raise InvalidRunMeta("Illegal meta key(s): " + ", ".join(illegal_keys)) self.check_df_access(df) From e83b008cc1cb28fa1073b8359aa4791f522cae36 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 22 Nov 2024 14:26:10 +0100 Subject: [PATCH 4/5] Make mypy, simplify check --- ixmp4/data/db/meta/repository.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ixmp4/data/db/meta/repository.py b/ixmp4/data/db/meta/repository.py index 47a7bc16..d07c0e13 100644 --- a/ixmp4/data/db/meta/repository.py +++ b/ixmp4/data/db/meta/repository.py @@ -1,6 +1,5 @@ from typing import Union -import numpy as np import pandas as pd import pandera as pa from pandera.typing import DataFrame, Series @@ -211,7 +210,8 @@ def map_value_column(df: pd.DataFrame): @check_types @guard("edit") def bulk_upsert(self, df: DataFrame[AddRunMetaEntryFrameSchema]) -> None: - if illegal_keys := (set(np.unique(df.key.values)) & ILLEGAL_META_KEYS): + + if illegal_keys := (set(df.key.values) & ILLEGAL_META_KEYS): raise InvalidRunMeta("Illegal meta key(s): " + ", ".join(illegal_keys)) self.check_df_access(df) From 4be614919a260d8b32b28c1fa46075f873540bb8 Mon Sep 17 00:00:00 2001 From: Daniel Huppmann Date: Fri, 22 Nov 2024 14:28:07 +0100 Subject: [PATCH 5/5] Make ruff again --- ixmp4/data/db/meta/repository.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ixmp4/data/db/meta/repository.py b/ixmp4/data/db/meta/repository.py index d07c0e13..09318d3c 100644 --- a/ixmp4/data/db/meta/repository.py +++ b/ixmp4/data/db/meta/repository.py @@ -210,7 +210,6 @@ def map_value_column(df: pd.DataFrame): @check_types @guard("edit") def bulk_upsert(self, df: DataFrame[AddRunMetaEntryFrameSchema]) -> None: - if illegal_keys := (set(df.key.values) & ILLEGAL_META_KEYS): raise InvalidRunMeta("Illegal meta key(s): " + ", ".join(illegal_keys))