Simpler Test Data (#110)

* use cast instead of type: ignore * add updateinfo migration * rework tests to use fixtures and cli option * adjust core tests * adjust data layer tests * adjust benchmark tests * remove create_filter_test_data * remove old code/comments * create deployment instead of trying to build docker img * adjust pytest wf * add medium test data * split check_dsn and make_engine * use medium test data where it makes sense * make postgres_dsn its own cli option * run one backend per test run * fix typo * remove copy disclaimer * backends -> backend * add type:ignore * change url to localhost * fix conflict resolution mixstakes * fix more conf. res. mistakes * try to start psotgres only when it is needed * fix gha syntax * group tests into groups of two * group default matrix as well * all platforms at once seems to be most efficient * make iamc tests a little bit faster * use explicit core import * remove run.set_as_default() * remove more set_as_default * remove runs.set_as_default_version from opt data layer tests * remove extra comment * missed a correct conflict res. * fix broken psql tests * remove extra comment * fix typo * simplify conftest.py * forgot sqlite_platform * more performance refactoring * reset is now called in conftest.py * reset after the test instead of before? * rev reset after * split up reset() * split reset for all fixtures * refactor reset() * more reset refactoring * avoid duplicate setup/td * try class scope for big td * switch close and td * NullPool is the answer for test runs * more efficient run filter test * touchups and docstrings * we can only use class scope for immutable pltf. fixtures * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_indexset.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_table.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_table.py Co-authored-by: Fridolin Glatter <[email protected]> * Update tests/data/test_optimization_table.py Co-authored-by: Fridolin Glatter <[email protected]> --------- Co-authored-by: Fridolin Glatter <[email protected]>
iiasa · Aug 13, 2024 · 227fbcb · 227fbcb
1 parent f57d533
commit 227fbcb
Show file tree

Hide file tree

Showing 69 changed files with 82,506 additions and 2,503 deletions.
diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml
@@ -0,0 +1,27 @@
+name: deploy
+
+on:
+  push:
+    tags: ["v*"]
+  release:
+    types: ["published"]
+
+jobs:
+  create_deployment:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+    steps:
+      - if: github.event_name == 'release'      
+        uses: chrnorm/deployment-action@v2
+        name: Create Testing Deployment
+        with:
+          token: ${{ github.token }}
+          environment: testing
+      - if: github.event_name != 'release'   
+        uses: chrnorm/deployment-action@v2
+        name: Create Development Deployment
+        with:
+          token: ${{ github.token }}
+          environment: development
diff --git a/.github/workflows/docker.yaml b/.github/workflows/docker.yaml
diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml
@@ -1,4 +1,3 @@
-# copied from https://github.com/marketplace/actions/install-poetry-action
 name: test
 
 on: pull_request
@@ -18,30 +17,40 @@ jobs:
           - "16"
         pandas-version:
           - false
+        backend:
+          - "sqlite,rest-sqlite,postgres,rest-postgres"
         include:
+          # with pyarrow
           - python-version: "3.12"
             with-pyarrow: true
             postgres-version: "16"
             pandas-version: false
+            backend: "sqlite,rest-sqlite"
+          # pgsql 15
           - python-version: "3.12"
             with-pyarrow: false
             postgres-version: "15"
             pandas-version: false
+            backend: "postgres,rest-postgres"
+
+          # pandas 2.1.3
           - python-version: "3.11"
             with-pyarrow: true
             postgres-version: "16"
+            backend: "sqlite,rest-sqlite"
             pandas-version: "2.1.3"
+          # pandas 2.0.0
           - python-version: "3.10"
             with-pyarrow: true
             postgres-version: "16"
+            backend: "sqlite,rest-sqlite"
             pandas-version: "2.0.0"
 
-
-    name: py${{ matrix.python-version }} | with-pyarrow=${{ matrix.with-pyarrow }} | pgsql=${{ matrix.postgres-version }} | pandas=${{ matrix.pandas-version }}
+    name: py${{ matrix.python-version }} | backend=${{ matrix.backend }} | with-pyarrow=${{ matrix.with-pyarrow }} | pgsql=${{ matrix.postgres-version }} | pandas=${{ matrix.pandas-version }}
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:${{ matrix.postgres-version }}
+        image: ${{ contains(matrix.backend, 'postgres') && format('postgres:{0}', matrix.postgres-version) || '' }}
         env:
           POSTGRES_PASSWORD: postgres
           POSTGRES_DB: test
@@ -120,7 +129,7 @@ jobs:
       - name: Run tests
         run: |
           source .venv/bin/activate
-          pytest --cov-report xml:.coverage.xml --cov-report term --cov=ixmp4 -rsx --benchmark-skip
+          pytest --backend ${{ matrix.backend }} --postgres-dsn "postgresql://postgres:postgres@localhost:5432/test" --cov-report xml:.coverage.xml --cov-report term --cov=ixmp4 -rsx --benchmark-skip
 
   pre-commit:
     name: Code quality

diff --git a/.github/workflows/release_event.json b/.github/workflows/release_event.json
@@ -0,0 +1,11 @@
+{
+    "action": "released",
+    "ref": "refs/tags/v0.0.0",
+    "release": {
+        "name": "v0.0.0"
+    },
+    "github": {
+        "event_name": "release"
+    }
+
+}
diff --git a/ixmp4/data/abstract/model.py b/ixmp4/data/abstract/model.py
@@ -96,6 +96,7 @@ def tabulate(
         self,
         *,
         name: str | None = None,
+        **kwargs,
     ) -> pd.DataFrame:
         """Tabulate models by specified criteria.
 

diff --git a/ixmp4/data/abstract/scenario.py b/ixmp4/data/abstract/scenario.py
@@ -85,11 +85,7 @@ def list(
         """
         ...
 
-    def tabulate(
-        self,
-        *,
-        name: str | None = None,
-    ) -> pd.DataFrame:
+    def tabulate(self, *, name: str | None = None, **kwargs) -> pd.DataFrame:
         """Tabulate scenarios by specified criteria.
 
         Parameters

diff --git a/ixmp4/data/backend/api.py b/ixmp4/data/backend/api.py
@@ -168,3 +168,9 @@ def make_client(self, rest_url: str, auth: BaseAuth):
     def close(self):
         self.client.close()
         self.executor.shutdown(cancel_futures=True)
+
+    def setup(self):
+        pass
+
+    def teardown(self):
+        pass
diff --git a/ixmp4/data/backend/db.py b/ixmp4/data/backend/db.py
@@ -5,7 +5,7 @@
 
 from sqlalchemy.engine import Engine, create_engine
 from sqlalchemy.orm.session import sessionmaker
-from sqlalchemy.pool import StaticPool
+from sqlalchemy.pool import NullPool, StaticPool
 
 from ixmp4.conf.base import PlatformInfo
 from ixmp4.conf.manager import ManagerConfig, ManagerPlatformInfo
@@ -72,16 +72,20 @@ class SqlAlchemyBackend(Backend):
     def __init__(self, info: PlatformInfo) -> None:
         super().__init__(info)
         logger.info(f"Creating database engine for platform '{info.name}'.")
-        self.make_engine(info.dsn)
+        dsn = self.check_dsn(info.dsn)
+        self.make_engine(dsn)
         self.make_repositories()
         self.event_handler = SqlaEventHandler(self)
 
-    def make_engine(self, dsn: str):
+    def check_dsn(self, dsn: str):
         if dsn.startswith("postgresql://"):
             logger.debug(
                 "Replacing the platform dsn prefix to use the new `psycopg` driver."
             )
             dsn = dsn.replace("postgresql://", "postgresql+psycopg://")
+        return dsn
+
+    def make_engine(self, dsn: str):
         self.engine = cached_create_engine(dsn)
         self.session = self.Session(bind=self.engine)
 
@@ -99,9 +103,6 @@ def make_repositories(self):
         self.scenarios = ScenarioRepository(self)
         self.units = UnitRepository(self)
 
-    def close(self):
-        self.session.close()
-
     @contextmanager
     def auth(
         self,
@@ -123,20 +124,26 @@ def _create_all(self):
     def _drop_all(self):
         BaseModel.metadata.drop_all(bind=self.engine, checkfirst=True)
 
-    def reset(self):
-        self.session.commit()
-        self._drop_all()
+    def setup(self):
         self._create_all()
 
+    def teardown(self):
+        self.session.rollback()
+        self._drop_all()
+        self.engine = None
+        self.session = None
+
+    def close(self):
+        self.session.close()
+        self.engine.dispose()
+
 
 class SqliteTestBackend(SqlAlchemyBackend):
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(
-            PlatformInfo(name="sqlite-test", dsn="sqlite:///:memory:"),
             *args,
             **kwargs,
         )
-        self.reset()
 
     def make_engine(self, dsn: str):
         self.engine = create_engine(
@@ -150,11 +157,10 @@ def make_engine(self, dsn: str):
 class PostgresTestBackend(SqlAlchemyBackend):
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(
-            PlatformInfo(
-                name="postgres-test",
-                dsn="postgresql://postgres:postgres@localhost/test",
-            ),
             *args,
             **kwargs,
         )
-        self.reset()
+
+    def make_engine(self, dsn: str):
+        self.engine = create_engine(dsn, poolclass=NullPool)
+        self.session = self.Session(bind=self.engine)
diff --git a/ixmp4/data/db/base.py b/ixmp4/data/db/base.py
@@ -12,6 +12,7 @@
     Iterator,
     Tuple,
     TypeVar,
+    cast,
 )
 
 import numpy as np
@@ -353,18 +354,19 @@ def yield_chunks(self, df: pd.DataFrame) -> Iterator[pd.DataFrame]:
 
 class BulkUpserter(BulkOperator[ModelType]):
     def bulk_upsert(self, df: pd.DataFrame) -> None:
-        # slight performance improvement on small operations
         if len(df.index) < self.max_list_length:
             self.bulk_upsert_chunk(df)
         else:
             for chunk_df in self.yield_chunks(df):
                 self.bulk_upsert_chunk(pd.DataFrame(chunk_df))
 
     def bulk_upsert_chunk(self, df: pd.DataFrame) -> None:
+        logger.debug(f"Starting `bulk_upsert_chunk` for {len(df)} rows.")
         columns = db.utils.get_columns(self.model_class)
         df = df[list(set(columns.keys()) & set(df.columns))]
         existing_df = self.tabulate_existing(df)
         if existing_df.empty:
+            logger.debug(f"Inserting {len(df)} rows.")
             self.bulk_insert(df, skip_validation=True)
         else:
             df = self.merge_existing(df, existing_df)
@@ -393,25 +395,37 @@ def bulk_upsert_chunk(self, df: pd.DataFrame) -> None:
             )
 
             if not insert_df.empty:
+                logger.debug(f"Inserting {len(insert_df)} rows.")
                 self.bulk_insert(insert_df, skip_validation=True)
             if not update_df.empty:
+                logger.debug(f"Updating {len(update_df)} rows.")
                 self.bulk_update(update_df, skip_validation=True)
 
         self.session.commit()
 
     def bulk_insert(self, df: pd.DataFrame, **kwargs) -> None:
         # to_dict returns a more general list[Mapping[Hashable, Unknown]]
-        m: list[dict[str, Any]] = df.to_dict("records")  # type: ignore
+        if "id" in df.columns:
+            raise ProgrammingError("You may not insert the 'id' column.")
+        m = cast(list[dict[str, Any]], df.to_dict("records"))
 
         try:
-            self.session.execute(db.insert(self.model_class), m)
+            self.session.execute(
+                db.insert(self.model_class),
+                m,
+                execution_options={"synchronize_session": False},
+            )
         except IntegrityError as e:
             raise self.model_class.NotUnique(*e.args)
 
     def bulk_update(self, df: pd.DataFrame, **kwargs) -> None:
         # to_dict returns a more general list[Mapping[Hashable, Unknown]]
-        m: list[dict[str, Any]] = df.to_dict("records")  # type: ignore
-        self.session.bulk_update_mappings(self.model_class, m)  # type: ignore
+        m = cast(list[dict[str, Any]], df.to_dict("records"))
+        self.session.execute(
+            db.update(self.model_class),
+            m,
+            execution_options={"synchronize_session": False},
+        )
 
 
 class BulkDeleter(BulkOperator[ModelType]):

diff --git a/ixmp4/data/db/events.py b/ixmp4/data/db/events.py
@@ -96,6 +96,8 @@ def receive_do_orm_execute(self, orm_execute_state: ORMExecuteState):
             if orm_execute_state.is_delete:
                 self.logger.debug("Operation: 'delete'")
                 return self.receive_delete(orm_execute_state)
+            else:
+                self.logger.debug(f"Ignoring operation: {orm_execute_state}")
 
     def receive_select(self, oes: ORMExecuteState):
         # select = cast(sql.Select, oes.statement)
@@ -105,7 +107,7 @@ def receive_insert(self, oes: ORMExecuteState):
         insert = cast(sql.Insert, oes.statement)
         entity = insert.entity_description
         type_ = entity["type"]
-        self.logger.info(f"Entity: '{entity['name']}'")
+        self.logger.debug(f"Entity: '{entity['name']}'")
 
         if issubclass(type_, mixins.HasCreationInfo):
             creation_info = {
@@ -121,7 +123,7 @@ def receive_update(self, oes: ORMExecuteState):
         update = cast(sql.Update, oes.statement)
         entity = update.entity_description
         type_ = entity["type"]
-        self.logger.info(f"Entity: '{entity['name']}'")
+        self.logger.debug(f"Entity: '{entity['name']}'")
 
         if issubclass(type_, mixins.HasUpdateInfo):
             update_info = {