Revert "Store everest results in ERT storage"

docs/everest/cli.rst

@@ -52,7 +52,78 @@ Using again the command `everest monitor config_file.yml`, will reattach to the
 Everest `export`
 ================
 
-The everest export has been removed. All data is now always exported to the optimization output directory.
+.. argparse::
+    :module: everest.bin.everexport_script
+    :func: _build_args_parser
+    :prog: everexport_entry
+
+
+The everest export functionality is configured in the export section of the config file.
+The following represents an export section a config file set with default values.
+
+.. code-block:: yaml
+
+    export:
+        skip_export: False
+        keywords:
+        batches:
+        discard_gradient: True # Export only non-gradient simulations
+        discard_rejected: True # Export only increased merit simulations
+        csv_output_filepath: everest_output_folder/config_file.csv
+
+When the export command `everest export config_file.yml` is run with a config file that does not define an export section default values will be used, a `config_file.csv` file in the Everest output folder will be created.
+By default Everest exports only non-gradient with increased merit simulations when no config section is defined in the config file.
+The file will contain optimization data for all the optimization batches and the available eclipse keywords (if a data file is available) for only the non-gradient simulations and the simulations that increase merit.
+
+**Examples**
+
+* Export only non-gradient simulation using the following export section in the config file
+
+.. code-block:: yaml
+
+    export:
+        discard_rejected: False
+
+* Export only increased merit simulation using the following export section in the config file
+
+.. code-block:: yaml
+
+    export:
+        discard_gradient: False
+
+
+* Export only a list of available batches even if they are gradient batches and if no export section is defined.
+
+ everest export config_file.yml --batches 0 2 4
+
+The command above is equivalent to having the following export section defined in the config file `config_file.yml`.
+
+.. code-block:: yaml
+
+    export:
+      batches: [0, 2, 4]
+
+* Exporting just a specific list of eclipse keywords requires the following export section defined in the config file.
+
+.. code-block:: yaml
+
+    export:
+      keywords: ['FOIP', 'FOPT']
+
+* Skip export by adding the following section in the config file.
+
+.. code-block:: yaml
+
+    export:
+      skip_export: True
+
+* Export will also be skipped if an empty list of batches is defined in the export section.
+
+.. code-block:: yaml
+
+    export:
+      batches: []
+
 
 ==============
 Everest `lint`

src/ert/run_models/everest_run_model.py

@@ -9,18 +9,21 @@
 import shutil
 from collections import defaultdict
 from collections.abc import Callable
+from dataclasses import dataclass
 from enum import IntEnum
 from pathlib import Path
 from types import TracebackType
 from typing import TYPE_CHECKING, Any, Protocol
 
 import numpy as np
+import seba_sqlite.sqlite_storage
 from numpy import float64
 from numpy._typing import NDArray
 from ropt.enums import EventType, OptimizerExitCode
 from ropt.evaluator import EvaluatorContext, EvaluatorResult
 from ropt.plan import BasicOptimizer
 from ropt.plan import Event as OptimizerEvent
+from seba_sqlite import SqliteStorage
 from typing_extensions import TypedDict
 
 from _ert.events import EESnapshot, EESnapshotUpdate, Event
@@ -29,7 +32,6 @@
 from ert.runpaths import Runpaths
 from ert.storage import open_storage
 from everest.config import EverestConfig
-from everest.everest_storage import EverestStorage, OptimalResult
 from everest.optimizer.everest2ropt import everest2ropt
 from everest.simulator.everest_to_ert import everest_to_ert_config
 from everest.strings import EVEREST
@@ -68,6 +70,24 @@ class OptimizerCallback(Protocol):
     def __call__(self) -> str | None: ...
 
 
+@dataclass
+class OptimalResult:
+    batch: int
+    controls: list[Any]
+    total_objective: float
+
+    @staticmethod
+    def from_seba_optimal_result(
+        o: seba_sqlite.sqlite_storage.OptimalResult | None = None,
+    ) -> OptimalResult | None:
+        if o is None:
+            return None
+
+        return OptimalResult(
+            batch=o.batch, controls=o.controls, total_objective=o.total_objective
+        )
+
+
 class EverestExitCode(IntEnum):
     COMPLETED = 1
     TOO_FEW_REALIZATIONS = 2
@@ -186,21 +206,23 @@ def run_experiment(
         # Initialize the ropt optimizer:
         optimizer = self._create_optimizer()
 
-        self.ever_storage = EverestStorage(
-            output_dir=Path(self._everest_config.optimization_output_dir),
-        )
-        self.ever_storage.observe_optimizer(
-            optimizer,
-            Path(self._everest_config.optimization_output_dir)
-            / "dakota"
-            / "OPT_DEFAULT.out",
+        # The SqliteStorage object is used to store optimization results from
+        # Seba in an sqlite database. It reacts directly to events emitted by
+        # Seba and is not called by Everest directly. The stored results are
+        # accessed by Everest via separate SebaSnapshot objects.
+        # This mechanism is outdated and not supported by the ropt package. It
+        # is retained for now via the seba_sqlite package.
+        seba_storage = SqliteStorage(  # type: ignore
+            optimizer, self._everest_config.optimization_output_dir
         )
 
         # Run the optimization:
         optimizer_exit_code = optimizer.run().exit_code
 
         # Extract the best result from the storage.
-        self._result = self.ever_storage.get_optimal_result()
+        self._result = OptimalResult.from_seba_optimal_result(
+            seba_storage.get_optimal_result()  # type: ignore
+        )
 
         if self._exit_code is None:
             match optimizer_exit_code:

src/everest/__init__.py

@@ -21,12 +21,18 @@
     __version__ = "0.0.0"
 
 from everest import detached, jobs, templates, util
+from everest.bin.utils import export_to_csv, export_with_progress
 from everest.config_keys import ConfigKeys
+from everest.export import MetaDataColumnNames, filter_data
 
 __author__ = "Equinor ASA and TNO"
 __all__ = [
     "ConfigKeys",
+    "MetaDataColumnNames",
     "detached",
+    "export_to_csv",
+    "export_with_progress",
+    "filter_data",
     "jobs",
     "load",
     "templates",