Improve handling of missing responses.json

src/ert/dark_storage/endpoints/experiments.py

@@ -22,11 +22,14 @@ def get_experiments(
         js.ExperimentOut(
             id=experiment.id,
             name=experiment.name,
-            ensemble_ids=[ens.id for ens in experiment.ensembles],
+            ensemble_ids=[
+                ens.id for ens in experiment.ensembles if experiment.is_valid()
+            ],
             priors=create_priors(experiment),
             userdata={},
         )
         for experiment in storage.experiments
+        if experiment.is_valid()
     ]
 
 

src/ert/dark_storage/endpoints/records.py

@@ -124,8 +124,7 @@ def get_ensemble_responses(
 
     response_names_with_observations = set()
     observations = ensemble.experiment.observations
-
-    if len(ensemble.has_data()) == 0:
+    if not ensemble.experiment.is_valid() or len(ensemble.has_data()) == 0:
         return {}
 
     for (

src/ert/gui/ertnotifier.py

@@ -63,3 +63,10 @@ def set_current_ensemble(self, ensemble: Ensemble | None = None) -> None:
     @Slot(bool)
     def set_is_simulation_running(self, is_running: bool) -> None:
         self._is_simulation_running = is_running
+        self.refresh()
+
+    def refresh(self) -> None:
+        if self._storage is None:
+            return
+        self._storage.refresh()
+        self.storage_changed.emit(self._storage)

src/ert/gui/ertwidgets/ensembleselector.py

@@ -4,6 +4,7 @@
 from typing import TYPE_CHECKING
 
 from qtpy.QtCore import Qt, Signal
+from qtpy.QtGui import QStandardItemModel
 from qtpy.QtWidgets import QComboBox
 
 from ert.gui.ertnotifier import ErtNotifier
@@ -22,6 +23,7 @@ def __init__(
         update_ert: bool = True,
         show_only_undefined: bool = False,
         show_only_no_children: bool = False,
+        show_only_with_valid_experiment: bool = False,
     ):
         super().__init__()
         self.notifier = notifier
@@ -36,6 +38,7 @@ def __init__(
         # if the ensemble has not been used in an update, as that would
         # invalidate the result
         self._show_only_no_children = show_only_no_children
+        self._show_only_with_valid_experiment = show_only_with_valid_experiment
         self.setSizeAdjustPolicy(QComboBox.AdjustToContents)
 
         self.setEnabled(False)
@@ -68,9 +71,28 @@ def populate(self) -> None:
             self.setEnabled(True)
 
         for ensemble in self._ensemble_list():
+            model = self.model()
+            assert isinstance(model, QStandardItemModel)
+            assert model is not None
             self.addItem(
                 f"{ensemble.experiment.name} : {ensemble.name}", userData=ensemble
             )
+            if (
+                self._show_only_with_valid_experiment
+                and not ensemble.experiment.is_valid()
+            ):
+                index = self.count() - 1
+                model_item = model.item(index)
+                assert model_item is not None
+                new_flags = (
+                    model_item.flags()
+                    & ~Qt.ItemFlags(Qt.ItemFlag.ItemIsEnabled)
+                    & ~Qt.ItemFlags(Qt.ItemFlag.ItemIsSelectable)
+                )
+                model_item.setFlags(new_flags)
+                self.setItemData(
+                    index, "This ensemble is invalid", Qt.ItemDataRole.ToolTipRole
+                )
 
         current_index = self.findData(
             self.notifier.current_ensemble, Qt.ItemDataRole.UserRole
@@ -95,6 +117,8 @@ def _ensemble_list(self) -> Iterable[Ensemble]:
         else:
             ensembles = self.notifier.storage.ensembles
         ensemble_list = list(ensembles)
+        if self._show_only_with_valid_experiment:
+            ensemble_list = [ens for ens in ensemble_list if ens.experiment.is_valid()]
         if self._show_only_no_children:
             parents = [
                 ens.parent for ens in self.notifier.storage.ensembles if ens.parent

src/ert/gui/main_window.py

@@ -5,7 +5,7 @@
 import webbrowser
 
 from qtpy.QtCore import QCoreApplication, QEvent, QSize, Qt, Signal, Slot
-from qtpy.QtGui import QCloseEvent, QCursor, QIcon, QMouseEvent
+from qtpy.QtGui import QCloseEvent, QCursor, QFocusEvent, QIcon, QMouseEvent
 from qtpy.QtWidgets import (
     QAction,
     QButtonGroup,
@@ -105,7 +105,8 @@
         self.side_frame = QFrame(self)
         self.button_group = QButtonGroup(self.side_frame)
         self._external_plot_windows: list[PlotWindow] = []
-
+        self.setFocusPolicy(Qt.FocusPolicy.TabFocus)
+        self.central_widget.setFocusPolicy(Qt.FocusPolicy.TabFocus)
         if self.is_dark_mode():
             self.side_frame.setStyleSheet("background-color: rgb(64, 64, 64);")
         else:
@@ -154,6 +155,7 @@
     def select_central_widget(self) -> None:
         actor = self.sender()
         if actor:
+            self.notifier.refresh()
             index_name = actor.property("index")
 
             for widget in self.central_panels_map.values():
@@ -364,3 +366,7 @@
     def __showAboutMessage(self) -> None:
         diag = AboutDialog(self)
         diag.show()
+
+    def focusInEvent(self, event: QFocusEvent) -> None:
+        self.notifier.refresh()
+        QMainWindow.focusInEvent(self, event)

src/ert/gui/model/real_list.py

@@ -1,4 +1,4 @@
-from typing import overload
+from typing import cast, overload
 
 from qtpy.QtCore import (
     QAbstractItemModel,
@@ -9,7 +9,12 @@
 )
 from typing_extensions import override
 
-from ert.gui.model.snapshot import IsEnsembleRole, IsRealizationRole, NodeRole
+from ert.gui.model.snapshot import (
+    IsEnsembleRole,
+    IsRealizationRole,
+    NodeRole,
+    SnapshotModel,
+)
 
 
 class RealListModel(QAbstractProxyModel):
@@ -67,7 +72,7 @@ def columnCount(self, parent: QModelIndex | None = None) -> int:
     def rowCount(self, parent: QModelIndex | None = None) -> int:
         parent = parent if parent else QModelIndex()
         if not parent.isValid():
-            source_model = self.sourceModel()
+            source_model = cast(SnapshotModel, self.sourceModel())
             assert source_model is not None
             iter_index = source_model.index(self._iter, 0, QModelIndex())
             if iter_index.isValid():

src/ert/gui/simulation/evaluate_ensemble_panel.py

@@ -35,7 +35,9 @@ def __init__(self, ensemble_size: int, run_path: str, notifier: ErtNotifier):
         lab.setWordWrap(True)
         lab.setAlignment(QtCore.Qt.AlignmentFlag.AlignLeft)
         layout.addRow(lab)
-        self._ensemble_selector = EnsembleSelector(notifier, show_only_no_children=True)
+        self._ensemble_selector = EnsembleSelector(
+            notifier, show_only_no_children=True, show_only_with_valid_experiment=True
+        )
         layout.addRow("Ensemble:", self._ensemble_selector)
         runpath_label = CopyableLabel(text=run_path)
         layout.addRow("Runpath:", runpath_label)

src/ert/gui/tools/manage_experiments/manage_experiments_panel.py

@@ -47,7 +47,6 @@ def __init__(self, config: ErtConfig, notifier: ErtNotifier, ensemble_size: int)
     def _add_create_new_ensemble_tab(self) -> None:
         panel = QWidget()
         panel.setObjectName("create_new_ensemble_tab")
-
         layout = QHBoxLayout()
         storage_widget = StorageWidget(
             self.notifier, self.ert_config, self.ensemble_size
@@ -75,7 +74,11 @@ def _add_initialize_from_scratch_tab(self) -> None:
 
         ensemble_layout = QHBoxLayout()
         ensemble_label = QLabel("Target ensemble:")
-        ensemble_selector = EnsembleSelector(self.notifier, show_only_undefined=True)
+        ensemble_selector = EnsembleSelector(
+            self.notifier,
+            show_only_undefined=True,
+            show_only_with_valid_experiment=True,
+        )
         ensemble_selector.setMinimumWidth(300)
         ensemble_layout.addWidget(ensemble_label)
         ensemble_layout.addWidget(ensemble_selector)

src/ert/gui/tools/manage_experiments/storage_model.py

@@ -63,6 +63,7 @@ def __init__(self, ensemble: Ensemble, parent: Any):
         self._id = ensemble.id
         self._start_time = ensemble.started_at
         self._children: list[RealizationModel] = []
+        self._error = ensemble.experiment.error_message
 
     def add_realization(self, realization: RealizationModel) -> None:
         self._children.append(realization)
@@ -75,25 +76,27 @@ def row(self) -> int:
     def data(self, index: QModelIndex, role: Qt.ItemDataRole) -> Any:
         if not index.isValid():
             return None
-
         col = index.column()
         if role == Qt.ItemDataRole.DisplayRole:
             if col == _Column.NAME:
                 return self._name
             if col == _Column.TIME:
                 return humanize.naturaltime(self._start_time)
         elif role == Qt.ItemDataRole.ToolTipRole:
+            if self._error:
+                return self._error
             if col == _Column.TIME:
                 return str(self._start_time)
-
         return None
 
 
-class ExperimentModel:
-    def __init__(self, experiment: Experiment, parent: Any):
+class ExperimentModel(QAbstractItemModel):
+    def __init__(self, experiment: Experiment, parent: "StorageModel"):
         self._parent = parent
         self._id = experiment.id
         self._name = experiment.name
+        self._is_valid = experiment.is_valid()
+        self._error = experiment.error_message
         self._experiment_type = experiment.metadata.get("ensemble_type")
         self._children: list[EnsembleModel] = []
 
@@ -105,9 +108,7 @@ def row(self) -> int:
             return self._parent._children.index(self)
         return 0
 
-    def data(
-        self, index: QModelIndex, role: Qt.ItemDataRole = Qt.ItemDataRole.DisplayRole
-    ) -> Any:
+    def data(self, index: QModelIndex, role: int = Qt.ItemDataRole.DisplayRole) -> Any:
         if not index.isValid():
             return None
 
@@ -128,7 +129,9 @@ def data(
                 qapp = QApplication.instance()
                 assert isinstance(qapp, QApplication)
                 return qapp.palette().mid()
-
+        elif role == Qt.ItemDataRole.ToolTipRole:
+            if self._error:
+                return self._error
         return None
 
 
@@ -211,9 +214,31 @@ def headerData(
     def data(self, index: QModelIndex, role: int = Qt.ItemDataRole.DisplayRole) -> Any:
         if not index.isValid():
             return None
-
         return index.internalPointer().data(index, role)
 
+    @override
+    def flags(self, index: QModelIndex) -> Qt.ItemFlags:
+        default_flags = super().flags(index)
+        if not index.isValid():
+            return default_flags
+        item = index.internalPointer()
+        if isinstance(item, ExperimentModel) and not item._is_valid:
+            new_flags = default_flags & ~Qt.ItemFlags(Qt.ItemFlag.ItemIsEnabled)
+            return Qt.ItemFlags(new_flags)
+        return default_flags
+
+    @override
+    def hasChildren(self, parent: QModelIndex | None = None) -> bool:
+        if parent is None or not parent.isValid():
+            return True
+
+        flags = self.flags(parent)
+        # hide children if disabled
+        if not (flags & Qt.ItemFlag.ItemIsEnabled):
+            return False
+
+        return super().hasChildren(parent)
+
     @override
     def index(
         self, row: int, column: int, parent: QModelIndex | None = None

src/ert/gui/tools/plot/plot_api.py

@@ -112,7 +112,7 @@ def all_data_type_keys(self) -> list[PlotApiKeyDefinition]:
                     f"/experiments/{experiment['id']}/ensembles", timeout=self._timeout
                 )
                 self._check_response(response)
-
+                print(f"{experiment=}")
                 for ensemble in response.json():
                     response = client.get(
                         f"/ensembles/{ensemble['id']}/responses", timeout=self._timeout

src/ert/storage/local_ensemble.py

@@ -92,6 +92,7 @@ def create_realization_dir(realization: int) -> Path:
             return self._path / f"realization-{realization}"
 
         self._realization_dir = create_realization_dir
+        self.has_valid_experiment: bool | None = None
 
     @classmethod
     def create(
@@ -406,6 +407,11 @@ def get_ensemble_state(self) -> list[set[RealizationStorageState]]:
         states : list of RealizationStorageState
             list of realization states.
         """
+        if not self.experiment.is_valid():
+            logger.warning(
+                f"Could not get ensemble state for ensemble ({self.id}) due to invalid experiment ({self.experiment_id}): {self.experiment.error_message}"
+            )
+            return []
 
         response_configs = self.experiment.response_configuration
 

src/ert/storage/local_experiment.py

@@ -75,6 +75,26 @@ def __init__(
         self._index = _Index.model_validate_json(
             (path / "index.json").read_text(encoding="utf-8")
         )
+        self._validate_files()
+
+    def _validate_files(self) -> None:
+        self.valid_parameters = (self._path / self._parameter_file).exists()
+        self.valid_responses = (self._path / self._responses_file).exists()
+        self.valid_metadata = (self._path / self._metadata_file).exists()
+
+    def is_valid(self) -> bool:
+        return self.valid_parameters and self.valid_responses and self.valid_metadata
+
+    @property
+    def error_message(self) -> str:
+        errors = []
+        if not self.valid_parameters:
+            errors.append("Parameter file is missing")
+        if not self.valid_responses:
+            errors.append("Responses file is missing")
+        if not self.valid_metadata:
+            errors.append("Metadata file is missing")
+        return "\n".join(errors)
 
     @classmethod
     def create(

src/ert/storage/local_storage.py

@@ -226,11 +226,15 @@ def _load_ensembles(self) -> dict[UUID, LocalEnsemble]:
         }
 
     def _load_experiments(self) -> dict[UUID, LocalExperiment]:
-        experiment_ids = {ens.experiment_id for ens in self._ensembles.values()}
-        return {
-            exp_id: LocalExperiment(self, self._experiment_path(exp_id), self.mode)
-            for exp_id in experiment_ids
-        }
+        experiments = {}
+        for ens in self._ensembles.values():
+            experiment = LocalExperiment(
+                self, self._experiment_path(ens.experiment_id), self.mode
+            )
+            ens.has_valid_experiment = experiment.is_valid()
+            experiments[ens.experiment_id] = experiment
+
+        return experiments
 
     def _ensemble_path(self, ensemble_id: UUID) -> Path:
         return self.path / self.ENSEMBLES_PATH / str(ensemble_id)

tests/ert/ui_tests/gui/conftest.py

@@ -394,6 +394,13 @@ def wait_for_child(gui, qtbot: QtBot, typ: type[V], timeout=5000, **kwargs) -> V
     return get_child(gui, typ, **kwargs)
 
 
+def wait_for_children(
+    gui, qtbot: QtBot, typ: type[V], timeout=5000, **kwargs
+) -> list[V]:
+    qtbot.waitUntil(lambda: gui.findChildren(typ) is not None, timeout=timeout)
+    return get_children(gui, typ, **kwargs)
+
+
 def get_child(gui: QWidget, typ: type[V], *args, **kwargs) -> V:
     child = gui.findChild(typ, *args, **kwargs)
     assert isinstance(child, typ)