From 2219377a0f14171281aa5d57a74ea314c804fa7d Mon Sep 17 00:00:00 2001 From: Raymond Wiker Date: Thu, 7 Nov 2024 14:12:11 +0100 Subject: [PATCH] Added support for, and use of, the new property. --- examples/table-aggregation.ipynb | 155 +++++++++--------- .../sumo/explorer/objects/_search_context.py | 59 ++++++- 2 files changed, 127 insertions(+), 87 deletions(-) diff --git a/examples/table-aggregation.ipynb b/examples/table-aggregation.ipynb index 0425b1e8..d9180eb5 100644 --- a/examples/table-aggregation.ipynb +++ b/examples/table-aggregation.ipynb @@ -7,6 +7,7 @@ "metadata": {}, "outputs": [], "source": [ + "import json\n", "import time\n", "class Timer:\n", " def __init__(self):\n", @@ -18,7 +19,7 @@ " t1 = time.perf_counter()\n", " print(f\"Elapsed: {t1-self._t0:0.3f} seconds.\")\n", " return\n", - " pass" + " pass\n" ] }, { @@ -59,17 +60,15 @@ " tot_size_bytes = total_blob_size(rels)\n", " print(f\"Total size of input: {tot_size_bytes / (1024*1024*1024):.3f} GiB\")\n", " with Timer():\n", - " agg=rels.filter(column=columns).aggregate(columns=columns)\n", + " agg=rels.filter(column=columns)._aggregate(columns=columns)\n", " print(agg.to_pandas().sort_values(by=[\"REAL\", \"DATE\"]))\n", "\n", "def run_exp(caseuuid, itername, tagname, columns):\n", " case = exp.get_case_by_uuid(caseuuid)\n", " print(f\"{case.asset}: {case.name}: {caseuuid}\")\n", - " rels=case.tables.filter(iteration=itername, realization=True, tagname=tagname, \n", - " complex={\"bool\": {\"must_not\": [{\"term\": {\"_sumo.hidden\": True}}]}})\n", + " rels=case.tables.visible.filter(iteration=itername, realization=True, tagname=tagname, column=columns)\n", " do_aggregate(tagname, rels, columns)\n", - " rels=case.tables.filter(iteration=itername, realization=True, tagname=tagname,\n", - " complex={\"term\": {\"_sumo.hidden\": True}})\n", + " rels=case.tables.hidden.filter(iteration=itername, realization=True, tagname=tagname, column=columns)\n", " do_aggregate(tagname, rels, columns)" ] }, @@ -86,36 +85,36 @@ "Troll: 24.0.0-20240828_ix_network_test5: 359e7c72-a4ca-43ee-9203-f09cd0f149a9\n", "summary: 27 objects, 64996 columns.\n", "Total size of input: 1.248 GiB\n", - "Elapsed: 15.166 seconds.\n", + "Elapsed: 15.108 seconds.\n", " DATE FOPT REAL\n", - "137 2024-07-02 282442208.0 6\n", - "138 2024-07-03 282451072.0 6\n", - "139 2024-08-01 282677120.0 6\n", - "140 2024-09-01 282889760.0 6\n", - "141 2024-10-01 283077440.0 6\n", + "198 2024-07-02 282442208.0 6\n", + "199 2024-07-03 282451072.0 6\n", + "200 2024-08-01 282677120.0 6\n", + "201 2024-09-01 282889760.0 6\n", + "202 2024-10-01 283077440.0 6\n", ".. ... ... ...\n", - "47 2025-02-15 286229120.0 249\n", - "48 2025-04-01 286425696.0 249\n", - "49 2025-09-01 287060416.0 249\n", - "50 2025-10-01 287176832.0 249\n", - "51 2026-01-01 287523552.0 249\n", + "193 2025-02-15 286229120.0 249\n", + "194 2025-04-01 286425696.0 249\n", + "195 2025-09-01 287060416.0 249\n", + "196 2025-10-01 287176832.0 249\n", + "197 2026-01-01 287523552.0 249\n", "\n", "[265 rows x 3 columns]\n", - "summary: 3537 objects, 64996 columns.\n", - "Total size of input: 1.087 GiB\n", - "Elapsed: 1.692 seconds.\n", + "summary: 27 objects, 554 columns.\n", + "Total size of input: 0.009 GiB\n", + "Elapsed: 1.351 seconds.\n", " DATE FOPT REAL\n", - "52 2024-07-02 282442208.0 6\n", - "53 2024-07-03 282451072.0 6\n", - "54 2024-08-01 282677120.0 6\n", - "55 2024-09-01 282889760.0 6\n", - "56 2024-10-01 283077440.0 6\n", + "114 2024-07-02 282442208.0 6\n", + "115 2024-07-03 282451072.0 6\n", + "116 2024-08-01 282677120.0 6\n", + "117 2024-09-01 282889760.0 6\n", + "118 2024-10-01 283077440.0 6\n", ".. ... ... ...\n", - "173 2025-02-15 286229120.0 249\n", - "174 2025-04-01 286425696.0 249\n", - "175 2025-09-01 287060416.0 249\n", - "176 2025-10-01 287176832.0 249\n", - "177 2026-01-01 287523552.0 249\n", + "16 2025-02-15 286229120.0 249\n", + "17 2025-04-01 286425696.0 249\n", + "18 2025-09-01 287060416.0 249\n", + "19 2025-10-01 287176832.0 249\n", + "20 2026-01-01 287523552.0 249\n", "\n", "[265 rows x 3 columns]\n" ] @@ -138,36 +137,36 @@ "Troll: 24.0.0-20240820: fc6cc7d3-6162-46a3-9d69-48ad1eaecdfb\n", "summary: 196 objects, 24568 columns.\n", "Total size of input: 30.013 GiB\n", - "Elapsed: 32.124 seconds.\n", + "Elapsed: 32.407 seconds.\n", " DATE FOPT REAL\n", - "708796 1990-02-01 0.000000e+00 1\n", - "708797 1990-03-01 1.445590e+05 1\n", - "708798 1990-04-01 2.741935e+05 1\n", - "708799 1990-05-01 4.145006e+05 1\n", - "708800 1990-06-01 5.512956e+05 1\n", + "778120 1990-02-01 0.000000e+00 1\n", + "778121 1990-03-01 1.445590e+05 1\n", + "778122 1990-04-01 2.741935e+05 1\n", + "778123 1990-05-01 4.145006e+05 1\n", + "778124 1990-06-01 5.512956e+05 1\n", "... ... ... ...\n", - "841571 2024-06-27 2.980280e+08 249\n", - "841572 2024-06-28 2.980311e+08 249\n", - "841573 2024-06-29 2.980342e+08 249\n", - "841574 2024-06-30 2.980384e+08 249\n", - "841575 2024-07-01 2.980405e+08 249\n", + "249139 2024-06-27 2.980280e+08 249\n", + "249140 2024-06-28 2.980311e+08 249\n", + "249141 2024-06-29 2.980342e+08 249\n", + "249142 2024-06-30 2.980384e+08 249\n", + "249143 2024-07-01 2.980405e+08 249\n", "\n", "[952560 rows x 3 columns]\n", - "summary: 9800 objects, 24568 columns.\n", - "Total size of input: 29.907 GiB\n", - "Elapsed: 4.722 seconds.\n", + "summary: 196 objects, 500 columns.\n", + "Total size of input: 1.328 GiB\n", + "Elapsed: 3.757 seconds.\n", " DATE FOPT REAL\n", - "34020 1990-02-01 0.000000e+00 1\n", - "34021 1990-03-01 1.445590e+05 1\n", - "34022 1990-04-01 2.741935e+05 1\n", - "34023 1990-05-01 4.145006e+05 1\n", - "34024 1990-06-01 5.512956e+05 1\n", + "488844 1990-02-01 0.000000e+00 1\n", + "488845 1990-03-01 1.445590e+05 1\n", + "488846 1990-04-01 2.741935e+05 1\n", + "488847 1990-05-01 4.145006e+05 1\n", + "488848 1990-06-01 5.512956e+05 1\n", "... ... ... ...\n", - "316447 2024-06-27 2.980280e+08 249\n", - "316448 2024-06-28 2.980311e+08 249\n", - "316449 2024-06-29 2.980342e+08 249\n", - "316450 2024-06-30 2.980384e+08 249\n", - "316451 2024-07-01 2.980405e+08 249\n", + "352759 2024-06-27 2.980280e+08 249\n", + "352760 2024-06-28 2.980311e+08 249\n", + "352761 2024-06-29 2.980342e+08 249\n", + "352762 2024-06-30 2.980384e+08 249\n", + "352763 2024-07-01 2.980405e+08 249\n", "\n", "[952560 rows x 3 columns]\n" ] @@ -190,36 +189,36 @@ "Drogon: ruaj_testcase: 5b558daf-61c5-400a-9aa2-c602bb471a16\n", "summary: 160 objects, 974 columns.\n", "Total size of input: 0.175 GiB\n", - "Elapsed: 2.485 seconds.\n", + "Elapsed: 2.270 seconds.\n", " DATE FOPT REAL\n", - "4910 2018-01-01 0.000000e+00 0\n", - "4911 2018-01-02 0.000000e+00 0\n", - "4912 2018-01-05 0.000000e+00 0\n", - "4913 2018-01-06 3.991868e+03 0\n", - "4914 2018-01-09 1.596676e+04 0\n", + "8097 2018-01-01 0.000000e+00 0\n", + "8098 2018-01-02 0.000000e+00 0\n", + "8099 2018-01-05 0.000000e+00 0\n", + "8100 2018-01-06 3.991868e+03 0\n", + "8101 2018-01-09 1.596676e+04 0\n", "... ... ... ...\n", - "36831 2020-06-14 7.278816e+06 159\n", - "36832 2020-06-27 7.349246e+06 159\n", - "36833 2020-06-28 7.354664e+06 159\n", - "36834 2020-06-30 7.365482e+06 159\n", - "36835 2020-07-01 7.370888e+06 159\n", + "39275 2020-06-14 7.278816e+06 159\n", + "39276 2020-06-27 7.349246e+06 159\n", + "39277 2020-06-28 7.354664e+06 159\n", + "39278 2020-06-30 7.365482e+06 159\n", + "39279 2020-07-01 7.370888e+06 159\n", "\n", "[39280 rows x 3 columns]\n", - "summary: 320 objects, 974 columns.\n", - "Total size of input: 0.163 GiB\n", - "Elapsed: 2.528 seconds.\n", + "summary: 160 objects, 500 columns.\n", + "Total size of input: 0.097 GiB\n", + "Elapsed: 1.794 seconds.\n", " DATE FOPT REAL\n", - "19394 2018-01-01 0.000000e+00 0\n", - "19395 2018-01-02 0.000000e+00 0\n", - "19396 2018-01-05 0.000000e+00 0\n", - "19397 2018-01-06 3.991868e+03 0\n", - "19398 2018-01-09 1.596676e+04 0\n", + "247 2018-01-01 0.000000e+00 0\n", + "248 2018-01-02 0.000000e+00 0\n", + "249 2018-01-05 0.000000e+00 0\n", + "250 2018-01-06 3.991868e+03 0\n", + "251 2018-01-09 1.596676e+04 0\n", "... ... ... ...\n", - "10795 2020-06-14 7.278816e+06 159\n", - "10796 2020-06-27 7.349246e+06 159\n", - "10797 2020-06-28 7.354664e+06 159\n", - "10798 2020-06-30 7.365482e+06 159\n", - "10799 2020-07-01 7.370888e+06 159\n", + "31149 2020-06-14 7.278816e+06 159\n", + "31150 2020-06-27 7.349246e+06 159\n", + "31151 2020-06-28 7.354664e+06 159\n", + "31152 2020-06-30 7.365482e+06 159\n", + "31153 2020-07-01 7.370888e+06 159\n", "\n", "[39280 rows x 3 columns]\n" ] diff --git a/src/fmu/sumo/explorer/objects/_search_context.py b/src/fmu/sumo/explorer/objects/_search_context.py index c264ccd7..26c20b86 100644 --- a/src/fmu/sumo/explorer/objects/_search_context.py +++ b/src/fmu/sumo/explorer/objects/_search_context.py @@ -276,10 +276,14 @@ def __init__( sumo: SumoClient, must: List = [], must_not: List = [], + hidden = False, + visible = True ): self._sumo = sumo self._must = must[:] self._must_not = must_not[:] + self._visible = visible + self._hidden = hidden self._field_values = {} self._hits = None self._cache = LRUCache(capacity=200) @@ -288,17 +292,24 @@ def __init__( @property def _query(self): - if len(self._must_not) == 0: - if len(self._must) == 1: - return self._must[0] + must = self._must[:] + must_not = self._must_not[:] + if self._visible and not self._hidden: + must_not.append({"term": {"_sumo.hidden": True}}) + elif not self._visible and self._hidden: + must.append({"term": {"_sumo.hidden": True}}) + pass + if len(must_not) == 0: + if len(must) == 1: + return must[0] else: - return {"bool": {"must": self._must}} + return {"bool": {"must": must}} else: - if len(self._must) == 0: - return {"bool": {"must_not": self._must_not}} + if len(must) == 0: + return {"bool": {"must_not": must_not}} else: return { - "bool": {"must": self._must, "must_not": self._must_not} + "bool": {"must": must, "must_not": must_not} } def _to_sumo(self, obj, blob=None): @@ -768,6 +779,30 @@ async def _get_field_values_async(self, field: str) -> List: def _context_for_class(self, cls): return self.filter(cls=cls) + @property + def hidden(self): + return SearchContext(sumo=self._sumo, + must=self._must, + must_not = self._must_not, + hidden = True, + visible = False) + + @property + def visible(self): + return SearchContext(sumo=self._sumo, + must=self._must, + must_not = self._must_not, + hidden = False, + visible = True) + + @property + def all(self): + return SearchContext(sumo=self._sumo, + must=self._must, + must_not = self._must_not, + hidden = True, + visible = True) + @property def cases(self): """Cases from current selection.""" @@ -874,7 +909,7 @@ def filter(self, **kwargs) -> "SearchContext": if _must_not is not None: must_not.append(_must_not) - sc = SearchContext(self._sumo, must=must, must_not=must_not) + sc = SearchContext(self._sumo, must=must, must_not=must_not, hidden=self._hidden, visible = self._visible) if "has" in kwargs: # Get list of cases matched by current filter set @@ -1161,7 +1196,7 @@ def _verify_aggregation_operation(self): rids = [hit["_source"]["fmu"]["realization"]["id"] for hit in hits] return prototype, uuids, rids - def aggregate(self, columns=None, operation=None): + def _aggregate(self, columns=None, operation=None): prototype, uuids, rids = self._verify_aggregation_operation() spec = { "object_ids": uuids, @@ -1199,6 +1234,12 @@ def aggregate(self, columns=None, operation=None): res._blob = blob return res + def aggregate(self, columns=None, operation=None): + if len(self.hidden) > 0: + return self.hidden._aggregate(columns=columns, operation=operation) + else: + return self.visible._aggregate(columns=columns, operation=operation) + @deprecation.deprecated(details="Use the method 'aggregate' instead, with parameter 'operation'.") def min(self): return self.aggregate(operation="min")