Added various metrics to SearchContext. (#346)

* Added various metrics to SearchContext. * Cleanup. * Only test write operations on cases that are writable (status=='scratch') * Adjust expected number of aggregations in test case; the value previously used (7) is not correct. * Skip bulk aggregation tests unless on linux, python 3.11 --------- Co-authored-by: Raymond Wiker <[email protected]>
equinor · Oct 17, 2024 · e935ad1 · e935ad1
1 parent 5b924b5
commit e935ad1
Show file tree

Hide file tree

Showing 10 changed files with 247 additions and 6 deletions.
diff --git a/examples/metrics.ipynb b/examples/metrics.ipynb
@@ -0,0 +1,171 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a26f4146-4206-4f74-aba4-a9b165b00666",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from fmu.sumo.explorer import Explorer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "9d04d0ea-b8ba-4891-ace1-4a5a8141e2c1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "exp=Explorer(env=\"preview\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0466932c-08c6-4a2b-991d-7ef752e37a87",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "case=exp.get_case_by_uuid(\"359e7c72-a4ca-43ee-9203-f09cd0f149a9\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5981d890-3797-497a-8543-80c6a0b9af2c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tables=case.tables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "73f4b0aa-ec42-40af-bf47-eea30e6c265a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries=tables.filter(tagname=\"summary\", realization=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "56160303-4881-4215-9daf-ce237c0b9bc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.min(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a5d63240-fb6e-477b-a1a7-41f99f2900b7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.max(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2c218f8b-e210-40f8-be5a-5202efa1ef9d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.avg(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "393b2551-7d25-435a-89a9-5c3dbb3b3e51",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.stats(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "07d28bb7-eee9-41e4-a3e6-491aba7a3f4b",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.extended_stats(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "0355fa71-3605-46f0-bd22-872bbdfa3ac7",
+   "metadata": {},
+   "source": [
+    "summaries.metrics.percentiles(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a2859bee-133e-4631-91d3-618c22d942eb",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.percentiles(field=\"_sumo.blob_size\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "d6cb14c9-0a9a-4c91-bd86-7f783e3dcfcf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.percentiles(field=\"_sumo.blob_size\", percents=[95, 99, 99.9])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "91355328-db37-473f-aad8-e2c7d2fd7d10",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "summaries.metrics.sum(field=\"_sumo.blob_size\")[\"value\"]/(1024*1024*1024)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cb3be689-6db2-483e-8195-2fcc3e1cdc69",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/src/fmu/sumo/explorer/objects/__init__.py b/src/fmu/sumo/explorer/objects/__init__.py
@@ -1,6 +1,7 @@
 """Sumo cases and child objects"""
 
 from fmu.sumo.explorer.objects._search_context import SearchContext
+from fmu.sumo.explorer.objects._metrics import Metrics
 from fmu.sumo.explorer.objects.case import Case
 from fmu.sumo.explorer.objects.cases import Cases
 from fmu.sumo.explorer.objects.cube import Cube

diff --git a/src/fmu/sumo/explorer/objects/_metrics.py b/src/fmu/sumo/explorer/objects/_metrics.py
@@ -0,0 +1,44 @@
+class Metrics:
+    def __init__(self, search_context):
+        self._search_context = search_context
+        return
+
+    def _aggregate(self, op, **kwargs):
+        aggs = {"agg": {op: {k: v for k, v in kwargs.items() if v is not None}}}
+        qdoc = {"query": self._search_context._query,
+                "aggs": aggs,
+                "size": 0}
+        res = self._search_context._sumo.post("/search", json=qdoc).json()
+        return res["aggregations"]["agg"]
+
+    def min(self, field):
+        return self._aggregate("min", field=field)
+
+    def max(self, field):
+        return self._aggregate("max", field=field)
+
+    def avg(self, field):
+        return self._aggregate("avg", field=field)
+
+    def sum(self, field):
+        return self._aggregate("sum", field=field)
+
+    def value_count(self, field):
+        return self._aggregate("value_count", field=field)
+
+    def cardinality(self, field):
+        return self._aggregate("cardinality", field=field)
+
+    def stats(self, field):
+        return self._aggregate("stats", field=field)
+
+    def extended_stats(self, field):
+        return self._aggregate("extended_stats", field=field)
+
+    def percentiles(self, field, percents=None):
+        return self._aggregate("percentiles", field=field,
+                               percents=percents)
+
+
+
+
diff --git a/src/fmu/sumo/explorer/objects/_search_context.py b/src/fmu/sumo/explorer/objects/_search_context.py
@@ -783,6 +783,11 @@ def realizations(self):
         """Realizations from current selection."""
         return objects.Realizations(self)
 
+    @property
+    def metrics(self):
+        """Metrics for current search context."""
+        return objects.Metrics(self)
+
     @property
     def timestamps(self) -> List[str]:
         """List of unique timestamps in SearchContext"""

diff --git a/tests/test_access/tst_access_drogon_affiliate_login.py b/tests/test_access/tst_access_drogon_affiliate_login.py
@@ -3,6 +3,7 @@
     specific access rights. Running this test with your personal login
     will fail."""
 import os
+import sys
 import json
 import inspect
 import pytest
@@ -203,6 +204,9 @@ def test_read_restricted_classification_data(explorer: Explorer):
     print("Hits on restricted:", hits)
     assert hits >= 1
 
+@pytest.mark.skipif(not (sys.platform == "linux" and
+                         sys.version_info[:2] == (3, 11)),
+                    reason="Test only on single platform/version.")
 def test_aggregate_bulk(explorer: Explorer):
     """Test a bulk aggregation method"""
     print("Running test:", inspect.currentframe().f_code.co_name)

diff --git a/tests/test_access/tst_access_drogon_manage_login.py b/tests/test_access/tst_access_drogon_manage_login.py
@@ -4,6 +4,7 @@
     will fail."""
 
 import os
+import sys
 import json
 import inspect
 import pytest
@@ -65,7 +66,7 @@ def test_get_cases(explorer: Explorer):
 def test_write(explorer: Explorer):
     """Test a write method"""
     print("Running test:", inspect.currentframe().f_code.co_name)
-    cases = explorer.cases
+    cases = explorer.cases.filter(status="scratch")
     print("Number of cases: ", len(cases))
     assert len(cases) > 0
     case = cases[0]
@@ -96,10 +97,13 @@ def test_read_restricted_classification_data(explorer: Explorer):
     assert hits > 0
 
 
+@pytest.mark.skipif(not (sys.platform == "linux" and
+                         sys.version_info[:2] == (3, 11)),
+                    reason="Test only on single platform/version.")
 def test_aggregations_bulk(explorer: Explorer):
     """Test a bulk aggregation method"""
     print("Running test:", inspect.currentframe().f_code.co_name)
-    cases = explorer.cases
+    cases = explorer.cases.filter(status="scratch")
     print("Number of cases: ", len(cases))
     assert len(cases) > 0
     case = None

diff --git a/tests/test_access/tst_access_drogon_read_login.py b/tests/test_access/tst_access_drogon_read_login.py
@@ -3,6 +3,7 @@
     specific access rights. Running this test with your personal login
     will fail."""
 import os
+import sys
 import json
 import inspect
 import pytest
@@ -150,6 +151,9 @@ def test_aggregations_fast(explorer: Explorer):
     print("Length of returned aggregate object:", len(response.text))
 
 
+@pytest.mark.skipif(not (sys.platform == "linux" and
+                         sys.version_info[:2] == (3, 11)),
+                    reason="Test only on single platform/version.")
 def test_aggregate_bulk(explorer: Explorer):
     """Test a bulk aggregation method"""
     print("Running test:", inspect.currentframe().f_code.co_name)

diff --git a/tests/test_access/tst_access_drogon_write_login.py b/tests/test_access/tst_access_drogon_write_login.py
@@ -4,6 +4,7 @@
     will fail."""
 
 import os
+import sys
 import json
 import inspect
 import pytest
@@ -65,7 +66,7 @@ def test_get_cases(explorer: Explorer):
 def test_write(explorer: Explorer):
     """Test a write method"""
     print("Running test:", inspect.currentframe().f_code.co_name)
-    cases = explorer.cases
+    cases = explorer.cases.filter(status="scratch")
     print("Number of cases: ", len(cases))
     assert len(cases) > 0
     case = cases[0]
@@ -96,10 +97,13 @@ def test_read_restricted_classification_data(explorer: Explorer):
     assert hits > 0
 
 
+@pytest.mark.skipif(not (sys.platform == "linux" and
+                         sys.version_info[:2] == (3, 11)),
+                    reason="Test only on single platform/version.")
 def test_aggregate_bulk(explorer: Explorer):
     """Test a bulk aggregation method"""
     print("Running test:", inspect.currentframe().f_code.co_name)
-    cases = explorer.cases
+    cases = explorer.cases.filter(status="scratch")
     print("Number of cases: ", len(cases))
     assert len(cases) > 0
     case = None

diff --git a/tests/test_access/tst_access_no_access_login.py b/tests/test_access/tst_access_no_access_login.py
@@ -4,6 +4,7 @@
     will fail."""
 
 import os
+import sys
 import json
 import inspect
 import pytest
@@ -157,6 +158,9 @@ def test_get_message_log_truncate(explorer: Explorer):
         print("Unexpected response: ", response.text)
 
 
+@pytest.mark.skipif(not (sys.platform == "linux" and
+                         sys.version_info[:2] == (3, 11)),
+                    reason="Test only on single platform/version.")
 def test_aggregate_bulk(explorer: Explorer):
     """Test a bulk aggregation method"""
     print("Running test:", inspect.currentframe().f_code.co_name)

diff --git a/tests/test_explorer.py b/tests/test_explorer.py
@@ -196,14 +196,14 @@ def test_case_surfaces_type(test_case: Case):
 
 def test_case_surfaces_size(test_case: Case):
     """Test that Case.surfaces has the correct size"""
-    assert len(test_case.surfaces) == 219
+    assert len(test_case.surfaces) == 271
 
 
 def test_case_surfaces_filter(test_case: Case):
     """Test that Case.surfaces has the correct size"""
     # filter on iteration stage
     agg_surfs = test_case.surfaces.filter(stage="iteration")
-    assert len(agg_surfs) == 7
+    assert len(agg_surfs) == 59
 
     agg_surfs = test_case.surfaces.filter(aggregation=True)
     assert len(agg_surfs)