First stab at using ecl2df for summary file extraction

Works, but there is a lot of cleanup to be done in realization.py
and ensemble.py to fully remove the eclsum objects from fmu-ensemble.

setup.cfg

@@ -8,6 +8,3 @@ exclude = docs,
 
 [aliases]
 test = pytest
-
-[tool:pytest]
-addopts = --verbose -x

src/fmu/ensemble/realization.py

@@ -33,14 +33,6 @@
 from .util.rates import compute_volumetric_rates
 from .util.dates import unionize_smry_dates
 
-HAVE_ECL2DF = False
-try:
-    import ecl2df
-
-    HAVE_ECL2DF = True
-except ImportError:
-    HAVE_ECL2DF = False
-
 fmux = Interaction()
 logger = fmux.basiclogger(__name__)
 
@@ -856,33 +848,51 @@ def get_eclfiles(self):
         Returns:
             ecl2df.EclFiles. None if nothing found
         """
-        if not HAVE_ECL2DF:
-            logger.warning("ecl2df not installed. Skipping")
-            return None
-        data_file_row = self.files[self.files["FILETYPE"] == "DATA"]
+        data_file_rows = self.files[self.files["FILETYPE"] == "DATA"]
         data_filename = None
-        if len(data_file_row) == 1:
-            data_filename = data_file_row["FULLPATH"].values[0]
+        unsmry_file_rows = self.files[self.files["FILETYPE"] == "UNSMRY"]
+        unsmry_filename = None
+        if len(data_file_rows) == 1:
+            data_filename = data_file_rows["FULLPATH"].values[0]
+        elif len(unsmry_file_rows) == 1:
+            unsmry_filename = unsmry_file_rows["FULLPATH"].values[0]
+            # We construct the DATA file, even though it might not exist:
+            data_filename = unsmry_filename.replace(".UNSMRY", ".DATA")
         elif self._autodiscovery:
             data_fileguess = os.path.join(self._origpath, "eclipse/model", "*.DATA")
             data_filenamelist = glob.glob(data_fileguess)
             if not data_filenamelist:
-                return None  # No filename matches *DATA
+                return None  # No filename matches *DATA or *UNSMRY
             if len(data_filenamelist) > 1:
                 logger.warning(
                     (
                         "Multiple DATA files found, "
                         "consider turning off auto-discovery"
                     )
                 )
-            data_filename = data_filenamelist[0]
-            self.find_files(data_filename)
+            if data_filenamelist:
+                data_filename = data_filenamelist[0]
+                self.find_files(data_filename)
+
+            unsmry_fileguess = os.path.join(self._origpath, "eclipse/model", "*.UNSMRY")
+            unsmry_filenamelist = glob.glob(unsmry_fileguess)
+            if not unsmry_filenamelist:
+                return None  # No filename matches
+            if len(unsmry_filenamelist) > 1:
+                logger.warning(
+                    "Multiple UNSMRY files found, consider turning off auto-discovery"
+                )
+            unsmry_filename = unsmry_filenamelist[0]
+            self.find_files(unsmry_filename)
+
         else:
-            # There is no DATA file to be found.
-            logger.warning("No DATA file found!")
+            logger.warning("No DATA and/or UNSMRY file found!")
             return None
         if not os.path.exists(data_filename):
-            return None
+            if unsmry_filename is not None:
+                return ecl2df.EclFiles(unsmry_filename.replace(".UNSMRY", ".DATA"))
+            else:
+                return None
         return ecl2df.EclFiles(data_filename)
 
     def get_eclsum(self, cache=True, include_restart=True):
@@ -951,20 +961,34 @@ def get_eclsum(self, cache=True, include_restart=True):
         return eclsum
 
     def load_smry(self, **kwargs):
+        """Wrap around get_smry(), but also cache the result"""
 
-        dframe = self.get_smry(**kwargs)
+        # This change of indexing is peculiar for load_smry() vs get_smry().
+        # It might change in fmu-ensemble 2.0 to always return a datetime64
+        # index.
+        dframe = self.get_smry(**kwargs).reset_index()
 
+        cachename = None
         # Cache the result for supported time indices:
-        if str(kwargs["time_index"]) in ["daily", "weekly", "monthly", "yearly"]:
-            localpath = (
-                "share/results/tables/unsmry--" + str(kwargs["time_index"]) + ".csv"
-            )
+        if "time_index" not in kwargs or kwargs["time_index"] is None:
+            cachename = "raw"
+        elif isinstance(kwargs["time_index"], list):
+            cachename = "custom"
+        elif str(kwargs["time_index"]) in [
+            "raw",
+            "first",
+            "last",
+            "report",
+            "daily",
+            "weekly",
+            "monthly",
+            "yearly",
+        ]:
+            cachename = kwargs["time_index"]
+
+        if cachename:
+            localpath = "share/results/tables/unsmry--" + cachename + ".csv"
             self.data[localpath] = dframe
-
-        # Do this to ensure that we cut the rope to the EclSum object
-        # Can be critical for garbage collection
-        if not kwargs.get("cache_eclsum", True):
-            self._eclsum = None
         return dframe
 
     def get_smry(
@@ -1006,17 +1030,31 @@ def get_smry(
         Returns empty dataframe if there is no summary file, or if the
         column_keys are not existing.
         """
-        return ecl2df.summary.df(
-            self.get_eclfiles(),
-            time_index=time_index,
-            column_keys=column_keys,
-            start_date=start_date,
-            end_date=end_date,
-            include_restart=include_restart,
-            params=False,
-            paramfile=None,
-            datetime=datetimeindex,
-        )
+        try:
+            dframe = ecl2df.summary.df(
+                self.get_eclfiles(),
+                time_index=time_index,
+                column_keys=column_keys,
+                start_date=start_date,
+                end_date=end_date,
+                include_restart=include_restart,
+                params=False,
+                paramfile=None,
+                datetime=datetimeindex,
+            )
+            if cache_eclsum:
+                if self.get_eclfiles():
+                    # This is necessary for tests to pass, but might not
+                    # be the way to do it since ecl2df should take full
+                    # responsibility for the eclsum objects.
+                    self._eclsum = self.get_eclfiles().get_eclsum()
+            else:
+                # Do this to ensure that we cut the rope to the EclSum object
+                # Can be critical for garbage collection
+                self._eclsum = None
+            return dframe
+        except FileNotFoundError:
+            return pd.DataFrame()
 
     def get_smry_meta(self, column_keys=None):
         """

tests/test_ecl2df.py

@@ -2,27 +2,18 @@
 
 import os
 
-import pytest
+import ecl2df
 
 from fmu.ensemble import etc
 from fmu.ensemble import ScratchEnsemble, ScratchRealization
 
-HAVE_ECL2DF = True
-try:
-    import ecl2df
-except ImportError:
-    HAVE_ECL2DF = False
-
 fmux = etc.Interaction()
 logger = fmux.basiclogger(__name__, level="INFO")
 
 
 def test_ecl2df_real():
     """Check that we can utilize ecl2df on single realizations"""
 
-    if not HAVE_ECL2DF:
-        pytest.skip()
-
     if "__file__" in globals():
         # Easen up copying test code into interactive sessions
         testdir = os.path.dirname(os.path.abspath(__file__))
@@ -50,8 +41,6 @@ def test_reek():
     reekens = ScratchEnsemble(
         "reektest", testdir + "/data/testensemble-reek001/" + "realization-*/iter-0"
     )
-    if not HAVE_ECL2DF:
-        pytest.skip()
 
     def extract_compdat(kwargs):
         """Callback fnction to extract compdata data using ecl2df
@@ -91,8 +80,6 @@ def get_smry(kwargs):
     reekens = ScratchEnsemble(
         "reektest", testdir + "/data/testensemble-reek001/" + "realization-*/iter-0"
     )
-    if not HAVE_ECL2DF:
-        pytest.skip()
 
     callback_smry = reekens.apply(get_smry, column_keys="FOPT", time_index="yearly")
     direct_smry = reekens.get_smry(column_keys="FOPT", time_index="yearly")

tests/test_ensemble.py

@@ -155,7 +155,6 @@ def test_reek001(tmpdir):
         ]
     )
     assert len(reekensemble) == 5
-    print(reekensemble.files)
     assert len(reekensemble.files) == 24
 
     # File discovery must be repeated for the newly added realizations