replace xlswriter (#182)

* replacing excel writer.save() with writer.close * replace another ExcelWriter.save with close * replace pd.append with concat * Change link address to docs * add missing end quotation mark * formatting * Update tests & group corr sheet by string --------- Co-authored-by: Trine Alsos <[email protected]> Co-authored-by: mferrera <[email protected]>
equinor · Apr 17, 2023 · 5db57b6 · 5db57b6
1 parent 2dcdaec
commit 5db57b6
Show file tree

Hide file tree

Showing 6 changed files with 29 additions and 21 deletions.
diff --git a/src/fmu/tools/rms/create_rft_ertobs.py b/src/fmu/tools/rms/create_rft_ertobs.py
@@ -86,7 +86,7 @@ def check_and_parse_config(config: Dict[str, Any]) -> Dict[str, Any]:
     required_input_dframe_columns = ["DATE", "MD", "WELL_NAME", "PRESSURE"]
     if "input_dframe" not in config and "input_file" in config:
         input_dframe = pd.read_csv(config["input_file"])
-        input_dframe["DATE"] = pd.to_datetime(input_dframe["DATE"], dayfirst=True)
+        input_dframe["DATE"] = pd.to_datetime(input_dframe["DATE"])
         config["input_dframe"] = input_dframe
     missing_columns = set(required_input_dframe_columns) - set(
         config["input_dframe"].columns
@@ -400,7 +400,7 @@ def ertobs_df_to_files(
     if "ZONE" not in dframe:
         dframe["ZONE"] = ""
 
-    dframe["DATE"] = pd.to_datetime(dframe["DATE"], dayfirst=True)
+    dframe["DATE"] = pd.to_datetime(dframe["DATE"])
     dframe["REPORT_STEP"] = (
         dframe.groupby("WELL_NAME")["DATE"]
         .rank(method="dense", ascending=True)

diff --git a/src/fmu/tools/sensitivities/_excel2dict.py b/src/fmu/tools/sensitivities/_excel2dict.py
@@ -286,7 +286,9 @@ def _excel2dict_onebyone(input_filename, sheetnames=None):
     inputdict["sensitivities"] = OrderedDict()
     designinput = pd.read_excel(input_filename, design_inp_sheet, engine="openpyxl")
     designinput.dropna(axis=0, how="all", inplace=True)
-    designinput = designinput.loc[:, ~designinput.columns.str.contains("^Unnamed")]
+    designinput = designinput.loc[
+        :, ~designinput.columns.astype(str).str.contains("^Unnamed")
+    ]
 
     # First column with parameter names should have spaces stripped,
     # but we need to preserve NaNs:
@@ -385,7 +387,9 @@ def _read_defaultvalues(filename, sheetname):
         filename, sheetname, header=0, index_col=0, engine="openpyxl"
     )
     default_df.dropna(axis=0, how="all", inplace=True)
-    default_df = default_df.loc[:, ~default_df.columns.str.contains("^Unnamed")]
+    default_df = default_df.loc[
+        :, ~default_df.columns.astype(str).str.contains("^Unnamed")
+    ]
 
     # Strip spaces before and after parameter names, if they are there
     # it is probably invisible user errors in Excel.
@@ -423,7 +427,9 @@ def _read_dependencies(filename, sheetname, from_parameter):
         filename, sheetname, dtype=str, na_values="", engine="openpyxl"
     )
     depend_df.dropna(axis=0, how="all", inplace=True)
-    depend_df = depend_df.loc[:, ~depend_df.columns.str.contains("^Unnamed")]
+    depend_df = depend_df.loc[
+        :, ~depend_df.columns.astype(str).str.contains("^Unnamed")
+    ]
 
     if from_parameter in depend_df.keys():
         depend_dict["from_values"] = depend_df[from_parameter].tolist()
@@ -454,7 +460,9 @@ def _read_background(inp_filename, bck_sheet):
     paramdict = OrderedDict()
     bck_input = pd.read_excel(inp_filename, bck_sheet, engine="openpyxl")
     bck_input.dropna(axis=0, how="all", inplace=True)
-    bck_input = bck_input.loc[:, ~bck_input.columns.str.contains("^Unnamed")]
+    bck_input = bck_input.loc[
+        :, ~bck_input.columns.astype(str).str.contains("^Unnamed")
+    ]
 
     backdict["correlations"] = None
     if "corr_sheet" in bck_input.keys():

diff --git a/src/fmu/tools/sensitivities/create_design.py b/src/fmu/tools/sensitivities/create_design.py
@@ -197,7 +197,7 @@ def to_xlsx(
         defaults.to_excel(
             xlsxwriter, sheet_name=defaultsheet, index=False, header=False
         )
-        xlsxwriter.save()
+        xlsxwriter.close()
         print(
             f"A total of {len(self.designvalues['REAL'])} realizations were generated"
         )
@@ -263,7 +263,7 @@ def background_to_excel(self, filename, backgroundsheet="Background"):
         self.backgroundvalues.to_excel(
             xlsxwriter, sheet_name=backgroundsheet, index=False, header=True
         )
-        xlsxwriter.save()
+        xlsxwriter.close()
         print(f"Backgroundvalues written to {filename}")
 
     def _add_sensitivity(self, sensitivity):
@@ -273,7 +273,7 @@ def _add_sensitivity(self, sensitivity):
             sensitivity of class Scenario, MonteCarlo or Extern
         """
         existing_values = self.designvalues.copy()
-        self.designvalues = existing_values.append(sensitivity.sensvalues, sort=False)
+        self.designvalues = pd.concat([existing_values, sensitivity.sensvalues])
 
     def _fill_with_background_values(self):
         """Substituting NaNs with background values if existing.
@@ -309,7 +309,7 @@ def _fill_with_background_values(self):
                                 )
                             )
                 existing_values = result_values.copy()
-                result_values = existing_values.append(temp_df, sort=False)
+                result_values = pd.concat([existing_values, temp_df])
 
             result_values = result_values.drop(["index"], axis=1)
             self.designvalues = result_values
@@ -688,7 +688,7 @@ def generate(self, realnums, parameters, seedvalues, corrdict):
             df_params["corr_sheet"].fillna("nocorr", inplace=True)
             df_params.reset_index(inplace=True)
             df_params.rename(columns={"index": "param_name"}, inplace=True)
-            param_groups = df_params.groupby(["corr_sheet"])
+            param_groups = df_params.groupby("corr_sheet")
 
             for correl, group in param_groups:
                 param_dict = OrderedDict()
@@ -897,8 +897,8 @@ def _printwarning(corrgroup):
         "This also means non-correlated parameters do not "
         "have to be included in correlation matrix. \n "
         "See documentation: \n"
-        "https://sdp.equinor.com/wikidocs/FMU/lib/fmu/tools/"
-        "html/examples.html#create-design-matrix-for-"
+        "https://equinor.github.io/fmu-tools/"
+        "fmudesign.html#create-design-matrix-for-"
         "one-by-one-sensitivities\n"
         "\n"
         "####################################################\n"

diff --git a/tests/rms/test_create_rft_ertobs.py b/tests/rms/test_create_rft_ertobs.py
@@ -485,7 +485,7 @@ def test_date_parsing(tmpdir):
     tmpdir.chdir()
     pd.DataFrame(
         columns=["DATE", "MD", "WELL_NAME", "PRESSURE"],
-        data=[["01 03 2009", "100", "A-1", "123"]],
+        data=[["2009-03-01", "100", "A-1", "123"]],
     ).to_csv("inputframe.csv", header=True, index=False)
 
     minimal_config = {"input_file": "inputframe.csv"}

diff --git a/tests/sensitivities/test_excel2dict.py b/tests/sensitivities/test_excel2dict.py
@@ -42,7 +42,7 @@ def test_excel2dict_design(tmpdir):
         writer, sheet_name="designinput", index=False, header=None
     )
     defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False, header=None)
-    writer.save()
+    writer.close()
 
     dict_design = excel2dict_design("designinput.xlsx")
     assert isinstance(dict_design, dict)
@@ -71,7 +71,7 @@ def test_excel2dict_design(tmpdir):
         writer, sheet_name="Design_input", index=False, header=None
     )
     defaultvalues.to_excel(writer, sheet_name="DefaultValues", index=False, header=None)
-    writer.save()
+    writer.close()
 
     dict_design = excel2dict_design("designinput2.xlsx")
     assert isinstance(dict_design, dict)
@@ -108,7 +108,7 @@ def test_duplicate_sensname_exception(tmpdir):
         writer, sheet_name="designinput", index=False, header=None
     )
     defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False, header=None)
-    writer.save()
+    writer.close()
 
     with pytest.raises(
         ValueError, match="Two sensitivities can not share the same sensname"
@@ -148,7 +148,7 @@ def test_strip_spaces(tmpdir):
     defaultvalues_spacious.to_excel(
         writer, sheet_name="defaultvalues", index=False, header=None
     )
-    writer.save()
+    writer.close()
 
     dict_design = excel2dict_design("designinput_spaces.xlsx")
     assert list(dict_design["sensitivities"].keys())[0] == "rms_seed"
@@ -183,7 +183,7 @@ def test_mixed_senstype_exception(tmpdir):
         writer, sheet_name="designinput", index=False, header=None
     )
     defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False, header=None)
-    writer.save()
+    writer.close()
 
     with pytest.raises(ValueError, match="contains more than one sensitivity type"):
         excel2dict_design("designinput4.xlsx")
@@ -231,7 +231,7 @@ def test_background_sheet(tmpdir):
     )
     defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False)
     background.to_excel(writer, sheet_name="backgroundsheet", index=False, header=None)
-    writer.save()
+    writer.close()
 
     dict_design = excel2dict_design("designinput.xlsx")
 

diff --git a/tests/sensitivities/test_use_cases.py b/tests/sensitivities/test_use_cases.py
@@ -73,7 +73,7 @@ def test_prediction_rejection_sampled_ensemble(tmpdir):
         ],
     ).to_excel(writer, sheet_name="design_input", index=False)
     defaultvalues.to_excel(writer, sheet_name="defaultvalues", index=False)
-    writer.save()
+    writer.close()
 
     dict_design = excel2dict_design("designinput.xlsx")
     design = DesignMatrix()