New dazls updates

.gitattributes

@@ -0,0 +1 @@
+*.sav filter=lfs diff=lfs merge=lfs -text

openstef/data/dazls_stored_new.sav.license

@@ -0,0 +1,3 @@
+SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]>
+
+SPDX-License-Identifier: MPL-2.0

openstef/data/dazls_stored_new_domain_model_features.z

@@ -0,0 +1,2 @@
+x^%�;
+�0DS����#U�}� $c���:i��\�׫h�7̼��=O��@
�����/��I�WB�h#�>W�PlOM��a��S�+y�fZ�<���b:ƎIQ}j����I�c|7#

openstef/model/regressors/dazls.py

@@ -155,3 +155,23 @@ def score(self, truth, prediction):
         rmse = (mean_squared_error(truth, prediction)) ** 0.5
         r2_score_value = r2_score(truth, prediction)
         return rmse, r2_score_value
+
+    def __str__(self):
+        """String method of the DAZLs model, provides a summary of the model for easy inspection.
+
+        Returns:
+            Summary represented by a string
+
+        """
+        summary_str = (
+            f"{self.__name__} model summary:\n\n"
+            f"Domain Model: {self.domain_model} \n"
+            f"\tInput columns: {self.domain_model_input_columns} \n"
+            f"\tScaler: {self.domain_model_scaler} \n\n"
+            f"Adaptation Model: {self.adaptation_model} \n"
+            f"\tInput columns: {self.adaptation_model_input_columns} \n"
+            f"\tScaler: {self.adaptation_model_scaler} \n\n"
+            f"Target columns: {self.target_columns}"
+        )
+
+        return summary_str

openstef/pipeline/create_component_forecast.py

@@ -12,8 +12,10 @@
 from openstef.enums import ForecastType
 from openstef.model.regressors.dazls import Dazls
 
+import numpy as np
+
 # Set the path for the Dazls stored model
-DAZLS_STORED = PROJECT_ROOT / "openstef" / "data" / "dazls_stored.sav"
+DAZLS_STORED = str(PROJECT_ROOT / "openstef" / "data" / "dazls_stored_new_")
 
 
 def create_input(
@@ -37,7 +39,7 @@ def create_input(
     input_df = (
         weather_data[["radiation", "windspeed_100m"]]
         .merge(
-            input_data[["forecast"]].rename(columns={"forecast": "total_substation"}),
+            input_data[["forecast"]].rename(columns={"forecast": "total_load"}),
             how="inner",
             right_index=True,
             left_index=True,
@@ -53,12 +55,20 @@ def create_input(
     input_df["hour"] = input_df.index.hour
     input_df["minute"] = input_df.index.minute
 
-    input_df["var0"] = input_df["total_substation"].var()
+    input_df["var0"] = input_df["total_load"].var()
     input_df["var1"] = input_df["radiation"].var()
     input_df["var2"] = input_df["windspeed_100m"].var()
 
-    input_df["sem0"] = input_df["total_substation"].sem()
+    input_df["sem0"] = input_df["total_load"].sem()
     input_df["sem1"] = input_df["radiation"].sem()
+    input_df["sem2"] = input_df["windspeed_100m"].sem()
+
+    # Features for the new model
+    # Periodic Month feature
+    c = (1 / 11) * np.pi - (1 / 365)
+    n = pd.to_datetime(input_df.index).strftime("%m").tolist()
+    n = np.array([float(x) for x in n])
+    input_df["month_ff"] = np.sin(c * (n - 1))
 
     return input_df
 
@@ -92,9 +102,29 @@ def create_components_forecast_pipeline(
     try:
         input_data = create_input(pj, input_data, weather_data)
 
-        # Save and load the model as .sav file
+        # Save and load the model as .sav file (or as .z file)
         # For the code contact: [email protected]
-        dazls_model: Dazls = joblib.load(DAZLS_STORED)
+        dazls_model = Dazls()
+        dazls_model.domain_model = joblib.load(DAZLS_STORED + "domain_model.z")
+        dazls_model.domain_model_scaler = joblib.load(
+            DAZLS_STORED + "domain_model_scaler.z"
+        )
+        dazls_model.domain_model_input_columns = joblib.load(
+            DAZLS_STORED + "domain_model_features.z"
+        )
+
+        dazls_model.adaptation_model = joblib.load(DAZLS_STORED + "adaptation_model.z")
+        dazls_model.adaptation_model_scaler = joblib.load(
+            DAZLS_STORED + "adaptation_model_scaler.z"
+        )
+        dazls_model.adaptation_model_input_columns = joblib.load(
+            DAZLS_STORED + "adaptation_model_features.z"
+        )
+
+        dazls_model.target_columns = joblib.load(DAZLS_STORED + "target.z")
+        dazls_model.target_scaler = joblib.load(DAZLS_STORED + "target_scaler.z")
+
+        print(dazls_model)
 
         # Use the predict function of Dazls model
         # As input data we use the input_data function which takes into consideration what we want as an input for the forecast and what Dazls can accept as an input
@@ -119,7 +149,7 @@ def create_components_forecast_pipeline(
 
         # Make forecast for the component: "forecast_other"
         forecasts["forecast_other"] = (
-            input_data["total_substation"]
+            input_data["total_load"]
             - forecasts["forecast_solar"]
             - forecasts["forecast_wind_on_shore"]
         )

requirements.txt

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 holidays==0.21
+joblib==1.3.2
 lightgbm~=3.3
 matplotlib~=3.7
 mlflow~=2.3
@@ -16,4 +17,4 @@ scikit-learn~=1.3
 scipy~=1.10
 statsmodels~=0.13.5
 structlog~=23.1
-xgboost~=1.7
+xgboost~=1.7

test/unit/pipeline/test_create_component_forecast.py

@@ -16,6 +16,10 @@
     create_components_forecast_pipeline,
 )
 
+from openstef.model.regressors.dazls import (
+    Dazls,
+)
+
 
 class TestComponentForecast(BaseTestCase):
     def setUp(self) -> None:
@@ -29,10 +33,32 @@ def test_load_dazls_model(self):
         """
 
         old_model_file = PROJECT_ROOT / "openstef/data/dazls_stored_3.2.49.sav"
-        new_model_file = PROJECT_ROOT / "openstef/data/dazls_stored.sav"
+        new_model_file = str(PROJECT_ROOT / "openstef/data/dazls_stored_new_")
 
         self.assertRaises(Exception, joblib.load, old_model_file)
-        dazls_model = joblib.load(new_model_file)
+        dazls_model = Dazls()
+
+        dazls_model.domain_model = joblib.load(new_model_file + "domain_model.z")
+        dazls_model.domain_model_scaler = joblib.load(
+            new_model_file + "domain_model_scaler.z"
+        )
+        dazls_model.domain_model_input_columns = joblib.load(
+            new_model_file + "domain_model_features.z"
+        )
+
+        dazls_model.adaptation_model = joblib.load(
+            new_model_file + "adaptation_model.z"
+        )
+        dazls_model.adaptation_model_scaler = joblib.load(
+            new_model_file + "adaptation_model_scaler.z"
+        )
+        dazls_model.adaptation_model_input_columns = joblib.load(
+            new_model_file + "adaptation_model_features.z"
+        )
+
+        dazls_model.target_columns = joblib.load(new_model_file + "target.z")
+        dazls_model.target_scaler = joblib.load(new_model_file + "target_scaler.z")
+
         assert dazls_model
 
     def test_component_forecast_pipeline_happy_flow(self):