Added loglevel in (new) appsetting.

openstef/app_settings.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
+#
+# SPDX-License-Identifier: MPL-2.0
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class AppSettings(BaseSettings):
+    """Global app settings."""
+
+    # Logging settings.
+    log_level: str = Field("INFO", description="Log level used for logging statements.")
+
+    model_config = SettingsConfigDict(
+        env_prefix="openstef_", env_file=".env", extra="ignore"
+    )

openstef/feature_engineering/data_preparation.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Alliander N.V. <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from abc import ABC, abstractmethod
 from datetime import timedelta
 from typing import Optional
@@ -20,6 +21,7 @@
 )
 from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.pipeline.utils import generate_forecast_datetime_range
+from openstef.settings import Settings
 
 
 class AbstractDataPreparation(ABC):
@@ -120,6 +122,11 @@ def prepare_train_data(self, data: pd.DataFrame) -> pd.DataFrame:
     def prepare_forecast_data(
         self, data: pd.DataFrame
     ) -> tuple[pd.DataFrame, pd.DataFrame]:
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         self.check_model()
         # Prep forecast input by selecting only the forecast datetime interval (this is much smaller than the input range)

openstef/feature_engineering/general.py

@@ -3,10 +3,14 @@
 # SPDX-License-Identifier: MPL-2.0
 """This modelu contains various helper functions."""
 
+import logging
+
 import numpy as np
 import pandas as pd
 import structlog
 
+from openstef.settings import Settings
+
 
 def add_missing_feature_columns(
     input_data: pd.DataFrame, features: list[str]
@@ -30,6 +34,11 @@ def add_missing_feature_columns(
         Input dataframe with missing columns filled with ``np.N=nan``.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if features is None:
@@ -61,6 +70,11 @@ def remove_non_requested_feature_columns(
         Model input data with features.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     if requested_features is None:

openstef/feature_engineering/weather_features.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: MPL-2.0
 
 """This module contains all wheather related functions used for feature engineering."""
+import logging
 from typing import Union
 
 import numpy as np
@@ -12,7 +13,13 @@
 from pvlib.location import Location
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 

openstef/metrics/reporter.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 """Defines reporter class."""
+import logging
 import os
 import warnings
 from dataclasses import dataclass
@@ -16,6 +17,7 @@
 from openstef.metrics import figure
 from openstef.metrics.metrics import bias, mae, nsme, r_mae, rmse
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 @dataclass
@@ -167,6 +169,11 @@ def get_metrics(y_pred: np.array, y_true: np.array) -> dict:
     def write_report_to_disk(report: Report, report_folder: str):
         """Write report to disk; e.g. for viewing report of latest models using grafana."""
         # Initialize logger and serializer
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         logger = structlog.get_logger(__name__)
         if report_folder:
             # create path if does not exist

openstef/model/confidence_interval_applicator.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from datetime import datetime
 
 import numpy as np
@@ -11,12 +12,18 @@
 
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.exceptions import ModelWithoutStDev
+from openstef.settings import Settings
 
 
 class ConfidenceIntervalApplicator:
     def __init__(self, model: RegressorMixin, forecast_input_data: pd.DataFrame):
         self.model = model
         self.forecast_input_data = forecast_input_data
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
 
     def add_confidence_interval(

openstef/model/model_creator.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import structlog
@@ -13,7 +14,13 @@
 from openstef.model.regressors.regressor import OpenstfRegressor
 from openstef.model.regressors.xgb import XGBOpenstfRegressor
 from openstef.model.regressors.xgb_quantile import XGBQuantileOpenstfRegressor
+from openstef.settings import Settings
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 valid_model_kwargs = {

openstef/model/serializer.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 import json
+import logging
 import os
 import shutil
 from datetime import datetime
@@ -20,10 +21,16 @@
 from openstef.data_classes.model_specifications import ModelSpecificationDataClass
 from openstef.metrics.reporter import Report
 from openstef.model.regressors.regressor import OpenstfRegressor
+from openstef.settings import Settings
 
 
 class MLflowSerializer:
     def __init__(self, mlflow_tracking_uri: str):
+        structlog.configure(
+            wrapper_class=structlog.make_filtering_bound_logger(
+                logging.getLevelName(Settings.log_level)
+            )
+        )
         self.logger = structlog.get_logger(self.__class__.__name__)
         mlflow.set_tracking_uri(mlflow_tracking_uri)
         self.logger.debug(f"MLflow tracking uri at init= {mlflow_tracking_uri}")

openstef/monitoring/teams.py

@@ -1,13 +1,16 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from typing import Union
 
 import pandas as pd
 import pymsteams
 import structlog
 from pymsteams import cardsection
 
+from openstef.settings import Settings
+
 
 def post_teams(
     msg: Union[str, dict],
@@ -38,6 +41,11 @@ def post_teams(
     Note:
         This function is namespace-specific.
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     # If no url is passed, give warning and don't send teams message
     if url is None:

openstef/pipeline/create_basecase_forecast.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 from pathlib import Path
 
 import pandas as pd
@@ -18,6 +19,7 @@
     add_components_base_case_forecast,
     add_prediction_job_properties_to_forecast,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
 
 MODEL_LOCATION = Path(".")
@@ -42,6 +44,11 @@ def create_basecase_forecast_pipeline(
         NoRealisedLoadError: When no realised load for given datetime range.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     logger.info("Preprocessing data for basecase forecast")

openstef/pipeline/create_component_forecast.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: MPL-2.0
 
+import logging
+
 import joblib
 import numpy as np
 import pandas as pd
@@ -12,6 +14,7 @@
 from openstef.data_classes.prediction_job import PredictionJobDataClass
 from openstef.enums import ForecastType
 from openstef.model.regressors.dazls import Dazls
+from openstef.settings import Settings
 
 # Set the path for the Dazls stored model
 DAZLS_STORED = str(
@@ -95,6 +98,11 @@ def create_components_forecast_pipeline(
         "algtype"
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
     logger.info("Make components prediction", pid=pj["id"])
 
@@ -124,7 +132,6 @@ def create_components_forecast_pipeline(
         dazls_model.target_columns = joblib.load(DAZLS_STORED + "target.z")
         dazls_model.target_scaler = joblib.load(DAZLS_STORED + "target_scaler.z")
 
-        logger = structlog.get_logger(__name__)
         logger.info("DAZLS model loaded", dazls_model=str(dazls_model))
 
         # Use the predict function of Dazls model

openstef/pipeline/create_forecast.py

@@ -1,6 +1,8 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
+
 import pandas as pd
 import structlog
 
@@ -18,6 +20,7 @@
     add_prediction_job_properties_to_forecast,
     sort_quantiles,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
 
 
@@ -85,6 +88,11 @@ def create_forecast_pipeline_core(
         InputDataOngoingZeroFlatlinerError: When all recent load measurements are zero.
 
     """
+    structlog.configure(
+        wrapper_class=structlog.make_filtering_bound_logger(
+            logging.getLevelName(Settings.log_level)
+        )
+    )
     logger = structlog.get_logger(__name__)
 
     fallback_strategy = "extreme_day"  # this can later be expanded

openstef/pipeline/optimize_hyperparameters.py

@@ -1,8 +1,9 @@
 # SPDX-FileCopyrightText: 2017-2023 Contributors to the OpenSTEF project <[email protected]> # noqa E501>
 #
 # SPDX-License-Identifier: MPL-2.0
+import logging
 import os
-from typing import Any, Union
+from typing import Any
 
 import optuna
 import pandas as pd
@@ -26,11 +27,17 @@
     DEFAULT_TRAIN_HORIZONS_HOURS,
     train_model_pipeline_core,
 )
+from openstef.settings import Settings
 from openstef.validation import validation
 
 optuna.logging.enable_propagation()  # Propagate logs to the root logger.
 optuna.logging.disable_default_handler()  # Stop showing logs in sys.stderr.
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 # See https://optuna.readthedocs.io/en/stable/reference/generated/optuna.study.Study.html#optuna.study.Study.optimize

openstef/pipeline/train_model.py

@@ -23,6 +23,7 @@
 from openstef.model.serializer import MLflowSerializer
 from openstef.model.standard_deviation_generator import StandardDeviationGenerator
 from openstef.model_selection.model_selection import split_data_train_validation_test
+from openstef.settings import Settings
 from openstef.validation import validation
 
 DEFAULT_TRAIN_HORIZONS_HOURS: list[float] = [0.25, 47.0]
@@ -31,6 +32,11 @@
 DEFAULT_EARLY_STOPPING_ROUNDS: int = 10
 PENALTY_FACTOR_OLD_MODEL: float = 1.2
 
+structlog.configure(
+    wrapper_class=structlog.make_filtering_bound_logger(
+        logging.getLevelName(Settings.log_level)
+    )
+)
 logger = structlog.get_logger(__name__)
 
 
@@ -180,8 +186,6 @@ def train_model_pipeline_core(
         - Datasets (tuple[pd.DataFrmae, pd.DataFrame, pd.Dataframe): The train, validation and test sets
 
     """
-    logger = structlog.get_logger(__name__)
-
     # Call common pipeline
     (
         model,