Merge pull request #898 from ekatef/keep_exceptions_logs

Keep exceptions logs

doc/release_notes.rst

@@ -12,7 +12,12 @@ Upcoming release
 Please add descriptive release notes like in `PyPSA-Eur <https://github.com/PyPSA/pypsa-eur/blob/master/doc/release_notes.rst>`__.
 E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does.
 
-**New Features and major Changes**
+* Keep all traceback in logs. `PR #898 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/898>`__
+
+PyPSA-Earth 0.2.3
+=================
+
+**New Features and major Changes (19th October 2023)**
 
 
 

scripts/_helpers.py

@@ -8,6 +8,7 @@
 import logging
 import os
 import subprocess
+import sys
 from pathlib import Path
 
 import country_converter as coco
@@ -22,6 +23,47 @@
 
 REGION_COLS = ["geometry", "name", "x", "y", "country"]
 
+logger = logging.getLogger(__name__)
+
+
+def handle_exception(exc_type, exc_value, exc_traceback):
+    """
+    Customise errors traceback.
+    """
+    tb = exc_traceback
+    while tb.tb_next:
+        tb = tb.tb_next
+    flname = tb.tb_frame.f_globals.get("__file__")
+    funcname = tb.tb_frame.f_code.co_name
+
+    if issubclass(exc_type, KeyboardInterrupt):
+        logger.error(
+            "Manual interruption %r, function %r: %s",
+            flname,
+            funcname,
+            exc_value,
+        )
+    else:
+        logger.error(
+            "An error happened in module %r, function %r: %s",
+            flname,
+            funcname,
+            exc_value,
+            exc_info=(exc_type, exc_value, exc_traceback),
+        )
+
+
+def create_logger(logger_name, level=logging.INFO):
+    """
+    Create a logger for a module and adds a handler needed to capture in logs
+    traceback from exceptions emerging during the workflow.
+    """
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(level)
+    handler = logging.StreamHandler(stream=sys.stdout)
+    logger.addHandler(handler)
+    sys.excepthook = handle_exception
+
 
 def read_osm_config(*args):
     """

scripts/add_electricity.py

@@ -94,12 +94,12 @@
 import powerplantmatching as pm
 import pypsa
 import xarray as xr
-from _helpers import configure_logging, read_csv_nafix, update_p_nom_max
+from _helpers import configure_logging, create_logger, read_csv_nafix, update_p_nom_max
 from powerplantmatching.export import map_country_bus
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def normed(s):

scripts/add_extra_components.py

@@ -58,7 +58,7 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
 from add_electricity import (
     _add_missing_carriers_from_costs,
     add_nice_carrier_names,
@@ -67,7 +67,7 @@
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def attach_storageunits(n, costs, config):

scripts/augmented_line_connections.py

@@ -35,14 +35,14 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
 from add_electricity import load_costs
 from base_network import _set_dc_underwater_fraction
 from networkx.algorithms import complement
 from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation
 from pypsa.geo import haversine_pts
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 # Functions

scripts/base_network.py

@@ -67,12 +67,12 @@
 import shapely.prepared
 import shapely.wkt
 import yaml
-from _helpers import configure_logging, read_csv_nafix
+from _helpers import configure_logging, create_logger, read_csv_nafix
 from scipy.sparse import csgraph
 from shapely.geometry import LineString, Point
 from shapely.ops import unary_union
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def _get_oid(df):

scripts/build_bus_regions.py

@@ -48,12 +48,12 @@
 import geopandas as gpd
 import pandas as pd
 import pypsa
-from _helpers import REGION_COLS, configure_logging
+from _helpers import REGION_COLS, configure_logging, create_logger
 from shapely.geometry import Point
 
 # from scripts.build_shapes import gadm
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multiplier=5):

scripts/build_cutout.py

@@ -99,9 +99,10 @@
 import atlite
 import geopandas as gpd
 import pandas as pd
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
+
+create_logger(__name__)
 
-logger = logging.getLogger(__name__)
 
 if __name__ == "__main__":
     if "snakemake" not in globals():

scripts/build_demand_profiles.py

@@ -51,11 +51,11 @@
 import pypsa
 import scipy.sparse as sparse
 import xarray as xr
-from _helpers import configure_logging, getContinent, update_p_nom_max
+from _helpers import configure_logging, create_logger, getContinent, update_p_nom_max
 from shapely.prepared import prep
 from shapely.validation import make_valid
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def normed(s):

scripts/build_natura_raster.py

@@ -58,11 +58,12 @@
 import geopandas as gpd
 import numpy as np
 import rasterio as rio
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
 from rasterio.features import geometry_mask
 from rasterio.warp import transform_bounds
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+
 
 CUTOUT_CRS = "EPSG:4326"
 

scripts/build_osm_network.py

@@ -13,6 +13,7 @@
 import pandas as pd
 from _helpers import (
     configure_logging,
+    create_logger,
     read_geojson,
     read_osm_config,
     sets_path_to_root,
@@ -22,7 +23,7 @@
 from shapely.ops import linemerge, split
 from tqdm import tqdm
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def line_endings_to_bus_conversion(lines):

scripts/build_powerplants.py

@@ -103,6 +103,7 @@
 import yaml
 from _helpers import (
     configure_logging,
+    create_logger,
     read_csv_nafix,
     to_csv_nafix,
     two_digits_2_name_country,
@@ -112,7 +113,7 @@
 from shapely import wkt
 from shapely.geometry import Point
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm):

scripts/build_renewable_profiles.py

@@ -203,15 +203,16 @@
 import pandas as pd
 import progressbar as pgb
 import xarray as xr
-from _helpers import configure_logging, read_csv_nafix, sets_path_to_root
+from _helpers import configure_logging, create_logger, read_csv_nafix, sets_path_to_root
 from add_electricity import load_powerplants
 from dask.distributed import Client, LocalCluster
 from pypsa.geo import haversine
 from shapely.geometry import LineString, Point, box
 
 cc = coco.CountryConverter()
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+
 
 COPERNICUS_CRS = "EPSG:4326"
 GEBCO_CRS = "EPSG:4326"

scripts/build_shapes.py

@@ -24,6 +24,7 @@
 import xarray as xr
 from _helpers import (
     configure_logging,
+    create_logger,
     sets_path_to_root,
     three_2_two_digits_country,
     two_2_three_digits_country,
@@ -36,11 +37,10 @@
 from shapely.validation import make_valid
 from tqdm import tqdm
 
-logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-
 sets_path_to_root("pypsa-earth")
 
+create_logger(__name__)
+
 
 def get_GADM_filename(country_code):
     """

scripts/clean_osm_data.py

@@ -12,9 +12,15 @@
 import numpy as np
 import pandas as pd
 import reverse_geocode as rg
-from _helpers import REGION_COLS, configure_logging, save_to_geojson, to_csv_nafix
-
-logger = logging.getLogger(__name__)
+from _helpers import (
+    REGION_COLS,
+    configure_logging,
+    create_logger,
+    save_to_geojson,
+    to_csv_nafix,
+)
+
+create_logger(__name__)
 
 
 def prepare_substation_df(df_all_substations):

scripts/cluster_network.py

@@ -135,6 +135,7 @@
 from _helpers import (
     REGION_COLS,
     configure_logging,
+    create_logger,
     get_aggregation_strategies,
     sets_path_to_root,
     update_p_nom_max,
@@ -151,7 +152,7 @@
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def normed(x):

scripts/download_osm_data.py

@@ -31,10 +31,10 @@
 import shutil
 from pathlib import Path
 
-from _helpers import configure_logging, read_osm_config
+from _helpers import configure_logging, create_logger, read_osm_config
 from earth_osm import eo
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def country_list_to_geofk(country_list):

scripts/make_statistics.py

@@ -32,11 +32,13 @@
 import pandas as pd
 import pypsa
 import xarray as xr
-from _helpers import mock_snakemake, sets_path_to_root, to_csv_nafix
+from _helpers import create_logger, mock_snakemake, sets_path_to_root, to_csv_nafix
 from build_test_configs import create_test_config
 from ruamel.yaml import YAML
 from shapely.validation import make_valid
 
+create_logger(__name__)
+
 
 def _multi_index_scen(rulename, keys):
     return pd.MultiIndex.from_product([[rulename], keys], names=["rule", "key"])

scripts/make_summary.py

@@ -47,11 +47,12 @@
 import pandas as pd
 import pypsa
 from _helpers import configure_logging
-from add_electricity import load_costs, update_transmission_costs
+from add_electricity import create_logger, load_costs, update_transmission_costs
 
 idx = pd.IndexSlice
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
+
 
 opt_name = {"Store": "e", "Line": "s", "Transformer": "s"}
 

scripts/monte_carlo.py

@@ -70,12 +70,12 @@
 import numpy as np
 import pandas as pd
 import pypsa
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
 from pyDOE2 import lhs
 from scipy.stats import qmc
 from solve_network import *
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def monte_carlo_sampling_pydoe2(

scripts/plot_network.py

@@ -29,14 +29,15 @@
     aggregate_costs,
     aggregate_p,
     configure_logging,
+    create_logger,
     load_network_for_plots,
 )
 from matplotlib.legend_handler import HandlerPatch
 from matplotlib.patches import Circle, Ellipse
 
 to_rgba = mpl.colors.colorConverter.to_rgba
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False):

scripts/plot_summary.py

@@ -21,9 +21,9 @@
 
 import matplotlib.pyplot as plt
 import pandas as pd
-from _helpers import configure_logging
+from _helpers import configure_logging, create_logger
 
-logger = logging.getLogger(__name__)
+create_logger(__name__)
 
 
 def rename_techs(label):