From 612dea70f887eb1d9a3574500a5c208d43f2dfdc Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 14 Oct 2023 23:49:44 +0300 Subject: [PATCH 01/18] Add a function to create a logger.error for an exception --- scripts/_helpers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 6ba0bac1e..ee9bfcd36 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -5,6 +5,7 @@ # -*- coding: utf-8 -*- +import logging import os from pathlib import Path @@ -18,6 +19,19 @@ REGION_COLS = ["geometry", "name", "x", "y", "country"] +logger = logging.getLogger(__name__) + + +def handle_exception(exc_type, exc_value, exc_traceback): + if issubclass(exc_type, KeyboardInterrupt): + sys.__excepthook__(exc_type, exc_value, exc_traceback) + return + + logger.error( + "An error happened during workflow execution.", + exc_info=(exc_type, exc_value, exc_traceback), + ) + def read_osm_config(*args): """ From 33787ccd1ffaf8af7a7119f3524e545060d157ce Mon Sep 17 00:00:00 2001 From: ekatef Date: Sat, 14 Oct 2023 23:50:21 +0300 Subject: [PATCH 02/18] Stream exception info into a log file --- scripts/build_shapes.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index dd8d8edcb..5ae293c89 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -24,6 +24,7 @@ import xarray as xr from _helpers import ( configure_logging, + handle_exception, sets_path_to_root, three_2_two_digits_country, two_2_three_digits_country, @@ -39,6 +40,11 @@ logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) +handler = logging.StreamHandler(stream=sys.stdout) +logger.addHandler(handler) +sys.excepthook = handle_exception + + sets_path_to_root("pypsa-earth") From efeb56bbe4e35bceffbb4795c749781cf68089bc Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 16 Oct 2023 00:14:28 +0300 Subject: [PATCH 03/18] Move logger definitions inside an exceptions hook --- scripts/_helpers.py | 6 ++++++ scripts/build_shapes.py | 6 ------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index ee9bfcd36..bcb0252d9 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -7,6 +7,7 @@ import logging import os +import sys from pathlib import Path import country_converter as coco @@ -23,6 +24,11 @@ def handle_exception(exc_type, exc_value, exc_traceback): + logger = logging.getLogger(__name__) + logger.setLevel(logging.INFO) + handler = logging.StreamHandler(stream=sys.stdout) + logger.addHandler(handler) + if issubclass(exc_type, KeyboardInterrupt): sys.__excepthook__(exc_type, exc_value, exc_traceback) return diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 5ae293c89..9cc053894 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -37,14 +37,8 @@ from shapely.validation import make_valid from tqdm import tqdm -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -handler = logging.StreamHandler(stream=sys.stdout) -logger.addHandler(handler) sys.excepthook = handle_exception - sets_path_to_root("pypsa-earth") From 01f9c41589be915879bf6af887ff4659a187eb41 Mon Sep 17 00:00:00 2001 From: ekatef Date: Mon, 16 Oct 2023 00:15:11 +0300 Subject: [PATCH 04/18] Improving an error message --- scripts/_helpers.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index bcb0252d9..b8b127461 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -33,8 +33,17 @@ def handle_exception(exc_type, exc_value, exc_traceback): sys.__excepthook__(exc_type, exc_value, exc_traceback) return + tb = exc_traceback + while tb.tb_next: + tb = tb.tb_next + flname = tb.tb_frame.f_globals.get("__file__") + funcname = tb.tb_frame.f_code.co_name + logger.error( - "An error happened during workflow execution.", + "An error happened in module %r, function %r: %s", + flname, + funcname, + exc_value, exc_info=(exc_type, exc_value, exc_traceback), ) From 2b1f73dab2de653772a6ceae4de5448673297080 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 19:39:16 +0000 Subject: [PATCH 05/18] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index beebd04dc..54ab47cf0 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -7,8 +7,8 @@ import logging import os -import sys import subprocess +import sys from pathlib import Path import country_converter as coco From 1fc05f6fb0dfe7ad9e3c5d4b71bb32fac53d06ad Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Oct 2023 00:32:44 +0300 Subject: [PATCH 06/18] Add keyboard interruptions to logs --- scripts/_helpers.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 54ab47cf0..ec8f09d6d 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -32,23 +32,27 @@ def handle_exception(exc_type, exc_value, exc_traceback): handler = logging.StreamHandler(stream=sys.stdout) logger.addHandler(handler) - if issubclass(exc_type, KeyboardInterrupt): - sys.__excepthook__(exc_type, exc_value, exc_traceback) - return - tb = exc_traceback while tb.tb_next: tb = tb.tb_next flname = tb.tb_frame.f_globals.get("__file__") funcname = tb.tb_frame.f_code.co_name - logger.error( - "An error happened in module %r, function %r: %s", - flname, - funcname, - exc_value, - exc_info=(exc_type, exc_value, exc_traceback), - ) + if issubclass(exc_type, KeyboardInterrupt): + logger.error( + "Manual interruption %r, function %r: %s", + flname, + funcname, + exc_value, + ) + else: + logger.error( + "An error happened in module %r, function %r: %s", + flname, + funcname, + exc_value, + exc_info=(exc_type, exc_value, exc_traceback), + ) def read_osm_config(*args): From 337fd781355d98b2bbc4505961b7e5046c00f874 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Oct 2023 01:02:54 +0300 Subject: [PATCH 07/18] Add a dedicated function to create a logger --- scripts/_helpers.py | 13 ++++++++----- scripts/build_shapes.py | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index ec8f09d6d..c726b8d0c 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -27,11 +27,6 @@ def handle_exception(exc_type, exc_value, exc_traceback): - logger = logging.getLogger(__name__) - logger.setLevel(logging.INFO) - handler = logging.StreamHandler(stream=sys.stdout) - logger.addHandler(handler) - tb = exc_traceback while tb.tb_next: tb = tb.tb_next @@ -55,6 +50,14 @@ def handle_exception(exc_type, exc_value, exc_traceback): ) +def create_logger(level=logging.INFO): + # logger = logging.getLogger('__main__.' + __name__) + logger = logging.getLogger("__main__") + logger.setLevel(level) + handler = logging.StreamHandler(stream=sys.stdout) + logger.addHandler(handler) + + def read_osm_config(*args): """ Read values from the osm_config.yaml file based on provided key arguments. diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 9cc053894..6c15e59af 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -37,7 +37,7 @@ from shapely.validation import make_valid from tqdm import tqdm -sys.excepthook = handle_exception +create_logger(__name___, level=logging.INFO) sets_path_to_root("pypsa-earth") From 5309f2623553bfef45e438fd30e9e32900a1628d Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Oct 2023 01:46:13 +0300 Subject: [PATCH 08/18] Apply log functions inside a module --- scripts/build_shapes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 6c15e59af..f96a27ce7 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -37,11 +37,13 @@ from shapely.validation import make_valid from tqdm import tqdm -create_logger(__name___, level=logging.INFO) - sets_path_to_root("pypsa-earth") +create_logger() +sys.excepthook = handle_exception + + def get_GADM_filename(country_code): """ Function to get the GADM filename given the country code. From 785e72b6e10197faa393bb1ab566d487bb0942f4 Mon Sep 17 00:00:00 2001 From: ekatef Date: Tue, 24 Oct 2023 01:46:13 +0300 Subject: [PATCH 09/18] Apply log functions inside a module --- scripts/build_shapes.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 6c15e59af..f96a27ce7 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -37,11 +37,13 @@ from shapely.validation import make_valid from tqdm import tqdm -create_logger(__name___, level=logging.INFO) - sets_path_to_root("pypsa-earth") +create_logger() +sys.excepthook = handle_exception + + def get_GADM_filename(country_code): """ Function to get the GADM filename given the country code. From 0dc22d7ad6107a874f3716c02100a533febb59d3 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 00:59:24 +0300 Subject: [PATCH 10/18] Fix a logger name --- scripts/_helpers.py | 5 ++--- scripts/build_shapes.py | 3 +-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index c726b8d0c..b42e811f5 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -50,9 +50,8 @@ def handle_exception(exc_type, exc_value, exc_traceback): ) -def create_logger(level=logging.INFO): - # logger = logging.getLogger('__main__.' + __name__) - logger = logging.getLogger("__main__") +def create_logger(logger_name, level=logging.INFO): + logger = logging.getLogger(logger_name) logger.setLevel(level) handler = logging.StreamHandler(stream=sys.stdout) logger.addHandler(handler) diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index f96a27ce7..fc2452b8b 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -39,8 +39,7 @@ sets_path_to_root("pypsa-earth") - -create_logger() +create_logger(__name__) sys.excepthook = handle_exception From 8931d5ab6f75fc4c909c344acc12e9453ca9e9b8 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:00:49 +0300 Subject: [PATCH 11/18] Add docstrings --- scripts/_helpers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index b42e811f5..4c5888dc2 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -27,6 +27,10 @@ def handle_exception(exc_type, exc_value, exc_traceback): + """ + Customise errors traceback. + """ + tb = exc_traceback while tb.tb_next: tb = tb.tb_next @@ -51,6 +55,11 @@ def handle_exception(exc_type, exc_value, exc_traceback): def create_logger(logger_name, level=logging.INFO): + """ + Create a logger for a module and adds a handler needed to capture in logs + traceback from exceptions emerging during the workflow. + """ + logger = logging.getLogger(logger_name) logger.setLevel(level) handler = logging.StreamHandler(stream=sys.stdout) From 8e4fc80f54ad067c9d758011f7ed7190b615ac01 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:02:49 +0300 Subject: [PATCH 12/18] Update logging setup across the scripts --- scripts/add_electricity.py | 11 +++++++++-- scripts/add_extra_components.py | 5 +++-- scripts/augmented_line_connections.py | 5 +++-- scripts/base_network.py | 5 +++-- scripts/build_bus_regions.py | 5 +++-- scripts/build_cutout.py | 5 +++-- scripts/build_demand_profiles.py | 11 +++++++++-- scripts/build_natura_raster.py | 5 +++-- scripts/build_osm_network.py | 5 ++++- scripts/build_powerplants.py | 5 ++++- scripts/build_renewable_profiles.py | 11 +++++++++-- scripts/build_shapes.py | 1 + 12 files changed, 54 insertions(+), 20 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 6cc7b634e..18eeacb2a 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -94,12 +94,19 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import configure_logging, read_csv_nafix, update_p_nom_max +from _helpers import ( + configure_logging, + create_logger, + handle_exception, + read_csv_nafix, + update_p_nom_max, +) from powerplantmatching.export import map_country_bus idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def normed(s): diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 65f21f075..5371030e5 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -58,7 +58,7 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from add_electricity import ( _add_missing_carriers_from_costs, add_nice_carrier_names, @@ -67,7 +67,8 @@ idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def attach_storageunits(n, costs, config): diff --git a/scripts/augmented_line_connections.py b/scripts/augmented_line_connections.py index 4f5dcfb30..2c687409f 100644 --- a/scripts/augmented_line_connections.py +++ b/scripts/augmented_line_connections.py @@ -35,14 +35,15 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from add_electricity import load_costs from base_network import _set_dc_underwater_fraction from networkx.algorithms import complement from networkx.algorithms.connectivity.edge_augmentation import k_edge_augmentation from pypsa.geo import haversine_pts -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception # Functions diff --git a/scripts/base_network.py b/scripts/base_network.py index c321f2c29..573152096 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -67,12 +67,13 @@ import shapely.prepared import shapely.wkt import yaml -from _helpers import configure_logging, read_csv_nafix +from _helpers import configure_logging, create_logger, handle_exception, read_csv_nafix from scipy.sparse import csgraph from shapely.geometry import LineString, Point from shapely.ops import unary_union -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def _get_oid(df): diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 340bbdeac..73d7c670c 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -48,12 +48,13 @@ import geopandas as gpd import pandas as pd import pypsa -from _helpers import REGION_COLS, configure_logging +from _helpers import REGION_COLS, configure_logging, create_logger, handle_exception from shapely.geometry import Point # from scripts.build_shapes import gadm -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multiplier=5): diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index ea72aa294..4e598b32b 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -99,9 +99,10 @@ import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception if __name__ == "__main__": if "snakemake" not in globals(): diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index ae031da47..29cb1aff3 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -51,11 +51,18 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, getContinent, update_p_nom_max +from _helpers import ( + configure_logging, + create_logger, + getContinent, + handle_exception, + update_p_nom_max, +) from shapely.prepared import prep from shapely.validation import make_valid -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def normed(s): diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index febedc98f..77eb6ab7d 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -58,11 +58,12 @@ import geopandas as gpd import numpy as np import rasterio as rio -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from rasterio.features import geometry_mask from rasterio.warp import transform_bounds -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception CUTOUT_CRS = "EPSG:4326" diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index dd70cbc1b..847d5b0b5 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -13,6 +13,8 @@ import pandas as pd from _helpers import ( configure_logging, + create_logger, + handle_exception, read_geojson, read_osm_config, sets_path_to_root, @@ -22,7 +24,8 @@ from shapely.ops import linemerge, split from tqdm import tqdm -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def line_endings_to_bus_conversion(lines): diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index b2f14d582..e355c3f68 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -103,6 +103,8 @@ import yaml from _helpers import ( configure_logging, + create_logger, + handle_exception, read_csv_nafix, to_csv_nafix, two_digits_2_name_country, @@ -112,7 +114,8 @@ from shapely import wkt from shapely.geometry import Point -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm): diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 99f3a79ba..38bf56d5b 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -203,7 +203,13 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import configure_logging, read_csv_nafix, sets_path_to_root +from _helpers import ( + configure_logging, + create_logger, + handle_exception, + read_csv_nafix, + sets_path_to_root, +) from add_electricity import load_powerplants from dask.distributed import Client, LocalCluster from pypsa.geo import haversine @@ -211,7 +217,8 @@ cc = coco.CountryConverter() -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception COPERNICUS_CRS = "EPSG:4326" GEBCO_CRS = "EPSG:4326" diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index fc2452b8b..0576a7f83 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -24,6 +24,7 @@ import xarray as xr from _helpers import ( configure_logging, + create_logger, handle_exception, sets_path_to_root, three_2_two_digits_country, From 2d3907a92fb2705519a43e1f6c95cfcb6630adfd Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:17:44 +0300 Subject: [PATCH 13/18] Add a missed import --- scripts/add_electricity.py | 1 + scripts/add_extra_components.py | 1 + scripts/augmented_line_connections.py | 1 + scripts/base_network.py | 1 + scripts/build_bus_regions.py | 1 + scripts/build_cutout.py | 1 + scripts/build_demand_profiles.py | 1 + scripts/build_natura_raster.py | 1 + scripts/build_osm_network.py | 1 + scripts/build_powerplants.py | 1 + scripts/build_renewable_profiles.py | 1 + scripts/build_shapes.py | 1 + scripts/clean_osm_data.py | 1 + scripts/cluster_network.py | 1 + scripts/download_osm_data.py | 6 ++++-- scripts/make_statistics.py | 1 + scripts/make_summary.py | 1 + scripts/monte_carlo.py | 1 + scripts/plot_network.py | 1 + scripts/plot_summary.py | 1 + scripts/prepare_network.py | 1 + scripts/retrieve_databundle_light.py | 1 + scripts/solve_network.py | 1 + 23 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 18eeacb2a..8c73cd4d9 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -87,6 +87,7 @@ import logging import os +import sys import geopandas as gpd import numpy as np diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index 5371030e5..b4f87281b 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -54,6 +54,7 @@ """ import logging import os +import sys import numpy as np import pandas as pd diff --git a/scripts/augmented_line_connections.py b/scripts/augmented_line_connections.py index 2c687409f..2509dbcd9 100644 --- a/scripts/augmented_line_connections.py +++ b/scripts/augmented_line_connections.py @@ -30,6 +30,7 @@ """ import logging import os +import sys import networkx as nx import numpy as np diff --git a/scripts/base_network.py b/scripts/base_network.py index 573152096..c6384f9e8 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -57,6 +57,7 @@ """ import logging import os +import sys import geopandas as gpd import networkx as nx diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 73d7c670c..2ac87d23b 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -44,6 +44,7 @@ """ import logging import os +import sys import geopandas as gpd import pandas as pd diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index 4e598b32b..cd2f919fc 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -95,6 +95,7 @@ """ import logging import os +import sys import atlite import geopandas as gpd diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 29cb1aff3..eac62188b 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -42,6 +42,7 @@ """ import logging import os +import sys from itertools import product import geopandas as gpd diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 77eb6ab7d..54d173a44 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -53,6 +53,7 @@ """ import logging import os +import sys import atlite import geopandas as gpd diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 847d5b0b5..0bd60f0ab 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -7,6 +7,7 @@ import logging import os +import sys import geopandas as gpd import numpy as np diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index e355c3f68..308a2d77c 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -94,6 +94,7 @@ """ import logging import os +import sys import geopandas as gpd import numpy as np diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 38bf56d5b..cc6451541 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -193,6 +193,7 @@ import functools import logging import os +import sys import time from math import isnan diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 0576a7f83..b86cc9a8e 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -9,6 +9,7 @@ import multiprocessing as mp import os import shutil +import sys import zipfile from itertools import takewhile from math import ceil diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index a3d1d22f9..db30c3b2f 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -7,6 +7,7 @@ import logging import os +import sys import geopandas as gpd import numpy as np diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 8a3674895..68fdedd12 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,6 +122,7 @@ """ import logging import os +import sys from functools import reduce import geopandas as gpd diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 24afa4587..8a66f61dd 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -29,12 +29,14 @@ import logging import os import shutil +import sys from pathlib import Path -from _helpers import configure_logging, read_osm_config +from _helpers import configure_logging, create_logger, handle_exception, read_osm_config from earth_osm import eo -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def country_list_to_geofk(country_list): diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index d87eb71c2..9d6942fe0 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -24,6 +24,7 @@ """ import os import shutil +import sys from datetime import datetime from pathlib import Path diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 4ecf727bc..f8d404669 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -43,6 +43,7 @@ """ import logging import os +import sys import pandas as pd import pypsa diff --git a/scripts/monte_carlo.py b/scripts/monte_carlo.py index f60180236..08c114921 100644 --- a/scripts/monte_carlo.py +++ b/scripts/monte_carlo.py @@ -65,6 +65,7 @@ """ import logging import os +import sys import chaospy import numpy as np diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 4084e0881..972a9b62b 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -18,6 +18,7 @@ """ import logging import os +import sys import cartopy.crs as ccrs import geopandas as gpd diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 6da5dd4f9..99fd0d8de 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -18,6 +18,7 @@ """ import logging import os +import sys import matplotlib.pyplot as plt import pandas as pd diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 90da72585..bc46465c5 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -59,6 +59,7 @@ import logging import os import re +import sys from zipfile import ZipFile import country_converter as cc diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 59a66346d..805bc4a33 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -83,6 +83,7 @@ import logging import os import re +import sys from zipfile import ZipFile import yaml diff --git a/scripts/solve_network.py b/scripts/solve_network.py index 0a00f89d7..a5309ef0d 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -80,6 +80,7 @@ import logging import os import re +import sys from pathlib import Path import numpy as np From e0d89fa9ce3bdda8e84b16872ca28332ba4e59fa Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:19:02 +0300 Subject: [PATCH 14/18] Add logging updates --- scripts/clean_osm_data.py | 14 +++++++++++--- scripts/cluster_network.py | 5 ++++- scripts/make_statistics.py | 11 ++++++++++- scripts/make_summary.py | 10 ++++++++-- scripts/monte_carlo.py | 5 +++-- scripts/plot_network.py | 5 ++++- scripts/plot_summary.py | 5 +++-- scripts/prepare_network.py | 5 +++-- scripts/retrieve_databundle_light.py | 5 ++++- scripts/simplify_network.py | 11 +++++++++-- scripts/solve_network.py | 5 +++-- 11 files changed, 62 insertions(+), 19 deletions(-) diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index db30c3b2f..f1504a43e 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -13,9 +13,17 @@ import numpy as np import pandas as pd import reverse_geocode as rg -from _helpers import REGION_COLS, configure_logging, save_to_geojson, to_csv_nafix - -logger = logging.getLogger(__name__) +from _helpers import ( + REGION_COLS, + configure_logging, + create_logger, + handle_exception, + save_to_geojson, + to_csv_nafix, +) + +create_logger(__name__) +sys.excepthook = handle_exception def prepare_substation_df(df_all_substations): diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 68fdedd12..2bf88051d 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -136,7 +136,9 @@ from _helpers import ( REGION_COLS, configure_logging, + create_logger, get_aggregation_strategies, + handle_exception, sets_path_to_root, update_p_nom_max, ) @@ -152,7 +154,8 @@ idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def normed(x): diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index 9d6942fe0..6e960826c 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -33,11 +33,20 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import mock_snakemake, sets_path_to_root, to_csv_nafix +from _helpers import ( + create_logger, + handle_exception, + mock_snakemake, + sets_path_to_root, + to_csv_nafix, +) from build_test_configs import create_test_config from ruamel.yaml import YAML from shapely.validation import make_valid +create_logger(__name__) +sys.excepthook = handle_exception + def _multi_index_scen(rulename, keys): return pd.MultiIndex.from_product([[rulename], keys], names=["rule", "key"]) diff --git a/scripts/make_summary.py b/scripts/make_summary.py index f8d404669..39eb51a9d 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -48,11 +48,17 @@ import pandas as pd import pypsa from _helpers import configure_logging -from add_electricity import load_costs, update_transmission_costs +from add_electricity import ( + create_logger, + handle_exception, + load_costs, + update_transmission_costs, +) idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} diff --git a/scripts/monte_carlo.py b/scripts/monte_carlo.py index 08c114921..4c159b547 100644 --- a/scripts/monte_carlo.py +++ b/scripts/monte_carlo.py @@ -71,12 +71,13 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from pyDOE2 import lhs from scipy.stats import qmc from solve_network import * -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def monte_carlo_sampling_pydoe2( diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 972a9b62b..86db58b2b 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -30,6 +30,8 @@ aggregate_costs, aggregate_p, configure_logging, + create_logger, + handle_exception, load_network_for_plots, ) from matplotlib.legend_handler import HandlerPatch @@ -37,7 +39,8 @@ to_rgba = mpl.colors.colorConverter.to_rgba -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False): diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index 99fd0d8de..d7ba85de7 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -22,9 +22,10 @@ import matplotlib.pyplot as plt import pandas as pd -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def rename_techs(label): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index bc46465c5..2dec46cd4 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -67,12 +67,13 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def download_emission_data(): diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 805bc4a33..1201ef051 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -90,12 +90,15 @@ from _helpers import ( configure_logging, create_country_list, + create_logger, + handle_exception, progress_retrieve, sets_path_to_root, ) from google_drive_downloader import GoogleDriveDownloader as gdd -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def load_databundle_config(config): diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index 4e2d1c2c0..aafa05aa0 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -93,7 +93,13 @@ import pandas as pd import pypsa import scipy as sp -from _helpers import configure_logging, get_aggregation_strategies, update_p_nom_max +from _helpers import ( + configure_logging, + create_logger, + get_aggregation_strategies, + handle_exception, + update_p_nom_max, +) from add_electricity import load_costs from cluster_network import cluster_regions, clustering_for_n_clusters from pypsa.clustering.spatial import ( @@ -107,7 +113,8 @@ sys.settrace -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def simplify_network_to_base_voltage(n, linetype, base_voltage): diff --git a/scripts/solve_network.py b/scripts/solve_network.py index a5309ef0d..acc9c6a1f 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -86,7 +86,7 @@ import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging +from _helpers import configure_logging, create_logger, handle_exception from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pypsa.linopf import ( define_constraints, @@ -98,7 +98,8 @@ network_lopf, ) -logger = logging.getLogger(__name__) +create_logger(__name__) +sys.excepthook = handle_exception def prepare_network(n, solve_opts): From bad5bb9191b5605efc18613866f5f6fcbf0ab1c9 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:23:43 +0300 Subject: [PATCH 15/18] Update a release note --- doc/release_notes.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 857010fcd..8d83457ce 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -12,7 +12,11 @@ Upcoming release Please add descriptive release notes like in `PyPSA-Eur `__. E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does. -**New Features and major Changes** + +PyPSA-Earth 0.2.3 +================= + +**New Features and major Changes (19th October 2023)** * Add params: section in rule definition to keep track of changed settings in config.yaml. `PR #823 `__ and `PR #880 `__ From 0d349f9943151d0f4d3b52d817b0ea2147670cd2 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 01:25:01 +0300 Subject: [PATCH 16/18] Amend a release note --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 8d83457ce..41c989826 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -12,6 +12,7 @@ Upcoming release Please add descriptive release notes like in `PyPSA-Eur `__. E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_tests` and in one sentence what it does. +* Keep all traceback in logs. `PR #898 `__ PyPSA-Earth 0.2.3 ================= From b75b1eb4a4a62fb9320c0fb817779076bce30692 Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 02:16:45 +0300 Subject: [PATCH 17/18] Remove empty lines --- scripts/_helpers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 4c5888dc2..8db056184 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -30,7 +30,6 @@ def handle_exception(exc_type, exc_value, exc_traceback): """ Customise errors traceback. """ - tb = exc_traceback while tb.tb_next: tb = tb.tb_next @@ -59,7 +58,6 @@ def create_logger(logger_name, level=logging.INFO): Create a logger for a module and adds a handler needed to capture in logs traceback from exceptions emerging during the workflow. """ - logger = logging.getLogger(logger_name) logger.setLevel(level) handler = logging.StreamHandler(stream=sys.stdout) From 64388a336200cfacd96a362c60bf7b0f2822bbcd Mon Sep 17 00:00:00 2001 From: ekatef Date: Sun, 29 Oct 2023 21:42:12 +0300 Subject: [PATCH 18/18] Fix logger calls and imports --- scripts/_helpers.py | 1 + scripts/add_electricity.py | 10 +--------- scripts/add_extra_components.py | 4 +--- scripts/augmented_line_connections.py | 4 +--- scripts/base_network.py | 4 +--- scripts/build_bus_regions.py | 4 +--- scripts/build_cutout.py | 5 ++--- scripts/build_demand_profiles.py | 10 +--------- scripts/build_natura_raster.py | 5 ++--- scripts/build_osm_network.py | 3 --- scripts/build_powerplants.py | 3 --- scripts/build_renewable_profiles.py | 11 ++--------- scripts/build_shapes.py | 3 --- scripts/clean_osm_data.py | 3 --- scripts/cluster_network.py | 3 --- scripts/download_osm_data.py | 4 +--- scripts/make_statistics.py | 10 +--------- scripts/make_summary.py | 10 ++-------- scripts/monte_carlo.py | 4 +--- scripts/plot_network.py | 3 --- scripts/plot_summary.py | 4 +--- scripts/prepare_network.py | 4 +--- scripts/retrieve_databundle_light.py | 3 --- scripts/simplify_network.py | 2 -- scripts/solve_network.py | 4 +--- 25 files changed, 21 insertions(+), 100 deletions(-) diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 8db056184..f5ae8aed4 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -62,6 +62,7 @@ def create_logger(logger_name, level=logging.INFO): logger.setLevel(level) handler = logging.StreamHandler(stream=sys.stdout) logger.addHandler(handler) + sys.excepthook = handle_exception def read_osm_config(*args): diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 8c73cd4d9..23bf15ad7 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -87,7 +87,6 @@ import logging import os -import sys import geopandas as gpd import numpy as np @@ -95,19 +94,12 @@ import powerplantmatching as pm import pypsa import xarray as xr -from _helpers import ( - configure_logging, - create_logger, - handle_exception, - read_csv_nafix, - update_p_nom_max, -) +from _helpers import configure_logging, create_logger, read_csv_nafix, update_p_nom_max from powerplantmatching.export import map_country_bus idx = pd.IndexSlice create_logger(__name__) -sys.excepthook = handle_exception def normed(s): diff --git a/scripts/add_extra_components.py b/scripts/add_extra_components.py index b4f87281b..1b2f3a92a 100644 --- a/scripts/add_extra_components.py +++ b/scripts/add_extra_components.py @@ -54,12 +54,11 @@ """ import logging import os -import sys import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from add_electricity import ( _add_missing_carriers_from_costs, add_nice_carrier_names, @@ -69,7 +68,6 @@ idx = pd.IndexSlice create_logger(__name__) -sys.excepthook = handle_exception def attach_storageunits(n, costs, config): diff --git a/scripts/augmented_line_connections.py b/scripts/augmented_line_connections.py index 2509dbcd9..7329049db 100644 --- a/scripts/augmented_line_connections.py +++ b/scripts/augmented_line_connections.py @@ -30,13 +30,12 @@ """ import logging import os -import sys import networkx as nx import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from add_electricity import load_costs from base_network import _set_dc_underwater_fraction from networkx.algorithms import complement @@ -44,7 +43,6 @@ from pypsa.geo import haversine_pts create_logger(__name__) -sys.excepthook = handle_exception # Functions diff --git a/scripts/base_network.py b/scripts/base_network.py index c6384f9e8..c302d5b4c 100644 --- a/scripts/base_network.py +++ b/scripts/base_network.py @@ -57,7 +57,6 @@ """ import logging import os -import sys import geopandas as gpd import networkx as nx @@ -68,13 +67,12 @@ import shapely.prepared import shapely.wkt import yaml -from _helpers import configure_logging, create_logger, handle_exception, read_csv_nafix +from _helpers import configure_logging, create_logger, read_csv_nafix from scipy.sparse import csgraph from shapely.geometry import LineString, Point from shapely.ops import unary_union create_logger(__name__) -sys.excepthook = handle_exception def _get_oid(df): diff --git a/scripts/build_bus_regions.py b/scripts/build_bus_regions.py index 2ac87d23b..66b12bbd5 100644 --- a/scripts/build_bus_regions.py +++ b/scripts/build_bus_regions.py @@ -44,18 +44,16 @@ """ import logging import os -import sys import geopandas as gpd import pandas as pd import pypsa -from _helpers import REGION_COLS, configure_logging, create_logger, handle_exception +from _helpers import REGION_COLS, configure_logging, create_logger from shapely.geometry import Point # from scripts.build_shapes import gadm create_logger(__name__) -sys.excepthook = handle_exception def custom_voronoi_partition_pts(points, outline, add_bounds_shape=True, multiplier=5): diff --git a/scripts/build_cutout.py b/scripts/build_cutout.py index cd2f919fc..05aa842f0 100644 --- a/scripts/build_cutout.py +++ b/scripts/build_cutout.py @@ -95,15 +95,14 @@ """ import logging import os -import sys import atlite import geopandas as gpd import pandas as pd -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger create_logger(__name__) -sys.excepthook = handle_exception + if __name__ == "__main__": if "snakemake" not in globals(): diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index eac62188b..4e57dc1e8 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -42,7 +42,6 @@ """ import logging import os -import sys from itertools import product import geopandas as gpd @@ -52,18 +51,11 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import ( - configure_logging, - create_logger, - getContinent, - handle_exception, - update_p_nom_max, -) +from _helpers import configure_logging, create_logger, getContinent, update_p_nom_max from shapely.prepared import prep from shapely.validation import make_valid create_logger(__name__) -sys.excepthook = handle_exception def normed(s): diff --git a/scripts/build_natura_raster.py b/scripts/build_natura_raster.py index 54d173a44..98406b6ea 100644 --- a/scripts/build_natura_raster.py +++ b/scripts/build_natura_raster.py @@ -53,18 +53,17 @@ """ import logging import os -import sys import atlite import geopandas as gpd import numpy as np import rasterio as rio -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from rasterio.features import geometry_mask from rasterio.warp import transform_bounds create_logger(__name__) -sys.excepthook = handle_exception + CUTOUT_CRS = "EPSG:4326" diff --git a/scripts/build_osm_network.py b/scripts/build_osm_network.py index 0bd60f0ab..fa71e5100 100644 --- a/scripts/build_osm_network.py +++ b/scripts/build_osm_network.py @@ -7,7 +7,6 @@ import logging import os -import sys import geopandas as gpd import numpy as np @@ -15,7 +14,6 @@ from _helpers import ( configure_logging, create_logger, - handle_exception, read_geojson, read_osm_config, sets_path_to_root, @@ -26,7 +24,6 @@ from tqdm import tqdm create_logger(__name__) -sys.excepthook = handle_exception def line_endings_to_bus_conversion(lines): diff --git a/scripts/build_powerplants.py b/scripts/build_powerplants.py index 308a2d77c..bfced3ce6 100644 --- a/scripts/build_powerplants.py +++ b/scripts/build_powerplants.py @@ -94,7 +94,6 @@ """ import logging import os -import sys import geopandas as gpd import numpy as np @@ -105,7 +104,6 @@ from _helpers import ( configure_logging, create_logger, - handle_exception, read_csv_nafix, to_csv_nafix, two_digits_2_name_country, @@ -116,7 +114,6 @@ from shapely.geometry import Point create_logger(__name__) -sys.excepthook = handle_exception def convert_osm_to_pm(filepath_ppl_osm, filepath_ppl_pm): diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index cc6451541..ce1128f16 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -193,7 +193,6 @@ import functools import logging import os -import sys import time from math import isnan @@ -204,13 +203,7 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import ( - configure_logging, - create_logger, - handle_exception, - read_csv_nafix, - sets_path_to_root, -) +from _helpers import configure_logging, create_logger, read_csv_nafix, sets_path_to_root from add_electricity import load_powerplants from dask.distributed import Client, LocalCluster from pypsa.geo import haversine @@ -219,7 +212,7 @@ cc = coco.CountryConverter() create_logger(__name__) -sys.excepthook = handle_exception + COPERNICUS_CRS = "EPSG:4326" GEBCO_CRS = "EPSG:4326" diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index b86cc9a8e..659fa2da4 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -9,7 +9,6 @@ import multiprocessing as mp import os import shutil -import sys import zipfile from itertools import takewhile from math import ceil @@ -26,7 +25,6 @@ from _helpers import ( configure_logging, create_logger, - handle_exception, sets_path_to_root, three_2_two_digits_country, two_2_three_digits_country, @@ -42,7 +40,6 @@ sets_path_to_root("pypsa-earth") create_logger(__name__) -sys.excepthook = handle_exception def get_GADM_filename(country_code): diff --git a/scripts/clean_osm_data.py b/scripts/clean_osm_data.py index f1504a43e..f7544bc83 100644 --- a/scripts/clean_osm_data.py +++ b/scripts/clean_osm_data.py @@ -7,7 +7,6 @@ import logging import os -import sys import geopandas as gpd import numpy as np @@ -17,13 +16,11 @@ REGION_COLS, configure_logging, create_logger, - handle_exception, save_to_geojson, to_csv_nafix, ) create_logger(__name__) -sys.excepthook = handle_exception def prepare_substation_df(df_all_substations): diff --git a/scripts/cluster_network.py b/scripts/cluster_network.py index 2bf88051d..3f783e7cc 100644 --- a/scripts/cluster_network.py +++ b/scripts/cluster_network.py @@ -122,7 +122,6 @@ """ import logging import os -import sys from functools import reduce import geopandas as gpd @@ -138,7 +137,6 @@ configure_logging, create_logger, get_aggregation_strategies, - handle_exception, sets_path_to_root, update_p_nom_max, ) @@ -155,7 +153,6 @@ idx = pd.IndexSlice create_logger(__name__) -sys.excepthook = handle_exception def normed(x): diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index 8a66f61dd..9498f215a 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -29,14 +29,12 @@ import logging import os import shutil -import sys from pathlib import Path -from _helpers import configure_logging, create_logger, handle_exception, read_osm_config +from _helpers import configure_logging, create_logger, read_osm_config from earth_osm import eo create_logger(__name__) -sys.excepthook = handle_exception def country_list_to_geofk(country_list): diff --git a/scripts/make_statistics.py b/scripts/make_statistics.py index 6e960826c..b883f478b 100644 --- a/scripts/make_statistics.py +++ b/scripts/make_statistics.py @@ -24,7 +24,6 @@ """ import os import shutil -import sys from datetime import datetime from pathlib import Path @@ -33,19 +32,12 @@ import pandas as pd import pypsa import xarray as xr -from _helpers import ( - create_logger, - handle_exception, - mock_snakemake, - sets_path_to_root, - to_csv_nafix, -) +from _helpers import create_logger, mock_snakemake, sets_path_to_root, to_csv_nafix from build_test_configs import create_test_config from ruamel.yaml import YAML from shapely.validation import make_valid create_logger(__name__) -sys.excepthook = handle_exception def _multi_index_scen(rulename, keys): diff --git a/scripts/make_summary.py b/scripts/make_summary.py index 39eb51a9d..f1e2f1399 100644 --- a/scripts/make_summary.py +++ b/scripts/make_summary.py @@ -43,22 +43,16 @@ """ import logging import os -import sys import pandas as pd import pypsa from _helpers import configure_logging -from add_electricity import ( - create_logger, - handle_exception, - load_costs, - update_transmission_costs, -) +from add_electricity import create_logger, load_costs, update_transmission_costs idx = pd.IndexSlice create_logger(__name__) -sys.excepthook = handle_exception + opt_name = {"Store": "e", "Line": "s", "Transformer": "s"} diff --git a/scripts/monte_carlo.py b/scripts/monte_carlo.py index 4c159b547..3a526063d 100644 --- a/scripts/monte_carlo.py +++ b/scripts/monte_carlo.py @@ -65,19 +65,17 @@ """ import logging import os -import sys import chaospy import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from pyDOE2 import lhs from scipy.stats import qmc from solve_network import * create_logger(__name__) -sys.excepthook = handle_exception def monte_carlo_sampling_pydoe2( diff --git a/scripts/plot_network.py b/scripts/plot_network.py index 86db58b2b..f72d721aa 100644 --- a/scripts/plot_network.py +++ b/scripts/plot_network.py @@ -18,7 +18,6 @@ """ import logging import os -import sys import cartopy.crs as ccrs import geopandas as gpd @@ -31,7 +30,6 @@ aggregate_p, configure_logging, create_logger, - handle_exception, load_network_for_plots, ) from matplotlib.legend_handler import HandlerPatch @@ -40,7 +38,6 @@ to_rgba = mpl.colors.colorConverter.to_rgba create_logger(__name__) -sys.excepthook = handle_exception def make_handler_map_to_scale_circles_as_in(ax, dont_resize_actively=False): diff --git a/scripts/plot_summary.py b/scripts/plot_summary.py index d7ba85de7..13215e674 100644 --- a/scripts/plot_summary.py +++ b/scripts/plot_summary.py @@ -18,14 +18,12 @@ """ import logging import os -import sys import matplotlib.pyplot as plt import pandas as pd -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger create_logger(__name__) -sys.excepthook = handle_exception def rename_techs(label): diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 2dec46cd4..2afa7a083 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -59,7 +59,6 @@ import logging import os import re -import sys from zipfile import ZipFile import country_converter as cc @@ -67,13 +66,12 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice create_logger(__name__) -sys.excepthook = handle_exception def download_emission_data(): diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index 1201ef051..e18e7e62b 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -83,7 +83,6 @@ import logging import os import re -import sys from zipfile import ZipFile import yaml @@ -91,14 +90,12 @@ configure_logging, create_country_list, create_logger, - handle_exception, progress_retrieve, sets_path_to_root, ) from google_drive_downloader import GoogleDriveDownloader as gdd create_logger(__name__) -sys.excepthook = handle_exception def load_databundle_config(config): diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py index aafa05aa0..7a2f01619 100644 --- a/scripts/simplify_network.py +++ b/scripts/simplify_network.py @@ -97,7 +97,6 @@ configure_logging, create_logger, get_aggregation_strategies, - handle_exception, update_p_nom_max, ) from add_electricity import load_costs @@ -114,7 +113,6 @@ sys.settrace create_logger(__name__) -sys.excepthook = handle_exception def simplify_network_to_base_voltage(n, linetype, base_voltage): diff --git a/scripts/solve_network.py b/scripts/solve_network.py index acc9c6a1f..589656e2d 100755 --- a/scripts/solve_network.py +++ b/scripts/solve_network.py @@ -80,13 +80,12 @@ import logging import os import re -import sys from pathlib import Path import numpy as np import pandas as pd import pypsa -from _helpers import configure_logging, create_logger, handle_exception +from _helpers import configure_logging, create_logger from pypsa.descriptors import get_switchable_as_dense as get_as_dense from pypsa.linopf import ( define_constraints, @@ -99,7 +98,6 @@ ) create_logger(__name__) -sys.excepthook = handle_exception def prepare_network(n, solve_opts):