From ca27e84f50fdce06f63f9b0f1a66380daa7cee2b Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 27 Sep 2024 16:12:01 +0200 Subject: [PATCH 01/21] Split inflow in ppl units + same hydro profile in alt. clustering --- scripts/add_electricity.py | 9 ++--- scripts/build_renewable_profiles.py | 53 +++++------------------------ 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 55c9749fc..3f034a36e 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -488,10 +488,7 @@ def attach_hydro(n, costs, ppl): ror = ppl.query('technology == "Run-Of-River"') phs = ppl.query('technology == "Pumped Storage"') hydro = ppl.query('technology == "Reservoir"') - if snakemake.params.alternative_clustering: - bus_id = ppl["region_id"] - else: - bus_id = ppl["bus"] + bus_id = ppl["bus"] inflow_idx = ror.index.union(hydro.index) if not inflow_idx.empty: @@ -530,12 +527,16 @@ def attach_hydro(n, costs, ppl): network_buses_to_keep = plants_with_data.index plants_to_keep = plants_with_data.to_numpy() + # hydro_inflow_factor is used to divide the inflow between the various units of each power plant + hydro_inflow_factor = hydro["p_nom"] / hydro.groupby("bus")["p_nom"].transform("sum") + inflow_t = ( inflow.sel(plant=plants_to_keep) .rename({"plant": "name"}) .assign_coords(name=network_buses_to_keep) .transpose("time", "name") .to_pandas() + *hydro_inflow_factor ) if "ror" in carriers and not ror.empty: diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 1ebf220b4..808363ac5 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -356,9 +356,6 @@ def rescale_hydro(plants, runoff, normalize_using_yearly, normalization_year): logger.info("No bus has installed hydro plants, ignoring normalization.") return runoff - if snakemake.params.alternative_clustering: - plants = plants.set_index("shape_id") - years_statistics = normalize_using_yearly.index if isinstance(years_statistics, pd.DatetimeIndex): years_statistics = years_statistics.year @@ -533,24 +530,6 @@ def create_scaling_factor( # the region should be restricted for non-hydro technologies, as the hydro potential is calculated across hydrobasins which may span beyond the region of the country cutout = filter_cutout_region(cutout, regions) - if snakemake.params.alternative_clustering: - regions = gpd.GeoDataFrame( - regions.reset_index() - .groupby("shape_id") - .agg( - { - "x": "mean", - "y": "mean", - "country": "first", - "geometry": "first", - "bus": "first", - } - ) - .reset_index() - .set_index("bus"), - crs=regions.crs, - ) - buses = regions.index func = getattr(cutout, resource.pop("method")) @@ -575,17 +554,10 @@ def create_scaling_factor( # select busbar whose location (p) belongs to at least one hydrobasin geometry # if extendable option is true, all buses are included # otherwise only where hydro powerplants are available are considered - if snakemake.params.alternative_clustering: - filter_bus_to_consider = regions.index.map( - lambda bus_id: config.get("extendable", False) - | (bus_id in hydro_ppls.region_id.values) - ) - ### TODO: quickfix. above case and the below case should by unified - if snakemake.params.alternative_clustering == False: - filter_bus_to_consider = regions.index.map( - lambda bus_id: config.get("extendable", False) - | (bus_id in hydro_ppls.bus.values) - ) + filter_bus_to_consider = regions.index.map( + lambda bus_id: config.get("extendable", False) + | (bus_id in hydro_ppls.bus.values) + ) bus_to_consider = regions.index[filter_bus_to_consider] # identify subset of buses within the hydrobasins @@ -603,17 +575,10 @@ def create_scaling_factor( columns={"x": "lon", "y": "lat", "country": "countries"} ).loc[bus_in_hydrobasins, ["lon", "lat", "countries", "shape_id"]] - # TODO: these cases shall be fixed by restructuring the alternative clustering procedure - if snakemake.params.alternative_clustering == False: - resource["plants"]["installed_hydro"] = [ - True if (bus_id in hydro_ppls.bus.values) else False - for bus_id in resource["plants"].index - ] - else: - resource["plants"]["installed_hydro"] = [ - True if (bus_id in hydro_ppls.region_id.values) else False - for bus_id in resource["plants"].shape_id.values - ] + resource["plants"]["installed_hydro"] = [ + True if (bus_id in hydro_ppls.bus.values) else False + for bus_id in resource["plants"].index + ] # get normalization before executing runoff normalization = None @@ -629,8 +594,6 @@ def create_scaling_factor( else: # otherwise perform the calculations inflow = correction_factor * func(capacity_factor=True, **resource) - if snakemake.params.alternative_clustering: - inflow["plant"] = regions.shape_id.loc[inflow["plant"]].values if "clip_min_inflow" in config: inflow = inflow.where(inflow >= config["clip_min_inflow"], 0) From 2084579bbc01532f883ab13012234740d34a2876 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 27 Sep 2024 14:13:50 +0000 Subject: [PATCH 02/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/add_electricity.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/add_electricity.py b/scripts/add_electricity.py index 3f034a36e..75ae9ce42 100755 --- a/scripts/add_electricity.py +++ b/scripts/add_electricity.py @@ -528,7 +528,9 @@ def attach_hydro(n, costs, ppl): plants_to_keep = plants_with_data.to_numpy() # hydro_inflow_factor is used to divide the inflow between the various units of each power plant - hydro_inflow_factor = hydro["p_nom"] / hydro.groupby("bus")["p_nom"].transform("sum") + hydro_inflow_factor = hydro["p_nom"] / hydro.groupby("bus")[ + "p_nom" + ].transform("sum") inflow_t = ( inflow.sel(plant=plants_to_keep) @@ -536,7 +538,7 @@ def attach_hydro(n, costs, ppl): .assign_coords(name=network_buses_to_keep) .transpose("time", "name") .to_pandas() - *hydro_inflow_factor + * hydro_inflow_factor ) if "ror" in carriers and not ror.empty: From fbe6530615feb1ee40e67ddb23033f1b5dde8f6c Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 4 Oct 2024 19:30:31 +0200 Subject: [PATCH 03/21] updated release_notes.rst --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 838a42011..54f25a1aa 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -20,6 +20,7 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o **Minor Changes and bug-fixing** +* The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `_ PyPSA-Earth 0.4.1 ================= From 20cbda3c997a59086b9ac48a17e9e11e675d29e3 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Mon, 14 Oct 2024 22:15:53 +0500 Subject: [PATCH 04/21] drop duplicate AL_production data for given country and year --- scripts/build_industry_demand.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 90922bb6d..c563c0248 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -205,6 +205,9 @@ def match_technology(df): aluminium_year = snakemake.params.aluminium_year AL = read_csv_nafix("data/AL_production.csv", index_col=0) + # drop duplicate entries for the same country and year by keeping highest production data + AL = AL.sort_values(by=['country', 'Year', 'production[ktons/a]'], ascending=[True, True, False]).reset_index() + AL = AL.drop_duplicates(subset=["country", "Year"], keep='first').set_index("country") AL_prod_tom = AL.query("Year == @aluminium_year and index in @countries_geo")[ "production[ktons/a]" ].reindex(countries_geo, fill_value=0.0) From 5334b63a5cdf83cd55caa53a286db380d92c80ab Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Mon, 14 Oct 2024 22:18:29 +0500 Subject: [PATCH 05/21] add release notes --- doc/release_notes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 1ea2f5393..a74b00236 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -26,6 +26,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Include a dedicated cutout for Europe in bundle_config.yaml `PR #1125 `_ +* Drop duplicate entries in `AL_production.csv` data used in `build_industry_demand` rule `PR #1143 `_ + PyPSA-Earth 0.4.1 ================= From c35dee9cd852df844e82a11b059f2019c31f5bc5 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 Oct 2024 07:40:45 +0000 Subject: [PATCH 06/21] Add H2 carrier if not exists --- scripts/prepare_sector_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 40b3400bf..786d546bb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -245,7 +245,8 @@ def H2_liquid_fossil_conversions(n, costs): def add_hydrogen(n, costs): "function to add hydrogen as an energy carrier with its conversion technologies from and to AC" - n.add("Carrier", "H2") + if not "H2" in n.carriers.index: + n.add("Carrier", "H2") n.madd( "Bus", From e9d6bb9d3c7b28a3935437c2318329e04588f427 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 15 Oct 2024 07:58:05 +0000 Subject: [PATCH 07/21] Add battery carrier if not exists only --- scripts/prepare_sector_network.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 786d546bb..6cb4be6ed 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1089,7 +1089,9 @@ def add_aviation(n, cost): def add_storage(n, costs): "function to add the different types of storage systems" - n.add("Carrier", "battery") + + if not "battery" in n.carriers.index: + n.add("Carrier", "battery") n.madd( "Bus", From 2add4a9a4a20d1c53e78aa8a56df194e0eef62a2 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 17 Oct 2024 17:02:19 +0000 Subject: [PATCH 08/21] drop duplicate entries in AL_production.csv --- data/AL_production.csv | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/data/AL_production.csv b/data/AL_production.csv index 5f0f6734d..f025ceb1e 100644 --- a/data/AL_production.csv +++ b/data/AL_production.csv @@ -174,7 +174,6 @@ NE,0,2019,no available information online assumed value of 0 NU,0,2019,no available information online assumed value of 0 NF,0,2019,no available information online assumed value of 0 MP,0,2019,no available information online assumed value of 0 -NO,0,2019,no available information online assumed value of 0 OM,0,2019,no available information online assumed value of 0 PK,0,2019,no available information online assumed value of 0 PS,0,2019,no available information online assumed value of 0 @@ -187,10 +186,7 @@ PN,0,2019,no available information online assumed value of 0 PL,0,2019,no available information online assumed value of 0 PT,0,2019,no available information online assumed value of 0 PR,0,2019,no available information online assumed value of 0 -QA,0,2019,no available information online assumed value of 0 MK,0,2019,no available information online assumed value of 0 -RO,0,2019,no available information online assumed value of 0 -RU,0,2019,no available information online assumed value of 0 RW,0,2019,no available information online assumed value of 0 RE,0,2019,no available information online assumed value of 0 BL,0,2019,no available information online assumed value of 0 @@ -203,18 +199,15 @@ VC,0,2019,no available information online assumed value of 0 WS,0,2019,no available information online assumed value of 0 SM,0,2019,no available information online assumed value of 0 ST,0,2019,no available information online assumed value of 0 -SA,0,2019,no available information online assumed value of 0 SN,0,2019,no available information online assumed value of 0 RS,0,2019,no available information online assumed value of 0 SC,0,2019,no available information online assumed value of 0 SL,0,2019,no available information online assumed value of 0 SG,0,2019,no available information online assumed value of 0 SX,0,2019,no available information online assumed value of 0 -SK,0,2019,no available information online assumed value of 0 SI,0,2019,no available information online assumed value of 0 SB,0,2019,no available information online assumed value of 0 SO,0,2019,no available information online assumed value of 0 -ZA,0,2019,no available information online assumed value of 0 GS,0,2019,no available information online assumed value of 0 SS,0,2019,no available information online assumed value of 0 ES,0,2019,no available information online assumed value of 0 @@ -222,7 +215,6 @@ LK,0,2019,no available information online assumed value of 0 SD,0,2019,no available information online assumed value of 0 SR,0,2019,no available information online assumed value of 0 SJ,0,2019,no available information online assumed value of 0 -SE,0,2019,no available information online assumed value of 0 CH,0,2019,no available information online assumed value of 0 SY,0,2019,no available information online assumed value of 0 TW,0,2019,no available information online assumed value of 0 @@ -242,12 +234,10 @@ TV,0,2019,no available information online assumed value of 0 UG,0,2019,no available information online assumed value of 0 UA,0,2019,no available information online assumed value of 0 GB,0,2019,no available information online assumed value of 0 -US,0,2019,no available information online assumed value of 0 UM,0,2019,no available information online assumed value of 0 UY,0,2019,no available information online assumed value of 0 UZ,0,2019,no available information online assumed value of 0 VU,0,2019,no available information online assumed value of 0 -VE,0,2019,no available information online assumed value of 0 VN,0,2019,no available information online assumed value of 0 VG,0,2019,no available information online assumed value of 0 VI,0,2019,no available information online assumed value of 0 From ba3bc91cebd1079b4ed84a5052614d3489a4c583 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 17 Oct 2024 17:31:08 +0000 Subject: [PATCH 09/21] rise error if duplicate entries for the country are present --- scripts/build_industry_demand.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index c563c0248..1bc66d985 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -205,12 +205,21 @@ def match_technology(df): aluminium_year = snakemake.params.aluminium_year AL = read_csv_nafix("data/AL_production.csv", index_col=0) - # drop duplicate entries for the same country and year by keeping highest production data - AL = AL.sort_values(by=['country', 'Year', 'production[ktons/a]'], ascending=[True, True, False]).reset_index() - AL = AL.drop_duplicates(subset=["country", "Year"], keep='first').set_index("country") + # Filter data for the given year and countries AL_prod_tom = AL.query("Year == @aluminium_year and index in @countries_geo")[ "production[ktons/a]" - ].reindex(countries_geo, fill_value=0.0) + ] + + # Check for duplicate entries per country + duplicates = AL_prod_tom.index[AL_prod_tom.index.duplicated(keep=False)].unique() + if len(duplicates) > 0: + raise ValueError( + f"There are more than one entry for aluminium production data for the following countries: {', '.join(duplicates)} in 'data/AL_production.csv'" + ) + + # Reindex and fill missing values with 0.0 + AL_prod_tom = AL_prod_tom.reindex(countries_geo, fill_value=0.0) + AL_emissions = AL_prod_tom * emission_factors["non-ferrous metals"] Steel_emissions = ( From baa850085f2e046e96045d4d34a03e29c50c4eb1 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 18 Oct 2024 16:10:23 +0000 Subject: [PATCH 10/21] remove 0 entries from AL_production.csv --- data/AL_production.csv | 216 ----------------------------------------- 1 file changed, 216 deletions(-) diff --git a/data/AL_production.csv b/data/AL_production.csv index f025ceb1e..4df9fc339 100644 --- a/data/AL_production.csv +++ b/data/AL_production.csv @@ -30,219 +30,3 @@ SK,212,2019,https://en.wikipedia.org/wiki/List_of_countries_by_aluminium_product GR,184,2019,https://en.wikipedia.org/wiki/List_of_countries_by_aluminium_production SE,126,2019,https://en.wikipedia.org/wiki/List_of_countries_by_aluminium_production VE,108,2019,https://en.wikipedia.org/wiki/List_of_countries_by_aluminium_production -NG,0,2019,no available information online assumed value of 0 -BJ,0,2019,no available information online assumed value of 0 -AF,0,2019,no available information online assumed value of 0 -AL,0,2019,no available information online assumed value of 0 -DZ,0,2019,no available information online assumed value of 0 -AS,0,2019,no available information online assumed value of 0 -AD,0,2019,no available information online assumed value of 0 -AO,0,2019,no available information online assumed value of 0 -AI,0,2019,no available information online assumed value of 0 -AQ,0,2019,no available information online assumed value of 0 -AG,0,2019,no available information online assumed value of 0 -AM,0,2019,no available information online assumed value of 0 -AW,0,2019,no available information online assumed value of 0 -AT,0,2019,no available information online assumed value of 0 -AZ,0,2019,no available information online assumed value of 0 -BS,0,2019,no available information online assumed value of 0 -BD,0,2019,no available information online assumed value of 0 -BB,0,2019,no available information online assumed value of 0 -BY,0,2019,no available information online assumed value of 0 -BE,0,2019,no available information online assumed value of 0 -BZ,0,2019,no available information online assumed value of 0 -BM,0,2019,no available information online assumed value of 0 -BT,0,2019,no available information online assumed value of 0 -BO,0,2019,no available information online assumed value of 0 -BA,0,2019,no available information online assumed value of 0 -BW,0,2019,no available information online assumed value of 0 -BV,0,2019,no available information online assumed value of 0 -IO,0,2019,no available information online assumed value of 0 -BN,0,2019,no available information online assumed value of 0 -BG,0,2019,no available information online assumed value of 0 -BF,0,2019,no available information online assumed value of 0 -BI,0,2019,no available information online assumed value of 0 -CV,0,2019,no available information online assumed value of 0 -KH,0,2019,no available information online assumed value of 0 -CM,0,2019,no available information online assumed value of 0 -KY,0,2019,no available information online assumed value of 0 -CF,0,2019,no available information online assumed value of 0 -TD,0,2019,no available information online assumed value of 0 -CL,0,2019,no available information online assumed value of 0 -CX,0,2019,no available information online assumed value of 0 -CC,0,2019,no available information online assumed value of 0 -CO,0,2019,no available information online assumed value of 0 -KM,0,2019,no available information online assumed value of 0 -CG,0,2019,no available information online assumed value of 0 -CD,0,2019,no available information online assumed value of 0 -CK,0,2019,no available information online assumed value of 0 -CR,0,2019,no available information online assumed value of 0 -CI,0,2019,no available information online assumed value of 0 -HR,0,2019,no available information online assumed value of 0 -CU,0,2019,no available information online assumed value of 0 -CW,0,2019,no available information online assumed value of 0 -CY,0,2019,no available information online assumed value of 0 -CZ,0,2019,no available information online assumed value of 0 -DK,0,2019,no available information online assumed value of 0 -DJ,0,2019,no available information online assumed value of 0 -DM,0,2019,no available information online assumed value of 0 -DO,0,2019,no available information online assumed value of 0 -EC,0,2019,no available information online assumed value of 0 -SV,0,2019,no available information online assumed value of 0 -GQ,0,2019,no available information online assumed value of 0 -ER,0,2019,no available information online assumed value of 0 -EE,0,2019,no available information online assumed value of 0 -SZ,0,2019,no available information online assumed value of 0 -ET,0,2019,no available information online assumed value of 0 -FK,0,2019,no available information online assumed value of 0 -FO,0,2019,no available information online assumed value of 0 -FJ,0,2019,no available information online assumed value of 0 -FI,0,2019,no available information online assumed value of 0 -GF,0,2019,no available information online assumed value of 0 -PF,0,2019,no available information online assumed value of 0 -TF,0,2019,no available information online assumed value of 0 -GA,0,2019,no available information online assumed value of 0 -GM,0,2019,no available information online assumed value of 0 -GE,0,2019,no available information online assumed value of 0 -GH,0,2019,no available information online assumed value of 0 -GI,0,2019,no available information online assumed value of 0 -GL,0,2019,no available information online assumed value of 0 -GD,0,2019,no available information online assumed value of 0 -GP,0,2019,no available information online assumed value of 0 -GU,0,2019,no available information online assumed value of 0 -GT,0,2019,no available information online assumed value of 0 -GG,0,2019,no available information online assumed value of 0 -GN,0,2019,no available information online assumed value of 0 -GW,0,2019,no available information online assumed value of 0 -GY,0,2019,no available information online assumed value of 0 -HT,0,2019,no available information online assumed value of 0 -HM,0,2019,no available information online assumed value of 0 -VA,0,2019,no available information online assumed value of 0 -HN,0,2019,no available information online assumed value of 0 -HK,0,2019,no available information online assumed value of 0 -HU,0,2019,no available information online assumed value of 0 -IM,0,2019,no available information online assumed value of 0 -IL,0,2019,no available information online assumed value of 0 -IT,0,2019,no available information online assumed value of 0 -JM,0,2019,no available information online assumed value of 0 -JP,0,2019,no available information online assumed value of 0 -JE,0,2019,no available information online assumed value of 0 -JO,0,2019,no available information online assumed value of 0 -KE,0,2019,no available information online assumed value of 0 -KI,0,2019,no available information online assumed value of 0 -KP,0,2019,no available information online assumed value of 0 -KW,0,2019,no available information online assumed value of 0 -KG,0,2019,no available information online assumed value of 0 -LA,0,2019,no available information online assumed value of 0 -LV,0,2019,no available information online assumed value of 0 -LB,0,2019,no available information online assumed value of 0 -LS,0,2019,no available information online assumed value of 0 -LR,0,2019,no available information online assumed value of 0 -LY,0,2019,no available information online assumed value of 0 -LI,0,2019,no available information online assumed value of 0 -LT,0,2019,no available information online assumed value of 0 -LU,0,2019,no available information online assumed value of 0 -MO,0,2019,no available information online assumed value of 0 -MG,0,2019,no available information online assumed value of 0 -MW,0,2019,no available information online assumed value of 0 -MV,0,2019,no available information online assumed value of 0 -ML,0,2019,no available information online assumed value of 0 -MT,0,2019,no available information online assumed value of 0 -MH,0,2019,no available information online assumed value of 0 -MQ,0,2019,no available information online assumed value of 0 -MR,0,2019,no available information online assumed value of 0 -MU,0,2019,no available information online assumed value of 0 -YT,0,2019,no available information online assumed value of 0 -MX,0,2019,no available information online assumed value of 0 -FM,0,2019,no available information online assumed value of 0 -MD,0,2019,no available information online assumed value of 0 -MC,0,2019,no available information online assumed value of 0 -MN,0,2019,no available information online assumed value of 0 -ME,0,2019,no available information online assumed value of 0 -MS,0,2019,no available information online assumed value of 0 -MA,0,2019,no available information online assumed value of 0 -MM,0,2019,no available information online assumed value of 0 -PW,0,2019,no available information online assumed value of 0 -NA,0,2019,no available information online assumed value of 0 -NR,0,2019,no available information online assumed value of 0 -NP,0,2019,no available information online assumed value of 0 -NL,0,2019,no available information online assumed value of 0 -NC,0,2019,no available information online assumed value of 0 -NZ,0,2019,no available information online assumed value of 0 -NI,0,2019,no available information online assumed value of 0 -NE,0,2019,no available information online assumed value of 0 -NU,0,2019,no available information online assumed value of 0 -NF,0,2019,no available information online assumed value of 0 -MP,0,2019,no available information online assumed value of 0 -OM,0,2019,no available information online assumed value of 0 -PK,0,2019,no available information online assumed value of 0 -PS,0,2019,no available information online assumed value of 0 -PA,0,2019,no available information online assumed value of 0 -PG,0,2019,no available information online assumed value of 0 -PY,0,2019,no available information online assumed value of 0 -PE,0,2019,no available information online assumed value of 0 -PH,0,2019,no available information online assumed value of 0 -PN,0,2019,no available information online assumed value of 0 -PL,0,2019,no available information online assumed value of 0 -PT,0,2019,no available information online assumed value of 0 -PR,0,2019,no available information online assumed value of 0 -MK,0,2019,no available information online assumed value of 0 -RW,0,2019,no available information online assumed value of 0 -RE,0,2019,no available information online assumed value of 0 -BL,0,2019,no available information online assumed value of 0 -SH,0,2019,no available information online assumed value of 0 -KN,0,2019,no available information online assumed value of 0 -LC,0,2019,no available information online assumed value of 0 -MF,0,2019,no available information online assumed value of 0 -PM,0,2019,no available information online assumed value of 0 -VC,0,2019,no available information online assumed value of 0 -WS,0,2019,no available information online assumed value of 0 -SM,0,2019,no available information online assumed value of 0 -ST,0,2019,no available information online assumed value of 0 -SN,0,2019,no available information online assumed value of 0 -RS,0,2019,no available information online assumed value of 0 -SC,0,2019,no available information online assumed value of 0 -SL,0,2019,no available information online assumed value of 0 -SG,0,2019,no available information online assumed value of 0 -SX,0,2019,no available information online assumed value of 0 -SI,0,2019,no available information online assumed value of 0 -SB,0,2019,no available information online assumed value of 0 -SO,0,2019,no available information online assumed value of 0 -GS,0,2019,no available information online assumed value of 0 -SS,0,2019,no available information online assumed value of 0 -ES,0,2019,no available information online assumed value of 0 -LK,0,2019,no available information online assumed value of 0 -SD,0,2019,no available information online assumed value of 0 -SR,0,2019,no available information online assumed value of 0 -SJ,0,2019,no available information online assumed value of 0 -CH,0,2019,no available information online assumed value of 0 -SY,0,2019,no available information online assumed value of 0 -TW,0,2019,no available information online assumed value of 0 -TJ,0,2019,no available information online assumed value of 0 -TZ,0,2019,no available information online assumed value of 0 -TH,0,2019,no available information online assumed value of 0 -TL,0,2019,no available information online assumed value of 0 -TG,0,2019,no available information online assumed value of 0 -TK,0,2019,no available information online assumed value of 0 -TO,0,2019,no available information online assumed value of 0 -TT,0,2019,no available information online assumed value of 0 -TN,0,2019,no available information online assumed value of 0 -TR,0,2019,no available information online assumed value of 0 -TM,0,2019,no available information online assumed value of 0 -TC,0,2019,no available information online assumed value of 0 -TV,0,2019,no available information online assumed value of 0 -UG,0,2019,no available information online assumed value of 0 -UA,0,2019,no available information online assumed value of 0 -GB,0,2019,no available information online assumed value of 0 -UM,0,2019,no available information online assumed value of 0 -UY,0,2019,no available information online assumed value of 0 -UZ,0,2019,no available information online assumed value of 0 -VU,0,2019,no available information online assumed value of 0 -VN,0,2019,no available information online assumed value of 0 -VG,0,2019,no available information online assumed value of 0 -VI,0,2019,no available information online assumed value of 0 -WF,0,2019,no available information online assumed value of 0 -EH,0,2019,no available information online assumed value of 0 -YE,0,2019,no available information online assumed value of 0 -ZM,0,2019,no available information online assumed value of 0 -ZW,0,2019,no available information online assumed value of 0 From c25aa265411c10487e09ba847c55f064a8715504 Mon Sep 17 00:00:00 2001 From: = Date: Fri, 18 Oct 2024 16:22:40 +0000 Subject: [PATCH 11/21] assume 0 for countries with missing data --- scripts/build_industry_demand.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 1bc66d985..65d2b9a01 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -210,12 +210,10 @@ def match_technology(df): "production[ktons/a]" ] - # Check for duplicate entries per country - duplicates = AL_prod_tom.index[AL_prod_tom.index.duplicated(keep=False)].unique() - if len(duplicates) > 0: - raise ValueError( - f"There are more than one entry for aluminium production data for the following countries: {', '.join(duplicates)} in 'data/AL_production.csv'" - ) + # Check if aluminum data is missing for any countries + for country in countries_geo: + if country not in AL_prod_tom.index: + _logger.warning(f"No aluminum production data found for {country}. Filled with 0.0.") # Reindex and fill missing values with 0.0 AL_prod_tom = AL_prod_tom.reindex(countries_geo, fill_value=0.0) From af1474984e2369b101cf6e54368a58793f5f98ec Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Oct 2024 19:04:21 +0000 Subject: [PATCH 12/21] use H2 in both AC and DC buses in h2_hc_conversions function --- scripts/prepare_sector_network.py | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index 6cb4be6ed..a00d224fe 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1180,11 +1180,11 @@ def h2_hc_conversions(n, costs): if snakemake.config["sector"]["hydrogen"]["hydrogen_colors"]: n.madd( "Bus", - nodes + " blue H2", - location=nodes, + spatial.nodes + " blue H2", + location=spatial.nodes, carrier="blue H2", - x=n.buses.loc[list(nodes)].x.values, - y=n.buses.loc[list(nodes)].y.values, + x=n.buses.loc[list(spatial.nodes)].x.values, + y=n.buses.loc[list(spatial.nodes)].y.values, ) n.madd( @@ -1192,7 +1192,7 @@ def h2_hc_conversions(n, costs): spatial.nodes, suffix=" SMR CC", bus0=spatial.gas.nodes, - bus1=nodes + " blue H2", + bus1=spatial.nodes + " blue H2", bus2="co2 atmosphere", bus3=spatial.co2.nodes, p_nom_extendable=True, @@ -1207,9 +1207,9 @@ def h2_hc_conversions(n, costs): n.madd( "Link", - nodes + " blue H2", - bus0=nodes + " blue H2", - bus1=nodes + " H2", + spatial.nodes + " blue H2", + bus0=spatial.nodes + " blue H2", + bus1=spatial.nodes + " H2", carrier="blue H2", capital_cost=0, p_nom_extendable=True, @@ -1222,7 +1222,7 @@ def h2_hc_conversions(n, costs): spatial.nodes, suffix=" SMR CC", bus0=spatial.gas.nodes, - bus1=nodes + " H2", + bus1=spatial.nodes + " H2", bus2="co2 atmosphere", bus3=spatial.co2.nodes, p_nom_extendable=True, @@ -1239,18 +1239,18 @@ def h2_hc_conversions(n, costs): if snakemake.config["sector"]["hydrogen"]["hydrogen_colors"]: n.madd( "Bus", - nodes + " grey H2", - location=nodes, + spatial.nodes + " grey H2", + location=spatial.nodes, carrier="grey H2", - x=n.buses.loc[list(nodes)].x.values, - y=n.buses.loc[list(nodes)].y.values, + x=n.buses.loc[list(spatial.nodes)].x.values, + y=n.buses.loc[list(spatial.nodes)].y.values, ) n.madd( "Link", - nodes + " SMR", + spatial.nodes + " SMR", bus0=spatial.gas.nodes, - bus1=nodes + " grey H2", + bus1=spatial.nodes + " grey H2", bus2="co2 atmosphere", p_nom_extendable=True, carrier="SMR", @@ -1262,9 +1262,9 @@ def h2_hc_conversions(n, costs): n.madd( "Link", - nodes + " grey H2", - bus0=nodes + " grey H2", - bus1=nodes + " H2", + spatial.nodes + " grey H2", + bus0=spatial.nodes + " grey H2", + bus1=spatial.nodes + " H2", carrier="grey H2", capital_cost=0, p_nom_extendable=True, @@ -1274,9 +1274,9 @@ def h2_hc_conversions(n, costs): else: n.madd( "Link", - nodes + " SMR", + spatial.nodes + " SMR", bus0=spatial.gas.nodes, - bus1=nodes + " H2", + bus1=spatial.nodes + " H2", bus2="co2 atmosphere", p_nom_extendable=True, carrier="SMR", From de5799f345f3151527ed4244fac16b4062cb1e80 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Oct 2024 19:22:54 +0000 Subject: [PATCH 13/21] use AC and DC based H2 to make parameters of m.add consistent shape --- scripts/prepare_sector_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_sector_network.py b/scripts/prepare_sector_network.py index a00d224fe..acb6394bb 100644 --- a/scripts/prepare_sector_network.py +++ b/scripts/prepare_sector_network.py @@ -1344,7 +1344,7 @@ def add_shipping(n, costs): if options["shipping_hydrogen_liquefaction"]: n.madd( "Bus", - nodes, + spatial.nodes, suffix=" H2 liquid", carrier="H2 liquid", location=spatial.nodes, @@ -1373,7 +1373,7 @@ def add_shipping(n, costs): ): n.madd( "Load", - nodes, + spatial.nodes, suffix=" H2 for shipping", bus=shipping_bus, carrier="H2 for shipping", From faaa0b6fe9389845ca14cf935114dd09cb01021c Mon Sep 17 00:00:00 2001 From: Davide Fioriti <67809479+davide-f@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:35:44 +0100 Subject: [PATCH 14/21] Add precommit fix (#1153) * add precommit fix * Add release_note * Temporary disable of docformatter --- .github/workflows/ci.yml | 2 +- .pre-commit-config.yaml | 15 +++++++++------ doc/release_notes.rst | 2 ++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e892057e2..5887ac014 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ on: - cron: "0 5 * * TUE" env: - CACHE_NUMBER: 2 # Change this value to manually reset the environment cache + CACHE_NUMBER: 1 # Change this value to manually reset the environment cache jobs: build: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 54c3bcd16..32f9b1175 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -35,12 +35,15 @@ repos: types_or: [python, rst, markdown] files: ^(scripts|doc)/ -# Make docstrings PEP 257 compliant -- repo: https://github.com/PyCQA/docformatter - rev: v1.7.5 - hooks: - - id: docformatter - args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] +# # Make docstrings PEP 257 compliant +# - repo: https://github.com/PyCQA/docformatter +# rev: v1.7.5 +# hooks: +# - id: docformatter +# args: ["--in-place", "--make-summary-multi-line", "--pre-summary-newline"] +# entry: docformatter +# language: python +# types: [python] - repo: https://github.com/keewis/blackdoc rev: v0.3.9 diff --git a/doc/release_notes.rst b/doc/release_notes.rst index a900e5860..b05a396d8 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -30,6 +30,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Enable configfile specification for mock_snakemake `PR #1135 `_ +* Fix pre-commit docformatter python issue. `PR #1153 `__ + PyPSA-Earth 0.4.1 ================= From d5f81952f1383888c63f7c33d51c6914ed1317ae Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 18:38:34 +0100 Subject: [PATCH 15/21] docs(contributor): contrib-readme-action has updated readme (#1156) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- README.md | 81 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/README.md b/README.md index c424dfe34..3a234c693 100644 --- a/README.md +++ b/README.md @@ -311,14 +311,35 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe Emmanuel Bolarinwa + + + Eddy-JV +
+ Eddy Jalbout + + + + + + doneachh +
+ Anton Achhammer + + Tomkourou
Thomas Kouroughli - - + + + + finozzifa +
+ Finozzifa + + GridGrapher @@ -327,12 +348,20 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - finozzifa + + glenkiely-ieg
- Finozzifa + Null + + + cpschau +
+ Cschau + + + Emre-Yorat89 @@ -360,14 +389,6 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Ekaterina-Vo - - - - - cpschau -
- Cschau - @@ -377,24 +398,25 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe - - doneachh + + carlosfv92
- Anton Achhammer + Carlos Fernandez - + + - - asolavi + + rajesh-ieg
Null - - carlosfv92 + + asolavi
- Carlos Fernandez + Null @@ -403,8 +425,7 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Stephen J Lee - - + rsparks3 @@ -425,7 +446,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Juli-a-ko - + + squoilin @@ -446,8 +468,7 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Pietro Monticone - - + Netotse @@ -468,7 +489,8 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
Jess - + + jarry7 @@ -489,8 +511,7 @@ The documentation is available here: [documentation](https://pypsa-earth.readthe
EmreYorat - - + AndreCNF From f1b5c96f5106de943b242afa818b306d8e4c1e9b Mon Sep 17 00:00:00 2001 From: Ekaterina Date: Mon, 28 Oct 2024 18:42:24 +0100 Subject: [PATCH 16/21] Fix CI badge (#1149) * Fix badge in README * Update badge in read-the-docs * Fix typos --- README.md | 4 +--- doc/index.rst | 14 +++----------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 3a234c693..c20cde919 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,7 @@ by ## Development Status: **Stable and Active** -[![Status Linux](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-linux.yaml/badge.svg?branch=main&event=push)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-linux.yaml) -[![Status Mac](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-mac.yaml/badge.svg?branch=main&event=push)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-mac.yaml) -[![Status Windows](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-windows.yaml/badge.svg?branch=main&event=push)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-windows.yaml) +[![Test workflows](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml/badge.svg)](https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml) [![Documentation Status](https://readthedocs.org/projects/pypsa-earth/badge/?version=latest)](https://pypsa-earth.readthedocs.io/en/latest/?badge=latest) ![Size](https://img.shields.io/github/repo-size/pypsa-meets-earth/pypsa-earth) [![License: AGPL v3](https://img.shields.io/badge/License-AGPLv3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) diff --git a/doc/index.rst b/doc/index.rst index 0f7e5c20c..52ef166ae 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -14,17 +14,9 @@ Welcome to the PyPSA-Earth documentation! .. image:: https://img.shields.io/github/v/release/pypsa-meets-earth/pypsa-earth?include_prereleases :alt: GitHub release (latest by date including pre-releases) -.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-linux.yaml/badge.svg - :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions - :alt: CI Linux - -.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-mac.yaml/badge.svg - :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions - :alt: CI Mac - -.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci-windows.yaml/badge.svg - :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions - :alt: CI Windows +.. image:: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml/badge.svg + :target: https://github.com/pypsa-meets-earth/pypsa-earth/actions/workflows/ci.yml + :alt: CI .. image:: https://readthedocs.org/projects/pypsa-earth/badge/?version=latest :target: https://pypsa-earth.readthedocs.io/en/latest/?badge=latest From a5a4fe560c6c895aca83b38854ee779a4a022bc4 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Tue, 29 Oct 2024 13:18:25 +0100 Subject: [PATCH 17/21] feat: add new cutout for oceania --- configs/bundle_config.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/configs/bundle_config.yaml b/configs/bundle_config.yaml index cd8bd286f..7d1f55103 100644 --- a/configs/bundle_config.yaml +++ b/configs/bundle_config.yaml @@ -281,6 +281,19 @@ databundles: disable_by_opt: build_cutout: [all] + # cutouts bundle of the cutouts folder for Oceania continent + # Coordinate bounds: [min_x = 80, max_x = 180.0, min_y = -50, max_y = 20.] + # Size about 19 GB (zipped) + bundle_cutouts_oceania: + countries: [Oceania] + category: cutouts + destination: "cutouts" + urls: + gdrive: https://drive.google.com/file/d/1R8ELkXmW8jBBUFWRY0sbf-T14SJl4EUY/view?usp=sharing + output: [cutouts/cutout-2013-era5.nc] + disable_by_opt: + build_cutout: [all] + # cutouts bundle of the cutouts folder for the northern hemisphere # Note: this includes nearly the entire northern emisphere [long +-180, lat 1-85]. Size about 81GB (zipped) #bundle_cutouts_northern_hemisphere: From de8e9c8b08b70d3b873c90bc00759cccc4872a96 Mon Sep 17 00:00:00 2001 From: Emmanuel Bolarinwa Date: Tue, 29 Oct 2024 13:25:41 +0100 Subject: [PATCH 18/21] update release note --- doc/release_notes.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b05a396d8..20f5d046b 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -32,6 +32,9 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix pre-commit docformatter python issue. `PR #1153 `__ +* Include a dedicated cutout for Oceania in bundle_config.yaml `PR #1157 `_ + + PyPSA-Earth 0.4.1 ================= From 3836aedb5b16fd5b53a283beddfecd12cc13c2af Mon Sep 17 00:00:00 2001 From: Yerbol Akhmetov <113768325+yerbol-akhmetov@users.noreply.github.com> Date: Tue, 29 Oct 2024 23:30:10 +0500 Subject: [PATCH 19/21] Use absolute path for `config.default.yaml` in `_helpers.py` (#1137) * add absolute path to config.default.yaml * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add release notes * download files using absolute path, not based on os.getcwd() * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * add prefix to outputs of retrieve_databundle_light if exists * use absolute path for osm data download * define prefix in _helpers * import PREFIX in retrieve_databundle_light * add PREFIX to search for .nc or .csv file in PREFIX/data * revert to BASE_DIR usage * BASE_DIR to prepare_network for emission data * store config.yaml using absolute path * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- doc/release_notes.rst | 2 ++ scripts/_helpers.py | 10 ++++++++-- scripts/build_demand_profiles.py | 10 ++++++++-- scripts/build_renewable_profiles.py | 4 ++-- scripts/build_shapes.py | 17 +++++++---------- scripts/download_osm_data.py | 8 +++++--- scripts/prepare_network.py | 13 ++++++++----- scripts/retrieve_databundle_light.py | 17 +++++++++-------- 8 files changed, 49 insertions(+), 32 deletions(-) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index b05a396d8..4fc2d0f75 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -32,6 +32,8 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * Fix pre-commit docformatter python issue. `PR #1153 `__ +* Use BASE_DIR in rules and `_helpers.py` script for facilitate module import in subworkflow `PR #1137 `_ + PyPSA-Earth 0.4.1 ================= diff --git a/scripts/_helpers.py b/scripts/_helpers.py index 76bf0268d..ce97f6171 100644 --- a/scripts/_helpers.py +++ b/scripts/_helpers.py @@ -36,8 +36,14 @@ # filename of the regions definition config file REGIONS_CONFIG = "regions_definition_config.yaml" +# prefix when running pypsa-earth rules in different directories (if running in pypsa-earth as subworkflow) +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) -def check_config_version(config, fp_config="config.default.yaml"): +# absolute path to config.default.yaml +CONFIG_DEFAULT_PATH = os.path.join(BASE_DIR, "config.default.yaml") + + +def check_config_version(config, fp_config=CONFIG_DEFAULT_PATH): """ Check that a version of the local config.yaml matches to the actual config version as defined in config.default.yaml. @@ -87,7 +93,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): def copy_default_files(): - fn = Path("config.yaml") + fn = Path(os.path.join(BASE_DIR, "config.yaml")) if not fn.exists(): fn.write_text( "# Write down config entries differing from config.default.yaml\n\nrun: {}" diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py index 51f1193c0..3686364b2 100644 --- a/scripts/build_demand_profiles.py +++ b/scripts/build_demand_profiles.py @@ -50,7 +50,13 @@ import pypsa import scipy.sparse as sparse import xarray as xr -from _helpers import configure_logging, create_logger, read_csv_nafix, read_osm_config +from _helpers import ( + BASE_DIR, + configure_logging, + create_logger, + read_csv_nafix, + read_osm_config, +) from shapely.prepared import prep from shapely.validation import make_valid @@ -121,7 +127,7 @@ def get_load_paths_gegis(ssp_parentfolder, config): for continent in region_load: sel_ext = ".nc" for ext in [".nc", ".csv"]: - load_path = os.path.join(str(load_dir), str(continent) + str(ext)) + load_path = os.path.join(BASE_DIR, str(load_dir), str(continent) + str(ext)) if os.path.exists(load_path): sel_ext = ext break diff --git a/scripts/build_renewable_profiles.py b/scripts/build_renewable_profiles.py index 1ebf220b4..2dce6032e 100644 --- a/scripts/build_renewable_profiles.py +++ b/scripts/build_renewable_profiles.py @@ -202,7 +202,7 @@ import pandas as pd import progressbar as pgb import xarray as xr -from _helpers import configure_logging, create_logger +from _helpers import BASE_DIR, configure_logging, create_logger from add_electricity import load_powerplants from dask.distributed import Client from pypsa.geo import haversine @@ -559,7 +559,7 @@ def create_scaling_factor( # filter plants for hydro if snakemake.wildcards.technology.startswith("hydro"): country_shapes = gpd.read_file(paths.country_shapes) - hydrobasins = gpd.read_file(resource["hydrobasins"]) + hydrobasins = gpd.read_file(os.path.join(BASE_DIR, resource["hydrobasins"])) ppls = load_powerplants(snakemake.input.powerplants) hydro_ppls = ppls[ppls.carrier == "hydro"] diff --git a/scripts/build_shapes.py b/scripts/build_shapes.py index 22e6b68cf..8d3722de6 100644 --- a/scripts/build_shapes.py +++ b/scripts/build_shapes.py @@ -19,6 +19,7 @@ import requests import xarray as xr from _helpers import ( + BASE_DIR, configure_logging, create_logger, three_2_two_digits_country, @@ -85,7 +86,7 @@ def download_GADM(country_code, update=False, out_logging=False): GADM_url = f"https://geodata.ucdavis.edu/gadm/gadm4.1/gpkg/{GADM_filename}.gpkg" GADM_inputfile_gpkg = os.path.join( - os.getcwd(), + BASE_DIR, "data", "gadm", GADM_filename, @@ -489,7 +490,7 @@ def download_WorldPop_standard( ] WorldPop_inputfile = os.path.join( - os.getcwd(), "data", "WorldPop", WorldPop_filename + BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif if not os.path.exists(WorldPop_inputfile) or update is True: @@ -543,7 +544,7 @@ def download_WorldPop_API( WorldPop_filename = f"{two_2_three_digits_country(country_code).lower()}_ppp_{year}_UNadj_constrained.tif" # Request to get the file WorldPop_inputfile = os.path.join( - os.getcwd(), "data", "WorldPop", WorldPop_filename + BASE_DIR, "data", "WorldPop", WorldPop_filename ) # Input filepath tif os.makedirs(os.path.dirname(WorldPop_inputfile), exist_ok=True) year_api = int(str(year)[2:]) @@ -580,12 +581,10 @@ def convert_GDP(name_file_nc, year=2015, out_logging=False): name_file_tif = name_file_nc[:-2] + "tif" # path of the nc file - GDP_nc = os.path.join(os.getcwd(), "data", "GDP", name_file_nc) # Input filepath nc + GDP_nc = os.path.join(BASE_DIR, "data", "GDP", name_file_nc) # Input filepath nc # path of the tif file - GDP_tif = os.path.join( - os.getcwd(), "data", "GDP", name_file_tif - ) # Input filepath nc + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath nc # Check if file exists, otherwise throw exception if not os.path.exists(GDP_nc): @@ -628,9 +627,7 @@ def load_GDP( # path of the nc file name_file_tif = name_file_nc[:-2] + "tif" - GDP_tif = os.path.join( - os.getcwd(), "data", "GDP", name_file_tif - ) # Input filepath tif + GDP_tif = os.path.join(BASE_DIR, "data", "GDP", name_file_tif) # Input filepath tif if update | (not os.path.exists(GDP_tif)): if out_logging: diff --git a/scripts/download_osm_data.py b/scripts/download_osm_data.py index c92fdc2b4..c327a7ae4 100644 --- a/scripts/download_osm_data.py +++ b/scripts/download_osm_data.py @@ -30,7 +30,7 @@ import shutil from pathlib import Path -from _helpers import configure_logging, create_logger, read_osm_config +from _helpers import BASE_DIR, configure_logging, create_logger, read_osm_config from earth_osm import eo logger = create_logger(__name__) @@ -99,8 +99,10 @@ def convert_iso_to_geofk( run = snakemake.config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" - store_path_resources = Path.joinpath(Path().cwd(), "resources", RDIR, "osm", "raw") - store_path_data = Path.joinpath(Path().cwd(), "data", "osm") + store_path_resources = Path.joinpath( + Path(BASE_DIR), "resources", RDIR, "osm", "raw" + ) + store_path_data = Path.joinpath(Path(BASE_DIR), "data", "osm") country_list = country_list_to_geofk(snakemake.params.countries) eo.save_osm_data( diff --git a/scripts/prepare_network.py b/scripts/prepare_network.py index 9106fc90d..47d847b78 100755 --- a/scripts/prepare_network.py +++ b/scripts/prepare_network.py @@ -65,7 +65,7 @@ import pandas as pd import pypsa import requests -from _helpers import configure_logging, create_logger +from _helpers import BASE_DIR, configure_logging, create_logger from add_electricity import load_costs, update_transmission_costs idx = pd.IndexSlice @@ -85,11 +85,14 @@ def download_emission_data(): try: url = "https://jeodpp.jrc.ec.europa.eu/ftp/jrc-opendata/EDGAR/datasets/v60_GHG/CO2_excl_short-cycle_org_C/v60_GHG_CO2_excl_short-cycle_org_C_1970_2018.zip" with requests.get(url) as rq: - with open("data/co2.zip", "wb") as file: + with open(os.path.join(BASE_DIR, "data/co2.zip"), "wb") as file: file.write(rq.content) - file_path = "data/co2.zip" + file_path = os.path.join(BASE_DIR, "data/co2.zip") with ZipFile(file_path, "r") as zipObj: - zipObj.extract("v60_CO2_excl_short-cycle_org_C_1970_2018.xls", "data") + zipObj.extract( + "v60_CO2_excl_short-cycle_org_C_1970_2018.xls", + os.path.join(BASE_DIR, "data"), + ) os.remove(file_path) return "v60_CO2_excl_short-cycle_org_C_1970_2018.xls" except: @@ -117,7 +120,7 @@ def emission_extractor(filename, emission_year, country_names): """ # data reading process - datapath = os.path.join(os.getcwd(), "data", filename) + datapath = os.path.join(BASE_DIR, "data", filename) df = pd.read_excel(datapath, sheet_name="v6.0_EM_CO2_fossil_IPCC1996", skiprows=8) df.columns = df.iloc[0] df = df.set_index("Country_code_A3") diff --git a/scripts/retrieve_databundle_light.py b/scripts/retrieve_databundle_light.py index cf6e4c3b9..297599d4a 100644 --- a/scripts/retrieve_databundle_light.py +++ b/scripts/retrieve_databundle_light.py @@ -89,6 +89,7 @@ import pandas as pd import yaml from _helpers import ( + BASE_DIR, configure_logging, create_country_list, create_logger, @@ -143,7 +144,7 @@ def download_and_unzip_zenodo(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["zenodo"] if hot_run: @@ -188,7 +189,7 @@ def download_and_unzip_gdrive(config, rootpath, hot_run=True, disable_progress=F """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["gdrive"] # retrieve file_id from path @@ -266,7 +267,7 @@ def download_and_unzip_protectedplanet( """ resource = config["category"] file_path = os.path.join(rootpath, "tempfile_wpda.zip") - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["protectedplanet"] def get_first_day_of_month(date): @@ -438,7 +439,7 @@ def download_and_unzip_direct(config, rootpath, hot_run=True, disable_progress=F True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url = config["urls"]["direct"] file_path = os.path.join(destination, os.path.basename(url)) @@ -492,7 +493,7 @@ def download_and_unzip_hydrobasins( True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) url_templ = config["urls"]["hydrobasins"]["base_url"] suffix_list = config["urls"]["hydrobasins"]["suffixes"] @@ -543,7 +544,7 @@ def download_and_unzip_post(config, rootpath, hot_run=True, disable_progress=Fal True when download is successful, False otherwise """ resource = config["category"] - destination = os.path.relpath(config["destination"]) + destination = os.path.join(BASE_DIR, config["destination"]) # load data for post method postdata = config["urls"]["post"] @@ -792,8 +793,8 @@ def datafiles_retrivedatabundle(config): def merge_hydrobasins_shape(config_hydrobasin, hydrobasins_level): - basins_path = config_hydrobasin["destination"] - output_fl = config_hydrobasin["output"][0] + basins_path = os.path.join(BASE_DIR, config_hydrobasin["destination"]) + output_fl = os.path.join(BASE_DIR, config_hydrobasin["output"][0]) files_to_merge = [ "hybas_{0:s}_lev{1:02d}_v1c.shp".format(suffix, hydrobasins_level) From 2cd79bea7dff0c38d83aaf8dcc8b9a8cce25e5f8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 29 Oct 2024 21:39:25 +0000 Subject: [PATCH 20/21] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- scripts/build_industry_demand.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/build_industry_demand.py b/scripts/build_industry_demand.py index 65d2b9a01..7074e5e89 100644 --- a/scripts/build_industry_demand.py +++ b/scripts/build_industry_demand.py @@ -213,7 +213,9 @@ def match_technology(df): # Check if aluminum data is missing for any countries for country in countries_geo: if country not in AL_prod_tom.index: - _logger.warning(f"No aluminum production data found for {country}. Filled with 0.0.") + _logger.warning( + f"No aluminum production data found for {country}. Filled with 0.0." + ) # Reindex and fill missing values with 0.0 AL_prod_tom = AL_prod_tom.reindex(countries_geo, fill_value=0.0) From 98121e81a916665bc7298e0076b6dc2913cc09b4 Mon Sep 17 00:00:00 2001 From: yerbol-akhmetov Date: Wed, 30 Oct 2024 19:40:38 +0500 Subject: [PATCH 21/21] add release notes --- doc/release_notes.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes.rst b/doc/release_notes.rst index 5b24e23d8..0d1b7c746 100644 --- a/doc/release_notes.rst +++ b/doc/release_notes.rst @@ -40,6 +40,7 @@ E.g. if a new rule becomes available describe how to use it `make test` and in o * The computation of `hydro_profile.nc` in `build_renewable_profiles.py` is not differentiated whether alternative clustering is applied or not; the indexing of the different power plants in `add_electricity.py` is performed according to the bus either in case alternative clustering is applied or not and a `hydro_inflow_factor` is computed prior to the computation of `inflow_t` to split the inflow according to the capacity of each different unit of each power plant (if more units are present). `PR #1119 `_ +* Fix bugs in `prepare_sector_network.py` related to links with H2 buses and bug of re-addition of H2 and battery carriers in present `PR #1145 `_ PyPSA-Earth 0.4.1 =================