From b992acd62ab0928b20f729e020240989726ee02a Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 19 Oct 2023 00:20:09 +0300
Subject: [PATCH 01/67] Add a function to fetch isolated nodes to network

---
 scripts/simplify_network.py | 113 ++++++++++++++++++++++++++++++++++++
 1 file changed, 113 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 4e2d1c2c0..9796aeea6 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -738,6 +738,119 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
+def merge_into_network(n, aggregation_strategies=dict()):
+    """
+    Find isolated nodes in the network and merge those of them which have load
+    value below than a specified threshold into a single isolated node which
+    represents all the remote generation.
+
+    Parameters
+    ----------
+    n : PyPSA.Network
+        Original network
+    threshold : float
+        Load power used as a threshold to merge isolated nodes
+
+    Returns
+    -------
+    modified network
+    """
+    # keep original values of the overall load and generation in the network
+    # to track changes due to drop of buses
+    generators_mean_origin = n.generators.p_nom.mean()
+    load_mean_origin = n.loads_t.p_set.mean().mean()
+
+    n.determine_network_topology()
+
+    # duplicated sub-networks mean that there is at least one interconnection between buses
+    i_islands = n.buses[
+        (~n.buses.duplicated(subset=["sub_network"], keep=False))
+        & (n.buses.carrier == "AC")
+    ].index
+
+    # TODO not sure filtering is needed
+    # # isolated buses with load below than a specified threshold should be merged
+    # i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
+    # i_suffic_load = i_load_islands[
+    #     n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
+    # ]
+
+    # TODO add a second nodes for debug purposes
+    # i_islands = i_islands.append(pd.Index(["10"]))
+
+    i_connected = n.buses.index.difference(i_islands)
+    points_buses = np.array(
+        list(zip(n.buses.loc[i_connected].x, n.buses.loc[i_connected].y))
+    )
+    islands_points = np.array(
+        list(zip(n.buses.loc[i_islands].x, n.buses.loc[i_islands].y))
+    )
+
+    btree = cKDTree(points_buses)
+    dist0, idx0 = btree.query(islands_points, k=1)
+
+    # works for a single isolated bus
+    # x_nearest, y_nearest = points_buses[idx0[0]]
+    # nearest_bus_df = n.buses.loc[(n.buses.x == x_nearest) & (n.buses.y == y_nearest)]
+
+    # TODO if there is only a single bus -> idx0 is np.array with a single element
+    # which leads to issues: ValueError: not enough values to unpack (expected 2, got 1)
+    x_nearest, y_nearest = points_buses[idx0]
+
+    nearest_bus_list = [
+        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)]
+        for x, y in list([x_nearest, y_nearest])
+    ]
+    nearest_bus_df = pd.concat(nearest_bus_list)
+
+    # each isolated node should be mapped into the closes non-isolated node
+    map_isolated_node_by_country = (
+        n.buses.assign(bus_id=n.buses.index)
+        .loc[nearest_bus_df.index]
+        .groupby("country")["bus_id"]
+        .first()
+        .to_dict()
+    )
+    isolated_buses_mapping = n.buses.loc[i_islands, "country"].replace(
+        map_isolated_node_by_country
+    )
+    busmap = (
+        n.buses.index.to_series()
+        .replace(isolated_buses_mapping)
+        .astype(str)
+        .rename("busmap")
+    )
+
+    # return the original network if no changes are detected
+    if (busmap.index == busmap).all():
+        return n, n.buses.index.to_series()
+
+    bus_strategies, generator_strategies = get_aggregation_strategies(
+        aggregation_strategies
+    )
+
+    clustering = get_clustering_from_busmap(
+        n,
+        busmap,
+        bus_strategies=bus_strategies,
+        aggregate_generators_weighted=True,
+        aggregate_generators_carriers=None,
+        aggregate_one_ports=["Load", "StorageUnit"],
+        line_length_factor=1.0,
+        generator_strategies=generator_strategies,
+        scale_link_capital_costs=False,
+    )
+
+    load_mean_final = n.loads_t.p_set.mean().mean()
+    generators_mean_final = n.generators.p_nom.mean()
+
+    logger.info(
+        f"Merged {len(islands_points)} isolated buses into the network. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
+    )
+
+    return clustering.network, busmap
+
+
 def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
     """
     Find isolated nodes in the network and merge those of them which have load

From f389d02d0b3c23490c23596c3b5141b0ec9a7fb1 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 19 Oct 2023 00:20:55 +0300
Subject: [PATCH 02/67] Apply fetch-isolated

---
 scripts/simplify_network.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9796aeea6..14242893a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -1094,6 +1094,13 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         )
         busmaps.append(merged_nodes_map)
 
+    # TODO Add a configuration option
+    n, fetched_nodes_map = merge_into_network(
+        n,
+        aggregation_strategies=aggregation_strategies,
+    )
+    busmaps.append(fetched_nodes_map)
+
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output.network)
 

From ea0eb882d5de6ef4447965e84f1f1a60236e109d Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 19 Oct 2023 00:21:06 +0300
Subject: [PATCH 03/67] Add import

---
 scripts/simplify_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 14242893a..a59387aed 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -104,6 +104,7 @@
 )
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
+from scipy.spatial import cKDTree
 
 sys.settrace
 

From a399ae1d9d354664292dcdf9d01f483b37e93aa0 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 19 Oct 2023 00:21:06 +0300
Subject: [PATCH 04/67] Add import

---
 scripts/simplify_network.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 14242893a..9312ea7b3 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -104,6 +104,7 @@
 )
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
+from scipy.spatial import cKDTree
 
 sys.settrace
 
@@ -798,8 +799,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
     x_nearest, y_nearest = points_buses[idx0]
 
     nearest_bus_list = [
-        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)]
-        for x, y in list([x_nearest, y_nearest])
+        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)] for x, y in points_nearest
     ]
     nearest_bus_df = pd.concat(nearest_bus_list)
 

From e8262b8cff6913535026280a866d6bb863a302d4 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 27 Oct 2023 02:18:57 +0300
Subject: [PATCH 05/67] Generalize sub-setting of an array

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9312ea7b3..0124795e4 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -796,7 +796,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
 
     # TODO if there is only a single bus -> idx0 is np.array with a single element
     # which leads to issues: ValueError: not enough values to unpack (expected 2, got 1)
-    x_nearest, y_nearest = points_buses[idx0]
+    points_nearest = [points_buses[i] for i in idx0]
 
     nearest_bus_list = [
         n.buses.loc[(n.buses.x == x) & (n.buses.y == y)] for x, y in points_nearest

From 52eb9a9eaddba7d9e36c149af2f97b1ff6123479 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 27 Oct 2023 02:19:13 +0300
Subject: [PATCH 06/67] Improve a message

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 0124795e4..60c96167c 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -845,7 +845,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
     generators_mean_final = n.generators.p_nom.mean()
 
     logger.info(
-        f"Merged {len(islands_points)} isolated buses into the network. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
+        f"Fetched {len(islands_points)} isolated buses into the network. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
     )
 
     return clustering.network, busmap

From 1af3d4153127c058a2cf6c357ab21a9acc5b50ca Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 27 Oct 2023 02:19:22 +0300
Subject: [PATCH 07/67] Remove comments

---
 scripts/simplify_network.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 60c96167c..9d4a3ecde 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -776,9 +776,6 @@ def merge_into_network(n, aggregation_strategies=dict()):
     #     n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
     # ]
 
-    # TODO add a second nodes for debug purposes
-    # i_islands = i_islands.append(pd.Index(["10"]))
-
     i_connected = n.buses.index.difference(i_islands)
     points_buses = np.array(
         list(zip(n.buses.loc[i_connected].x, n.buses.loc[i_connected].y))
@@ -790,12 +787,6 @@ def merge_into_network(n, aggregation_strategies=dict()):
     btree = cKDTree(points_buses)
     dist0, idx0 = btree.query(islands_points, k=1)
 
-    # works for a single isolated bus
-    # x_nearest, y_nearest = points_buses[idx0[0]]
-    # nearest_bus_df = n.buses.loc[(n.buses.x == x_nearest) & (n.buses.y == y_nearest)]
-
-    # TODO if there is only a single bus -> idx0 is np.array with a single element
-    # which leads to issues: ValueError: not enough values to unpack (expected 2, got 1)
     points_nearest = [points_buses[i] for i in idx0]
 
     nearest_bus_list = [

From 6391b21846a8913c896de154a44ff1f3b32ce389 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 27 Oct 2023 02:45:02 +0300
Subject: [PATCH 08/67] Add a return on condition in case there is no isolated
 nodes

---
 scripts/simplify_network.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9d4a3ecde..724ddacf0 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -777,6 +777,11 @@ def merge_into_network(n, aggregation_strategies=dict()):
     # ]
 
     i_connected = n.buses.index.difference(i_islands)
+
+    # return the original network if no isolated nodes are detected
+    if len(i_islands) == 0:
+        return n, n.buses.index.to_series()
+
     points_buses = np.array(
         list(zip(n.buses.loc[i_connected].x, n.buses.loc[i_connected].y))
     )

From a92a8fc9e1383f045c57641913cc090d23416dd7 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 1 Nov 2023 00:43:35 +0300
Subject: [PATCH 09/67] Clarify a comment

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 724ddacf0..e961243be 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -769,7 +769,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
         & (n.buses.carrier == "AC")
     ].index
 
-    # TODO not sure filtering is needed
+    # TODO filtering may be applied to decide if the isolated buses should be fetched
     # # isolated buses with load below than a specified threshold should be merged
     # i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
     # i_suffic_load = i_load_islands[

From 4d46fe55171c794acef842adb18492bd768f442e Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 2 Nov 2023 22:45:48 +0300
Subject: [PATCH 10/67] Add a TODO comment

---
 scripts/simplify_network.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index e961243be..140240152 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -789,6 +789,8 @@ def merge_into_network(n, aggregation_strategies=dict()):
         list(zip(n.buses.loc[i_islands].x, n.buses.loc[i_islands].y))
     )
 
+    # gdf_map = gpd.sjoin_nearest(gdf1, gdf2, how="left")
+
     btree = cKDTree(points_buses)
     dist0, idx0 = btree.query(islands_points, k=1)
 

From e641447eb609a1a25d854f06339ba3afeca1b80f Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 8 Dec 2023 01:32:38 +0300
Subject: [PATCH 11/67] Add filtering by carrier for the network to merge into

---
 scripts/simplify_network.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 140240152..97acc8403 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -776,12 +776,12 @@ def merge_into_network(n, aggregation_strategies=dict()):
     #     n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
     # ]
 
-    i_connected = n.buses.index.difference(i_islands)
-
     # return the original network if no isolated nodes are detected
     if len(i_islands) == 0:
         return n, n.buses.index.to_series()
 
+    i_connected = n.buses.loc[n.buses.carrier == "AC"].index.difference(i_islands)
+
     points_buses = np.array(
         list(zip(n.buses.loc[i_connected].x, n.buses.loc[i_connected].y))
     )

From febe2a7b17c26d2e3b4552d10aa399e18e2901d1 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 8 Dec 2023 01:48:32 +0300
Subject: [PATCH 12/67] Remove a not-necessary comment

---
 scripts/simplify_network.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 97acc8403..1126cbbd4 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -789,8 +789,6 @@ def merge_into_network(n, aggregation_strategies=dict()):
         list(zip(n.buses.loc[i_islands].x, n.buses.loc[i_islands].y))
     )
 
-    # gdf_map = gpd.sjoin_nearest(gdf1, gdf2, how="left")
-
     btree = cKDTree(points_buses)
     dist0, idx0 = btree.query(islands_points, k=1)
 

From dc9be87883c95155db589aafc2a294d13b83ff23 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 9 Dec 2023 01:54:59 +0300
Subject: [PATCH 13/67] Improve implementation of buses merging

---
 scripts/simplify_network.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 1126cbbd4..c9aeb1847 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -89,6 +89,7 @@
 import sys
 from functools import reduce
 
+import geopandas as gpd
 import numpy as np
 import pandas as pd
 import pypsa
@@ -105,6 +106,7 @@
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
 from scipy.spatial import cKDTree
+from shapely.geometry import Point
 
 sys.settrace
 
@@ -789,13 +791,17 @@ def merge_into_network(n, aggregation_strategies=dict()):
         list(zip(n.buses.loc[i_islands].x, n.buses.loc[i_islands].y))
     )
 
-    btree = cKDTree(points_buses)
-    dist0, idx0 = btree.query(islands_points, k=1)
+    gds_buses = gpd.GeoSeries(map(Point, points_buses))
+    gds_islands = gpd.GeoSeries(map(Point, islands_points))
 
-    points_nearest = [points_buses[i] for i in idx0]
+    gdf_buses = gpd.GeoDataFrame(geometry=gds_buses)
+    gdf_islands = gpd.GeoDataFrame(geometry=gds_islands)
+
+    gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_buses, how="left")
 
     nearest_bus_list = [
-        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)] for x, y in points_nearest
+        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)]
+        for x, y in zip(gdf_map["geometry"].x, gdf_map["geometry"].y)
     ]
     nearest_bus_df = pd.concat(nearest_bus_list)
 

From dd3a03945d87d707f87181da7d196920253e86de Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 19:11:27 +0300
Subject: [PATCH 14/67] Wrap-up into a function transformation of a buses
 dataframe into geopandas dataframe

---
 scripts/simplify_network.py | 23 ++++++++++-------------
 1 file changed, 10 insertions(+), 13 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index fc9a1af02..78a9063c9 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,6 +746,14 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
+def transform_to_gdf(buses_df, i_buses):
+    points_buses = np.array(list(zip(buses_df.loc[i_buses].x, buses_df.loc[i_buses].y)))
+    gds_buses = gpd.GeoSeries(map(Point, points_buses))
+    gdf_buses = gpd.GeoDataFrame(geometry=gds_buses)
+
+    return gdf_buses
+
+
 def merge_into_network(n, aggregation_strategies=dict()):
     """
     Find isolated nodes in the network and merge those of them which have load
@@ -788,19 +796,8 @@ def merge_into_network(n, aggregation_strategies=dict()):
         return n, n.buses.index.to_series()
 
     i_connected = n.buses.loc[n.buses.carrier == "AC"].index.difference(i_islands)
-
-    points_buses = np.array(
-        list(zip(n.buses.loc[i_connected].x, n.buses.loc[i_connected].y))
-    )
-    islands_points = np.array(
-        list(zip(n.buses.loc[i_islands].x, n.buses.loc[i_islands].y))
-    )
-
-    gds_buses = gpd.GeoSeries(map(Point, points_buses))
-    gds_islands = gpd.GeoSeries(map(Point, islands_points))
-
-    gdf_buses = gpd.GeoDataFrame(geometry=gds_buses)
-    gdf_islands = gpd.GeoDataFrame(geometry=gds_islands)
+    gdf_buses = transform_to_gdf(buses_df=n.buses, i_buses=i_connected)
+    gdf_islands = transform_to_gdf(buses_df=n.buses, i_buses=i_islands)
 
     gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_buses, how="left")
 

From 831182b3567ffbc430078fe70c84184fc0167c3b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 22:09:52 +0300
Subject: [PATCH 15/67] Fix selection of the buses to be merged with

---
 scripts/simplify_network.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 78a9063c9..36b3d72d0 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -801,9 +801,12 @@ def merge_into_network(n, aggregation_strategies=dict()):
 
     gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_buses, how="left")
 
+    buses_to_merge_with = gdf_buses.loc[gdf_map.index_right]
     nearest_bus_list = [
         n.buses.loc[(n.buses.x == x) & (n.buses.y == y)]
-        for x, y in zip(gdf_map["geometry"].x, gdf_map["geometry"].y)
+        for x, y in zip(
+            buses_to_merge_with["geometry"].x, buses_to_merge_with["geometry"].y
+        )
     ]
     nearest_bus_df = pd.concat(nearest_bus_list)
 

From 813ce0aadc504187ca530d604f00a8e77b600beb Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 23:37:33 +0300
Subject: [PATCH 16/67] Read CRS from the network

---
 scripts/simplify_network.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 36b3d72d0..bed00f90a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -776,6 +776,8 @@ def merge_into_network(n, aggregation_strategies=dict()):
     generators_mean_origin = n.generators.p_nom.mean()
     load_mean_origin = n.loads_t.p_set.mean().mean()
 
+    network_crs = n.meta["crs"]["geo_crs"]
+
     n.determine_network_topology()
 
     # duplicated sub-networks mean that there is at least one interconnection between buses

From aa0f079d5d0f67813619af70ab0debba7ea0e79b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 23:47:48 +0300
Subject: [PATCH 17/67] Re-implement the spatial transformation function

---
 scripts/simplify_network.py | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index bed00f90a..8bccfd408 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,11 +746,13 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
-def transform_to_gdf(buses_df, i_buses):
-    points_buses = np.array(list(zip(buses_df.loc[i_buses].x, buses_df.loc[i_buses].y)))
-    gds_buses = gpd.GeoSeries(map(Point, points_buses))
-    gdf_buses = gpd.GeoDataFrame(geometry=gds_buses)
-
+def transform_to_gdf(buses_df, i_buses, network_crs):
+    points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y"]]
+    gdf_buses = gpd.GeoDataFrame(
+        points_buses,
+        geometry=gpd.points_from_xy(points_buses.x, points_buses.y),
+        crs=network_crs,
+    )
     return gdf_buses
 
 
@@ -798,19 +800,17 @@ def merge_into_network(n, aggregation_strategies=dict()):
         return n, n.buses.index.to_series()
 
     i_connected = n.buses.loc[n.buses.carrier == "AC"].index.difference(i_islands)
-    gdf_buses = transform_to_gdf(buses_df=n.buses, i_buses=i_connected)
-    gdf_islands = transform_to_gdf(buses_df=n.buses, i_buses=i_islands)
+    gdf_buses = transform_to_gdf(
+        buses_df=n.buses, i_buses=i_connected, network_crs=network_crs
+    )
+    gdf_islands = transform_to_gdf(
+        buses_df=n.buses, i_buses=i_islands, network_crs=network_crs
+    )
 
+    # map each isolated bus into the closest non-isolated
     gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_buses, how="left")
 
-    buses_to_merge_with = gdf_buses.loc[gdf_map.index_right]
-    nearest_bus_list = [
-        n.buses.loc[(n.buses.x == x) & (n.buses.y == y)]
-        for x, y in zip(
-            buses_to_merge_with["geometry"].x, buses_to_merge_with["geometry"].y
-        )
-    ]
-    nearest_bus_df = pd.concat(nearest_bus_list)
+    nearest_bus_df = n.buses.loc[n.buses.bus_id.isin(gdf_map.bus_id_right)]
 
     # each isolated node should be mapped into the closes non-isolated node
     map_isolated_node_by_country = (
@@ -854,7 +854,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
     generators_mean_final = n.generators.p_nom.mean()
 
     logger.info(
-        f"Fetched {len(islands_points)} isolated buses into the network. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
+        f"Fetched {len(gdf_islands)} isolated buses into the network. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
     )
 
     return clustering.network, busmap

From 6743186015ee5e654dec25c7254123b6d6c349fb Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 23:55:32 +0300
Subject: [PATCH 18/67] Improve naming

---
 scripts/simplify_network.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 8bccfd408..45fbf0f09 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -800,7 +800,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
         return n, n.buses.index.to_series()
 
     i_connected = n.buses.loc[n.buses.carrier == "AC"].index.difference(i_islands)
-    gdf_buses = transform_to_gdf(
+    gdf_backbone_buses = transform_to_gdf(
         buses_df=n.buses, i_buses=i_connected, network_crs=network_crs
     )
     gdf_islands = transform_to_gdf(
@@ -808,8 +808,7 @@ def merge_into_network(n, aggregation_strategies=dict()):
     )
 
     # map each isolated bus into the closest non-isolated
-    gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_buses, how="left")
-
+    gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_backbone_buses, how="left")
     nearest_bus_df = n.buses.loc[n.buses.bus_id.isin(gdf_map.bus_id_right)]
 
     # each isolated node should be mapped into the closes non-isolated node

From b79b3647021cb9f0ceb185802b08f3078cd368c8 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 4 Jan 2024 00:17:29 +0300
Subject: [PATCH 19/67] Add bus_id column

---
 scripts/simplify_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 45fbf0f09..09a22f8a7 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -747,6 +747,7 @@ def drop_isolated_nodes(n, threshold):
 
 
 def transform_to_gdf(buses_df, i_buses, network_crs):
+    buses_df["bus_id"] = buses_df.index
     points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y"]]
     gdf_buses = gpd.GeoDataFrame(
         points_buses,

From 4af66f00209d26e5989fba6df1ae1a5070e8696d Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 4 Jan 2024 00:18:10 +0300
Subject: [PATCH 20/67] Add a function to identify isolated networks

---
 scripts/simplify_network.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 09a22f8a7..c6c523192 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,6 +746,14 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
+def find_isolated_sub_networks(buses_df, n_buses_thresh):
+    buses_df["bus_id"] = buses_df.index
+    grouped_by_subnw = buses_df.groupby("sub_network").count()
+    i_isol_subnetw = grouped_by_subnw[grouped_by_subnw.bus_id < n_buses_thresh].index
+    i_islands = buses_df.loc[buses_df.sub_network.isin(i_isol_subnetw)]
+    return i_islands
+
+
 def transform_to_gdf(buses_df, i_buses, network_crs):
     buses_df["bus_id"] = buses_df.index
     points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y"]]

From b8d127e883fc30320fca50826fc9f70d1581bb9d Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 4 Jan 2024 00:18:51 +0300
Subject: [PATCH 21/67] Handle isolated networks

---
 scripts/simplify_network.py | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index c6c523192..82ed9f382 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -765,7 +765,7 @@ def transform_to_gdf(buses_df, i_buses, network_crs):
     return gdf_buses
 
 
-def merge_into_network(n, aggregation_strategies=dict()):
+def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
     """
     Find isolated nodes in the network and merge those of them which have load
     value below than a specified threshold into a single isolated node which
@@ -791,11 +791,16 @@ def merge_into_network(n, aggregation_strategies=dict()):
 
     n.determine_network_topology()
 
-    # duplicated sub-networks mean that there is at least one interconnection between buses
-    i_islands = n.buses[
-        (~n.buses.duplicated(subset=["sub_network"], keep=False))
-        & (n.buses.carrier == "AC")
-    ].index
+    if isol_threshold_n > 1:
+        i_islands = find_isolated_sub_networks(
+            buses_df=n.buses, n_buses_thresh=isol_threshold_n
+        ).index
+    else:
+        # duplicated sub-networks mean that there is at least one interconnection between buses
+        i_islands = n.buses[
+            (~n.buses.duplicated(subset=["sub_network"], keep=False))
+            & (n.buses.carrier == "AC")
+        ].index
 
     # TODO filtering may be applied to decide if the isolated buses should be fetched
     # # isolated buses with load below than a specified threshold should be merged

From 2460f028820e6fd1ee073d017c8db4278513b3fb Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 4 Jan 2024 00:33:51 +0300
Subject: [PATCH 22/67] Fix threshold

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 82ed9f382..5f6769dc6 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -749,7 +749,7 @@ def drop_isolated_nodes(n, threshold):
 def find_isolated_sub_networks(buses_df, n_buses_thresh):
     buses_df["bus_id"] = buses_df.index
     grouped_by_subnw = buses_df.groupby("sub_network").count()
-    i_isol_subnetw = grouped_by_subnw[grouped_by_subnw.bus_id < n_buses_thresh].index
+    i_isol_subnetw = grouped_by_subnw[grouped_by_subnw.bus_id <= n_buses_thresh].index
     i_islands = buses_df.loc[buses_df.sub_network.isin(i_isol_subnetw)]
     return i_islands
 

From dc75850a2a77c60e065457c38c6b99f4987eaee7 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 4 Jan 2024 00:34:16 +0300
Subject: [PATCH 23/67] Remove lines corresponding to the isolated buses

---
 scripts/simplify_network.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 5f6769dc6..963341b21 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -825,6 +825,9 @@ def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
     gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_backbone_buses, how="left")
     nearest_bus_df = n.buses.loc[n.buses.bus_id.isin(gdf_map.bus_id_right)]
 
+    i_lines_islands = n.lines.loc[n.lines.bus1.isin(i_islands)].index
+    n.mremove("Line", i_lines_islands)
+
     # each isolated node should be mapped into the closes non-isolated node
     map_isolated_node_by_country = (
         n.buses.assign(bus_id=n.buses.index)

From 1f7cc78f7e3f03bcf8aa82f8f3d01c48d757966d Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 1 Jan 2024 18:55:50 +0300
Subject: [PATCH 24/67] Exclude population raster from demand distribution

---
 scripts/build_demand_profiles.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py
index 6a76c2275..3e3c44104 100644
--- a/scripts/build_demand_profiles.py
+++ b/scripts/build_demand_profiles.py
@@ -170,15 +170,19 @@ def upsample(cntry, group):
             gdp_n = pd.Series(
                 transfer.dot(shapes_cntry["gdp"].fillna(1.0).values), index=group.index
             )
-            pop_n = pd.Series(
-                transfer.dot(shapes_cntry["pop"].fillna(1.0).values), index=group.index
-            )
+            # TODO replace quick-and-dirty fix with a more appropriate approach
+            # pop_n = pd.Series(
+            #     transfer.dot(shapes_cntry["pop"].fillna(1.0).values), index=group.index
+            # )
 
             # relative factors 0.6 and 0.4 have been determined from a linear
             # regression on the country to EU continent load data
             # (refer to vresutils.load._upsampling_weights)
             # TODO: require adjustment for Africa
-            factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n))
+            # factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n))
+
+            # TODO replace quick-and-dirty fix with a more appropriate approach
+            factors = normed(gdp_n)
             return pd.DataFrame(
                 factors.values * l.values[:, np.newaxis],
                 index=l.index,

From 1430c66b4823e61ce87a1753c88b25a5cd570b28 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 5 Jan 2024 01:36:01 +0300
Subject: [PATCH 25/67] Account for the carriers

---
 scripts/simplify_network.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 963341b21..1bafef959 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -748,7 +748,9 @@ def drop_isolated_nodes(n, threshold):
 
 def find_isolated_sub_networks(buses_df, n_buses_thresh):
     buses_df["bus_id"] = buses_df.index
-    grouped_by_subnw = buses_df.groupby("sub_network").count()
+    grouped_by_subnw = (
+        buses_df.loc[n.buses.carrier == "AC"].groupby("sub_network").count()
+    )
     i_isol_subnetw = grouped_by_subnw[grouped_by_subnw.bus_id <= n_buses_thresh].index
     i_islands = buses_df.loc[buses_df.sub_network.isin(i_isol_subnetw)]
     return i_islands

From 1031926e6a737ad9b499720f968d4a27e882ac6b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 5 Jan 2024 01:37:18 +0300
Subject: [PATCH 26/67] Hardcode CRS

---
 scripts/simplify_network.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 1bafef959..1a354e9af 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -789,7 +789,8 @@ def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
     generators_mean_origin = n.generators.p_nom.mean()
     load_mean_origin = n.loads_t.p_set.mean().mean()
 
-    network_crs = n.meta["crs"]["geo_crs"]
+    # TODO Replace by reading from the config
+    network_crs = "EPSG:4326"
 
     n.determine_network_topology()
 

From 085ee0cca0424f61707a25efb0a3812dcf0c54a2 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 5 Jan 2024 01:40:15 +0300
Subject: [PATCH 27/67] Revert "Exclude population raster from demand
 distribution"

This reverts commit 1f7cc78f7e3f03bcf8aa82f8f3d01c48d757966d.
---
 scripts/build_demand_profiles.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/scripts/build_demand_profiles.py b/scripts/build_demand_profiles.py
index 3e3c44104..6a76c2275 100644
--- a/scripts/build_demand_profiles.py
+++ b/scripts/build_demand_profiles.py
@@ -170,19 +170,15 @@ def upsample(cntry, group):
             gdp_n = pd.Series(
                 transfer.dot(shapes_cntry["gdp"].fillna(1.0).values), index=group.index
             )
-            # TODO replace quick-and-dirty fix with a more appropriate approach
-            # pop_n = pd.Series(
-            #     transfer.dot(shapes_cntry["pop"].fillna(1.0).values), index=group.index
-            # )
+            pop_n = pd.Series(
+                transfer.dot(shapes_cntry["pop"].fillna(1.0).values), index=group.index
+            )
 
             # relative factors 0.6 and 0.4 have been determined from a linear
             # regression on the country to EU continent load data
             # (refer to vresutils.load._upsampling_weights)
             # TODO: require adjustment for Africa
-            # factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n))
-
-            # TODO replace quick-and-dirty fix with a more appropriate approach
-            factors = normed(gdp_n)
+            factors = normed(0.6 * normed(gdp_n) + 0.4 * normed(pop_n))
             return pd.DataFrame(
                 factors.values * l.values[:, np.newaxis],
                 index=l.index,

From 3c9b651ce2afb81302d0e97dbc4c37a146aec843 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:01:15 +0300
Subject: [PATCH 28/67] Fix a typo in a comment

---
 config.default.yaml  | 2 +-
 config.tutorial.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index f062bbb75..54e519c51 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -64,7 +64,7 @@ cluster_options:
     remove_stubs: true
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
-    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if bus mean power is below the specified threshold
+    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index 47092ff55..153b45e7f 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -77,7 +77,7 @@ cluster_options:
     remove_stubs: true
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
-    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if bus mean power is below the specified threshold
+    p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

From 4e45ecb725d1d351f2d324871ed1fdbc2aa8acc3 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:02:00 +0300
Subject: [PATCH 29/67] Add a fetch threshold to the config

---
 config.default.yaml  | 1 +
 config.tutorial.yaml | 1 +
 2 files changed, 2 insertions(+)

diff --git a/config.default.yaml b/config.default.yaml
index 54e519c51..1459405e0 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,6 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
+    p_threshold_fetch_isolated: 500 # [MW] isolated buses and networks are being merged into a backbone network if their mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index 153b45e7f..104724e4b 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,6 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
+    p_threshold_fetch_isolated: 500 # [MW] isolated buses and networks are being merged into a backbone network if their mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

From c103064846f95cafec3ae28ee6b657c0d3af5796 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:08:44 +0300
Subject: [PATCH 30/67] Improve docstrings

---
 scripts/simplify_network.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 1a354e9af..9a9660f65 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -769,9 +769,9 @@ def transform_to_gdf(buses_df, i_buses, network_crs):
 
 def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
     """
-    Find isolated nodes in the network and merge those of them which have load
-    value below than a specified threshold into a single isolated node which
-    represents all the remote generation.
+    Find isolated AC nodes and sub-networks in the network and merge those of
+    them which have load value and a number of buses below than the specified
+    thresholds into a backbone network.
 
     Parameters
     ----------
@@ -779,6 +779,8 @@ def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
         Original network
     threshold : float
         Load power used as a threshold to merge isolated nodes
+    aggregation_strategies: dictionary
+        Functions to be applied to calculate parameters of the aggregated grid
 
     Returns
     -------
@@ -891,6 +893,8 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         Original network
     threshold : float
         Load power used as a threshold to merge isolated nodes
+    aggregation_strategies: dictionary
+        Functions to be applied to calculate parameters of the aggregated grid
 
     Returns
     -------

From 16ff175fadff8f348c90af45fc1c4aadff30f890 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:11:10 +0300
Subject: [PATCH 31/67] Fix a fetch threshold

---
 scripts/simplify_network.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9a9660f65..57a1c27d3 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -767,7 +767,7 @@ def transform_to_gdf(buses_df, i_buses, network_crs):
     return gdf_buses
 
 
-def merge_into_network(n, isol_threshold_n=5, aggregation_strategies=dict()):
+def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict()):
     """
     Find isolated AC nodes and sub-networks in the network and merge those of
     them which have load value and a number of buses below than the specified
@@ -1116,6 +1116,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         0.0, cluster_config.get("p_threshold_drop_isolated", 0.0)
     )
     p_threshold_merge_isolated = cluster_config.get("p_threshold_merge_isolated", False)
+    p_threshold_fetch_isolated = cluster_config.get("p_threshold_fetch_isolated", False)
 
     n = drop_isolated_nodes(n, threshold=p_threshold_drop_isolated)
     if p_threshold_merge_isolated:
@@ -1126,12 +1127,13 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         )
         busmaps.append(merged_nodes_map)
 
-    # TODO Add a configuration option
-    n, fetched_nodes_map = merge_into_network(
-        n,
-        aggregation_strategies=aggregation_strategies,
-    )
-    busmaps.append(fetched_nodes_map)
+    if p_threshold_fetch_isolated:
+        n, fetched_nodes_map = merge_into_network(
+            n,
+            threshold=p_threshold_fetch_isolated,
+            aggregation_strategies=aggregation_strategies,
+        )
+        busmaps.append(fetched_nodes_map)
 
     n.meta = dict(snakemake.config, **dict(wildcards=dict(snakemake.wildcards)))
     n.export_to_netcdf(snakemake.output.network)

From bd7166394457fe8972bab8cf3df01ca60bcb2d4c Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:12:02 +0300
Subject: [PATCH 32/67] Fix CRS input

---
 scripts/simplify_network.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 57a1c27d3..ad7fd6fed 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -791,8 +791,7 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
     generators_mean_origin = n.generators.p_nom.mean()
     load_mean_origin = n.loads_t.p_set.mean().mean()
 
-    # TODO Replace by reading from the config
-    network_crs = "EPSG:4326"
+    network_crs = snakemake.params.geo_crs
 
     n.determine_network_topology()
 

From fef6935a75b6eb3ffc68dfa4d93880ffea6a911a Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:13:15 +0300
Subject: [PATCH 33/67] Switch-on filtering by load

---
 scripts/simplify_network.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index ad7fd6fed..3cad73a46 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -807,11 +807,11 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
         ].index
 
     # TODO filtering may be applied to decide if the isolated buses should be fetched
-    # # isolated buses with load below than a specified threshold should be merged
-    # i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
-    # i_suffic_load = i_load_islands[
-    #     n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
-    # ]
+    # isolated buses with load below than a specified threshold should be merged
+    i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
+    i_islands_fetch = i_load_islands[
+        n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
+    ]
 
     # return the original network if no isolated nodes are detected
     if len(i_islands) == 0:

From db139ad792903ba3b2bac277d6e810c4d316f38b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Wed, 17 Jan 2024 01:13:28 +0300
Subject: [PATCH 34/67] Improve naming

---
 scripts/simplify_network.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 3cad73a46..ac9d651e3 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -795,9 +795,9 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
 
     n.determine_network_topology()
 
-    if isol_threshold_n > 1:
+    if threshold_n > 1:
         i_islands = find_isolated_sub_networks(
-            buses_df=n.buses, n_buses_thresh=isol_threshold_n
+            buses_df=n.buses, n_buses_thresh=threshold_n
         ).index
     else:
         # duplicated sub-networks mean that there is at least one interconnection between buses
@@ -914,19 +914,19 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
 
     # isolated buses with load below than a specified threshold should be merged
     i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
-    i_suffic_load = i_load_islands[
+    i_islands_fetch = i_load_islands[
         n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
     ]
 
     # all the nodes to be merged should be mapped into a single node
     map_isolated_node_by_country = (
         n.buses.assign(bus_id=n.buses.index)
-        .loc[i_suffic_load]
+        .loc[i_islands_fetch]
         .groupby("country")["bus_id"]
         .first()
         .to_dict()
     )
-    isolated_buses_mapping = n.buses.loc[i_suffic_load, "country"].replace(
+    isolated_buses_mapping = n.buses.loc[i_islands_fetch, "country"].replace(
         map_isolated_node_by_country
     )
     busmap = (
@@ -960,7 +960,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
     generators_mean_final = n.generators.p_nom.mean()
 
     logger.info(
-        f"Merged {len(i_suffic_load)} buses. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
+        f"Merged {len(i_islands_fetch)} buses. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
     )
 
     return clustering.network, busmap

From 2bf970413012ed44e07b1809d5162328a70ad3ff Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:46:05 +0300
Subject: [PATCH 35/67] Revise finding isolated networks to respect national
 partition

---
 scripts/simplify_network.py | 51 ++++++++++++++++++++++++++++++++++---
 1 file changed, 47 insertions(+), 4 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index ac9d651e3..586114f02 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -748,11 +748,54 @@ def drop_isolated_nodes(n, threshold):
 
 def find_isolated_sub_networks(buses_df, n_buses_thresh):
     buses_df["bus_id"] = buses_df.index
-    grouped_by_subnw = (
-        buses_df.loc[n.buses.carrier == "AC"].groupby("sub_network").count()
+
+    # don't merge dc networks and sub-networks spanning through multiple countries
+    subnetw_ac_df = (
+        buses_df.loc[n.buses.carrier == "AC"]
+        .groupby(["sub_network", "country"])
+        .count()
+        # .apply(lambda x: x)
+        # .to_frame()
     )
-    i_isol_subnetw = grouped_by_subnw[grouped_by_subnw.bus_id <= n_buses_thresh].index
-    i_islands = buses_df.loc[buses_df.sub_network.isin(i_isol_subnetw)]
+    # subnetw_ac_df = subnetw_ac_df.to_frame()
+    multicountry_subnetworks = subnetw_ac_df[
+        subnetw_ac_df.index.droplevel("country").duplicated()
+    ].index.get_level_values("sub_network")
+    subnetw_ac_monocountry_df = subnetw_ac_df.loc[
+        subnetw_ac_df.index.get_level_values("sub_network").difference(
+            multicountry_subnetworks
+        )
+    ]
+
+    island_sbntw = []
+    for cnt in buses_df.country.unique():
+        # load may attached to multi-country networks only
+        country_load = (
+            n.loads_t.p_set[
+                n.loads_t.p_set.columns.intersection(
+                    n.buses.index[n.buses.country == cnt]
+                )
+            ]
+            .sum()
+            .sum()
+        )
+
+        sbntw = subnetw_ac_monocountry_df.query(
+            "country == @cnt"
+        ).index.get_level_values("sub_network")
+
+        i_island_ntw = [
+            s
+            for s in sbntw
+            if (
+                n.loads_t.p_set[n.buses[n.buses.sub_network == s].bus_id].sum().sum()
+                < threshold * country_load
+            )
+        ]
+        island_sbntw.extend(i_island_ntw)
+
+    i_islands = n.buses[n.buses.sub_network.isin(island_sbntw)].bus_id
+
     return i_islands
 
 

From 453583c27409281c313617c690275d73d65cb70e Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:46:43 +0300
Subject: [PATCH 36/67] Add a country column to the buses dataframe

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 586114f02..555df0d53 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -801,7 +801,7 @@ def find_isolated_sub_networks(buses_df, n_buses_thresh):
 
 def transform_to_gdf(buses_df, i_buses, network_crs):
     buses_df["bus_id"] = buses_df.index
-    points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y"]]
+    points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y", "country"]]
     gdf_buses = gpd.GeoDataFrame(
         points_buses,
         geometry=gpd.points_from_xy(points_buses.x, points_buses.y),

From 7ad085369689ad9f3389e90c411476ea5dcb0b2c Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:52:57 +0300
Subject: [PATCH 37/67] Move filtering to the find-isolated function

---
 scripts/simplify_network.py | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 555df0d53..bea6c36dc 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -838,23 +838,7 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
 
     n.determine_network_topology()
 
-    if threshold_n > 1:
-        i_islands = find_isolated_sub_networks(
-            buses_df=n.buses, n_buses_thresh=threshold_n
-        ).index
-    else:
-        # duplicated sub-networks mean that there is at least one interconnection between buses
-        i_islands = n.buses[
-            (~n.buses.duplicated(subset=["sub_network"], keep=False))
-            & (n.buses.carrier == "AC")
-        ].index
-
-    # TODO filtering may be applied to decide if the isolated buses should be fetched
-    # isolated buses with load below than a specified threshold should be merged
-    i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
-    i_islands_fetch = i_load_islands[
-        n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
-    ]
+    i_islands = find_isolated_sub_networks(buses_df=n.buses, threshold=threshold)
 
     # return the original network if no isolated nodes are detected
     if len(i_islands) == 0:

From 9efcdc61b932ae20b35b977c82b345d15a98d68e Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:55:07 +0300
Subject: [PATCH 38/67] Re-factor filtering of backbone buses

---
 scripts/simplify_network.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index bea6c36dc..86d3533c5 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -844,13 +844,18 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
     if len(i_islands) == 0:
         return n, n.buses.index.to_series()
 
-    i_connected = n.buses.loc[n.buses.carrier == "AC"].index.difference(i_islands)
-    gdf_backbone_buses = transform_to_gdf(
-        buses_df=n.buses, i_buses=i_connected, network_crs=network_crs
-    )
     gdf_islands = transform_to_gdf(
         buses_df=n.buses, i_buses=i_islands, network_crs=network_crs
     )
+    # backbone buses should be from same countries as isolated ones
+    i_backbone = (
+        n.buses.query("country in @gdf_islands.country.unique()")
+        .query("carrier == 'AC'")
+        .index.difference(i_islands)
+    )
+    gdf_backbone_buses = transform_to_gdf(
+        buses_df=n.buses, i_buses=i_backbone, network_crs=network_crs
+    )
 
     # map each isolated bus into the closest non-isolated
     gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_backbone_buses, how="left")

From 92f9d698430a60f53b4d7cb489c24940ed05d4ef Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:55:36 +0300
Subject: [PATCH 39/67] Account for geography when finding the closest buses

---
 scripts/simplify_network.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 86d3533c5..0d35c9fa1 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -857,8 +857,10 @@ def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict(
         buses_df=n.buses, i_buses=i_backbone, network_crs=network_crs
     )
 
-    # map each isolated bus into the closest non-isolated
-    gdf_map = gpd.sjoin_nearest(gdf_islands, gdf_backbone_buses, how="left")
+    islands_bcountry = {k: d for k, d in gdf_islands.groupby("country")}
+    gdf_map = gdf_backbone_buses.groupby("country").apply(
+        lambda d: gpd.sjoin_nearest(islands_bcountry[d["country"].values[0]], d)
+    )
     nearest_bus_df = n.buses.loc[n.buses.bus_id.isin(gdf_map.bus_id_right)]
 
     i_lines_islands = n.lines.loc[n.lines.bus1.isin(i_islands)].index

From 222ab30cf6d213e6f7339f0ef0e886c4eda3f22d Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Thu, 8 Feb 2024 17:56:20 +0300
Subject: [PATCH 40/67] Revise threshold configuration parameters

---
 config.default.yaml         |  2 +-
 config.tutorial.yaml        |  2 +-
 scripts/simplify_network.py | 12 +++++++-----
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 1459405e0..dfe57399a 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,7 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 500 # [MW] isolated buses and networks are being merged into a backbone network if their mean power is below the specified threshold
+    share_threshold_fetch_isolated: 0.3 # [-] share of the national load to fetch isolated buses and networks into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index 104724e4b..ed961100d 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,7 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 500 # [MW] isolated buses and networks are being merged into a backbone network if their mean power is below the specified threshold
+    share_threshold_fetch_isolated: 0.3 # [-] share of the national load to fetch isolated buses and networks into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 0d35c9fa1..bad75516a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,7 +746,7 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
-def find_isolated_sub_networks(buses_df, n_buses_thresh):
+def find_isolated_sub_networks(buses_df, threshold):
     buses_df["bus_id"] = buses_df.index
 
     # don't merge dc networks and sub-networks spanning through multiple countries
@@ -810,7 +810,7 @@ def transform_to_gdf(buses_df, i_buses, network_crs):
     return gdf_buses
 
 
-def merge_into_network(n, threshold, threshold_n=5, aggregation_strategies=dict()):
+def merge_into_network(n, threshold, aggregation_strategies=dict()):
     """
     Find isolated AC nodes and sub-networks in the network and merge those of
     them which have load value and a number of buses below than the specified
@@ -1149,7 +1149,9 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         0.0, cluster_config.get("p_threshold_drop_isolated", 0.0)
     )
     p_threshold_merge_isolated = cluster_config.get("p_threshold_merge_isolated", False)
-    p_threshold_fetch_isolated = cluster_config.get("p_threshold_fetch_isolated", False)
+    share_threshold_fetch_isolated = cluster_config.get(
+        "share_threshold_fetch_isolated", False
+    )
 
     n = drop_isolated_nodes(n, threshold=p_threshold_drop_isolated)
     if p_threshold_merge_isolated:
@@ -1160,10 +1162,10 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         )
         busmaps.append(merged_nodes_map)
 
-    if p_threshold_fetch_isolated:
+    if share_threshold_fetch_isolated:
         n, fetched_nodes_map = merge_into_network(
             n,
-            threshold=p_threshold_fetch_isolated,
+            threshold=share_threshold_fetch_isolated,
             aggregation_strategies=aggregation_strategies,
         )
         busmaps.append(fetched_nodes_map)

From d1ef4417f9f28e5b1f01adf76702a9d4cd0931b6 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 9 Feb 2024 00:18:06 +0300
Subject: [PATCH 41/67] Code clean-up

---
 scripts/simplify_network.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index bad75516a..90cb07c4a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -749,18 +749,16 @@ def drop_isolated_nodes(n, threshold):
 def find_isolated_sub_networks(buses_df, threshold):
     buses_df["bus_id"] = buses_df.index
 
-    # don't merge dc networks and sub-networks spanning through multiple countries
     subnetw_ac_df = (
         buses_df.loc[n.buses.carrier == "AC"]
         .groupby(["sub_network", "country"])
         .count()
-        # .apply(lambda x: x)
-        # .to_frame()
     )
-    # subnetw_ac_df = subnetw_ac_df.to_frame()
+    # don't merge dc networks and sub-networks spanning through multiple countries
     multicountry_subnetworks = subnetw_ac_df[
         subnetw_ac_df.index.droplevel("country").duplicated()
     ].index.get_level_values("sub_network")
+
     subnetw_ac_monocountry_df = subnetw_ac_df.loc[
         subnetw_ac_df.index.get_level_values("sub_network").difference(
             multicountry_subnetworks
@@ -847,7 +845,6 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     gdf_islands = transform_to_gdf(
         buses_df=n.buses, i_buses=i_islands, network_crs=network_crs
     )
-    # backbone buses should be from same countries as isolated ones
     i_backbone = (
         n.buses.query("country in @gdf_islands.country.unique()")
         .query("carrier == 'AC'")
@@ -877,6 +874,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     isolated_buses_mapping = n.buses.loc[i_islands, "country"].replace(
         map_isolated_node_by_country
     )
+
     busmap = (
         n.buses.index.to_series()
         .replace(isolated_buses_mapping)

From d4a212b9c2bd66e61f4ee7daf082365b5e724b75 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 9 Feb 2024 00:18:55 +0300
Subject: [PATCH 42/67] Avoid global variables

---
 scripts/simplify_network.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 90cb07c4a..23536a7ae 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,7 +746,8 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
-def find_isolated_sub_networks(buses_df, threshold):
+def find_isolated_sub_networks(n, threshold):
+    buses_df = n.buses
     buses_df["bus_id"] = buses_df.index
 
     subnetw_ac_df = (
@@ -836,7 +837,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
 
     n.determine_network_topology()
 
-    i_islands = find_isolated_sub_networks(buses_df=n.buses, threshold=threshold)
+    i_islands = find_isolated_sub_networks(n=n, threshold=threshold)
 
     # return the original network if no isolated nodes are detected
     if len(i_islands) == 0:

From a0e9c870249548aa49388744ab62e3572882ee6e Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 9 Feb 2024 00:19:24 +0300
Subject: [PATCH 43/67] Fix mapping

---
 scripts/simplify_network.py | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 23536a7ae..f7fa2cb0a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -864,16 +864,8 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     i_lines_islands = n.lines.loc[n.lines.bus1.isin(i_islands)].index
     n.mremove("Line", i_lines_islands)
 
-    # each isolated node should be mapped into the closes non-isolated node
-    map_isolated_node_by_country = (
-        n.buses.assign(bus_id=n.buses.index)
-        .loc[nearest_bus_df.index]
-        .groupby("country")["bus_id"]
-        .first()
-        .to_dict()
-    )
-    isolated_buses_mapping = n.buses.loc[i_islands, "country"].replace(
-        map_isolated_node_by_country
+    isolated_buses_mapping = (
+        gdf_map[["bus_id_right"]].droplevel("country").to_dict()["bus_id_right"]
     )
 
     busmap = (

From 71c398dff5142b1d6fb69471e8f5532ee561e17b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Fri, 9 Feb 2024 00:20:00 +0300
Subject: [PATCH 44/67] Avoid merging into multi-country networks

---
 scripts/simplify_network.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index f7fa2cb0a..e2e6d463a 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -846,8 +846,24 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     gdf_islands = transform_to_gdf(
         buses_df=n.buses, i_buses=i_islands, network_crs=network_crs
     )
+
+    multicountry_subnetworks = (
+        n.buses.groupby(["sub_network", "country"])
+        .count()
+        .index.droplevel("country")
+        .duplicated()
+    )
+
+    # don't into sub-networks spanning through multiple countries
+    subnetw_by_countries_df = n.buses.groupby(["sub_network", "country"]).count()
+    multicountry_subnetworks = subnetw_by_countries_df[
+        subnetw_by_countries_df.index.droplevel("country").duplicated()
+    ].index.get_level_values("sub_network")
+
+    # backbone buses should be from the same countries as isolated ones
     i_backbone = (
         n.buses.query("country in @gdf_islands.country.unique()")
+        .query("sub_network not in @multicountry_subnetworks")
         .query("carrier == 'AC'")
         .index.difference(i_islands)
     )

From b167bf03869452d39e7ea746da3df6e38d2a3f92 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 10 Feb 2024 00:59:21 +0300
Subject: [PATCH 45/67] Use an absolute power threshold

---
 config.default.yaml         |  2 +-
 config.tutorial.yaml        |  2 +-
 scripts/simplify_network.py | 33 ++++++++++++++++-----------------
 3 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index dfe57399a..a5d89fe4b 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,7 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    share_threshold_fetch_isolated: 0.3 # [-] share of the national load to fetch isolated buses and networks into a backbone network
+    p_threshold_fetch_isolated: 20 # [-] isolated buses are being merged into a backbone network if a network mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index ed961100d..0a8b062dd 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,7 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    share_threshold_fetch_isolated: 0.3 # [-] share of the national load to fetch isolated buses and networks into a backbone network
+    p_threshold_fetch_isolated: 20 # [-] isolated buses are being merged into a backbone network if a network mean power is below the specified threshold
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index e2e6d463a..4a6150532 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -768,16 +768,17 @@ def find_isolated_sub_networks(n, threshold):
 
     island_sbntw = []
     for cnt in buses_df.country.unique():
-        # load may attached to multi-country networks only
-        country_load = (
-            n.loads_t.p_set[
-                n.loads_t.p_set.columns.intersection(
-                    n.buses.index[n.buses.country == cnt]
-                )
-            ]
-            .sum()
-            .sum()
-        )
+        ## TODO May be worth to investigate a threshold for the load share
+        ## load may attached to multi-country networks only
+        # country_load = (
+        #    n.loads_t.p_set[
+        #        n.loads_t.p_set.columns.intersection(
+        #            n.buses.index[n.buses.country == cnt]
+        #        )
+        #    ]
+        #    .sum()
+        #    .sum()
+        # )
 
         sbntw = subnetw_ac_monocountry_df.query(
             "country == @cnt"
@@ -787,8 +788,8 @@ def find_isolated_sub_networks(n, threshold):
             s
             for s in sbntw
             if (
-                n.loads_t.p_set[n.buses[n.buses.sub_network == s].bus_id].sum().sum()
-                < threshold * country_load
+                n.loads_t.p_set[n.buses[n.buses.sub_network == s].bus_id].mean().mean()
+                < threshold
             )
         ]
         island_sbntw.extend(i_island_ntw)
@@ -1156,9 +1157,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         0.0, cluster_config.get("p_threshold_drop_isolated", 0.0)
     )
     p_threshold_merge_isolated = cluster_config.get("p_threshold_merge_isolated", False)
-    share_threshold_fetch_isolated = cluster_config.get(
-        "share_threshold_fetch_isolated", False
-    )
+    p_threshold_fetch_isolated = cluster_config.get("p_threshold_fetch_isolated", False)
 
     n = drop_isolated_nodes(n, threshold=p_threshold_drop_isolated)
     if p_threshold_merge_isolated:
@@ -1169,10 +1168,10 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         )
         busmaps.append(merged_nodes_map)
 
-    if share_threshold_fetch_isolated:
+    if p_threshold_fetch_isolated:
         n, fetched_nodes_map = merge_into_network(
             n,
-            threshold=share_threshold_fetch_isolated,
+            threshold=p_threshold_fetch_isolated,
             aggregation_strategies=aggregation_strategies,
         )
         busmaps.append(fetched_nodes_map)

From 3656788cdbb540d20299d309b84a8ffe2e89dea4 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 18 Mar 2024 22:09:06 +0300
Subject: [PATCH 46/67] Remove a redundant column

---
 scripts/simplify_network.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index dadf11813..15e3ffea7 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -748,7 +748,6 @@ def drop_isolated_nodes(n, threshold):
 
 def find_isolated_sub_networks(n, threshold):
     buses_df = n.buses
-    buses_df["bus_id"] = buses_df.index
 
     subnetw_ac_df = (
         buses_df.loc[n.buses.carrier == "AC"]
@@ -788,13 +787,13 @@ def find_isolated_sub_networks(n, threshold):
             s
             for s in sbntw
             if (
-                n.loads_t.p_set[n.buses[n.buses.sub_network == s].bus_id].mean().mean()
+                n.loads_t.p_set[n.buses[n.buses.sub_network == s].index].mean().mean()
                 < threshold
             )
         ]
         island_sbntw.extend(i_island_ntw)
 
-    i_islands = n.buses[n.buses.sub_network.isin(island_sbntw)].bus_id
+    i_islands = n.buses[n.buses.sub_network.isin(island_sbntw)].index
 
     return i_islands
 
@@ -876,7 +875,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     gdf_map = gdf_backbone_buses.groupby("country").apply(
         lambda d: gpd.sjoin_nearest(islands_bcountry[d["country"].values[0]], d)
     )
-    nearest_bus_df = n.buses.loc[n.buses.bus_id.isin(gdf_map.bus_id_right)]
+    nearest_bus_df = n.buses.loc[n.buses.index.isin(gdf_map.bus_id_right)]
 
     i_lines_islands = n.lines.loc[n.lines.bus1.isin(i_islands)].index
     n.mremove("Line", i_lines_islands)

From fd6354ee6271e3ca56148fe15ad5972ba5ea1076 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 18 Mar 2024 22:22:31 +0300
Subject: [PATCH 47/67] Improve comments

---
 scripts/simplify_network.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 15e3ffea7..f7c16e4b7 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -749,6 +749,7 @@ def drop_isolated_nodes(n, threshold):
 def find_isolated_sub_networks(n, threshold):
     buses_df = n.buses
 
+    # handling isolated networks make sense primarily for AC networks
     subnetw_ac_df = (
         buses_df.loc[n.buses.carrier == "AC"]
         .groupby(["sub_network", "country"])
@@ -758,13 +759,14 @@ def find_isolated_sub_networks(n, threshold):
     multicountry_subnetworks = subnetw_ac_df[
         subnetw_ac_df.index.droplevel("country").duplicated()
     ].index.get_level_values("sub_network")
-
+    # process further only networks which entirely belongs to a single country
     subnetw_ac_monocountry_df = subnetw_ac_df.loc[
         subnetw_ac_df.index.get_level_values("sub_network").difference(
             multicountry_subnetworks
         )
     ]
 
+    # all relevant isolated sub-networks should be identified across all the countries
     island_sbntw = []
     for cnt in buses_df.country.unique():
         ## TODO May be worth to investigate a threshold for the load share
@@ -783,6 +785,7 @@ def find_isolated_sub_networks(n, threshold):
             "country == @cnt"
         ).index.get_level_values("sub_network")
 
+        # power threshold should be accounted to identify the relevant networks
         i_island_ntw = [
             s
             for s in sbntw
@@ -854,7 +857,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
         .duplicated()
     )
 
-    # don't into sub-networks spanning through multiple countries
+    # don't go into sub-networks spanning through multiple countries
     subnetw_by_countries_df = n.buses.groupby(["sub_network", "country"]).count()
     multicountry_subnetworks = subnetw_by_countries_df[
         subnetw_by_countries_df.index.droplevel("country").duplicated()

From f8cb01f87161ed00e15c495da76430efda106c5c Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 20:04:17 +0300
Subject: [PATCH 48/67] Add load data to the geo-transform function

---
 scripts/simplify_network.py | 29 ++++++++++++++++++++++++-----
 1 file changed, 24 insertions(+), 5 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index f7c16e4b7..9a4d8ca65 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -801,12 +801,31 @@ def find_isolated_sub_networks(n, threshold):
     return i_islands
 
 
-def transform_to_gdf(buses_df, i_buses, network_crs):
-    buses_df["bus_id"] = buses_df.index
-    points_buses = buses_df.loc[i_buses, ["bus_id", "x", "y", "country"]]
+def transform_to_gdf(n, network_crs):
+    buses_df = n.buses.copy()
+
+    # load data are crucial to deal with sub-networks
+    buses_df = buses_df.join(n.loads_t.p_set.sum().T.rename("load"))
+    buses_df["load_in_subnetw"] = buses_df.groupby(["country", "sub_network"])[
+        "load"
+    ].transform("sum")
+    buses_df["load_in_country"] = buses_df.groupby(["country"])["load"].transform("sum")
+    buses_df["sbntw_share_of_country_load"] = buses_df.apply(
+        lambda row: (
+            row.load_in_subnetw / row.load_in_country if row.load_in_country > 0 else 0
+        ),
+        axis=1,
+    )
+    buses_df["is_backbone_sbntw"] = (
+        buses_df.groupby(["country"], as_index=True)[
+            "sbntw_share_of_country_load"
+        ].transform("max")
+        == buses_df["sbntw_share_of_country_load"]
+    )
+
     gdf_buses = gpd.GeoDataFrame(
-        points_buses,
-        geometry=gpd.points_from_xy(points_buses.x, points_buses.y),
+        buses_df,
+        geometry=gpd.points_from_xy(buses_df.x, buses_df.y),
         crs=network_crs,
     )
     return gdf_buses

From b1697a0c1d45c9fc153f82854e3c0f0c4a0d6e7b Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 20:52:05 +0300
Subject: [PATCH 49/67] Update definitions of the geo-dataframes

---
 scripts/simplify_network.py | 46 +++++++++++++++----------------------
 1 file changed, 18 insertions(+), 28 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 9a4d8ca65..03f5bc7ce 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -859,38 +859,28 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
 
     n.determine_network_topology()
 
-    i_islands = find_isolated_sub_networks(n=n, threshold=threshold)
-
-    # return the original network if no isolated nodes are detected
-    if len(i_islands) == 0:
-        return n, n.buses.index.to_series()
-
-    gdf_islands = transform_to_gdf(
-        buses_df=n.buses, i_buses=i_islands, network_crs=network_crs
+    n_buses_gdf = transform_to_gdf(n, network_crs=network_crs)
+    # do not merge sub-networks spanned through a number of countries
+    n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
+        n_buses_gdf.groupby(["sub_network"])["country"].apply(
+            lambda x: len(pd.unique(x)) > 1
+        )
     )
 
-    multicountry_subnetworks = (
-        n.buses.groupby(["sub_network", "country"])
-        .count()
-        .index.droplevel("country")
-        .duplicated()
+    gdf_islands = (
+        n_buses_gdf.query("~is_multicnt_subntw")
+        .query("carrier=='AC'")
+        .query("sbntw_share_of_country_load < @threshold")
     )
 
-    # don't go into sub-networks spanning through multiple countries
-    subnetw_by_countries_df = n.buses.groupby(["sub_network", "country"]).count()
-    multicountry_subnetworks = subnetw_by_countries_df[
-        subnetw_by_countries_df.index.droplevel("country").duplicated()
-    ].index.get_level_values("sub_network")
+    ## return the original network if no isolated nodes are detected
+    # if len(gdf_islands) == 0:
+    #    return n, n.buses.index.to_series()
 
-    # backbone buses should be from the same countries as isolated ones
-    i_backbone = (
-        n.buses.query("country in @gdf_islands.country.unique()")
-        .query("sub_network not in @multicountry_subnetworks")
-        .query("carrier == 'AC'")
-        .index.difference(i_islands)
-    )
-    gdf_backbone_buses = transform_to_gdf(
-        buses_df=n.buses, i_buses=i_backbone, network_crs=network_crs
+    gdf_backbone_buses = (
+        n_buses_gdf.query("is_backbone_sbntw")
+        .query("carrier=='AC'")
+        .query("load_in_subnetw>0")
     )
 
     islands_bcountry = {k: d for k, d in gdf_islands.groupby("country")}
@@ -899,7 +889,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     )
     nearest_bus_df = n.buses.loc[n.buses.index.isin(gdf_map.bus_id_right)]
 
-    i_lines_islands = n.lines.loc[n.lines.bus1.isin(i_islands)].index
+    i_lines_islands = n.lines.loc[n.lines.bus1.isin(gdf_islands.index)].index
     n.mremove("Line", i_lines_islands)
 
     isolated_buses_mapping = (

From fc976493bdb596cb80c0ec1eba03fb37fda165f2 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 20:52:22 +0300
Subject: [PATCH 50/67] Update configuration files

---
 config.default.yaml  | 2 +-
 config.tutorial.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 8a43121c7..96dfeefdb 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,7 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 20 # [-] isolated buses are being merged into a backbone network if a network mean power is below the specified threshold
+    p_threshold_fetch_isolated: 0.2 # [-] isolated buses are being merged into a backbone network if a network load is below the specified share of the overall national load
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index d8d12f2e1..758c2ea03 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,7 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 20 # [-] isolated buses are being merged into a backbone network if a network mean power is below the specified threshold
+    p_threshold_fetch_isolated: 0.2 # [-] isolated buses are being merged into a backbone network if a network load is below the specified share of the overall national load
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

From 60ae7cd6617c5a440479610fff5cc3ea5def6bf5 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 20:59:25 +0300
Subject: [PATCH 51/67] Remove an outdated function

---
 scripts/simplify_network.py | 55 -------------------------------------
 1 file changed, 55 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 03f5bc7ce..3f0a6fb90 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -746,61 +746,6 @@ def drop_isolated_nodes(n, threshold):
     return n
 
 
-def find_isolated_sub_networks(n, threshold):
-    buses_df = n.buses
-
-    # handling isolated networks make sense primarily for AC networks
-    subnetw_ac_df = (
-        buses_df.loc[n.buses.carrier == "AC"]
-        .groupby(["sub_network", "country"])
-        .count()
-    )
-    # don't merge dc networks and sub-networks spanning through multiple countries
-    multicountry_subnetworks = subnetw_ac_df[
-        subnetw_ac_df.index.droplevel("country").duplicated()
-    ].index.get_level_values("sub_network")
-    # process further only networks which entirely belongs to a single country
-    subnetw_ac_monocountry_df = subnetw_ac_df.loc[
-        subnetw_ac_df.index.get_level_values("sub_network").difference(
-            multicountry_subnetworks
-        )
-    ]
-
-    # all relevant isolated sub-networks should be identified across all the countries
-    island_sbntw = []
-    for cnt in buses_df.country.unique():
-        ## TODO May be worth to investigate a threshold for the load share
-        ## load may attached to multi-country networks only
-        # country_load = (
-        #    n.loads_t.p_set[
-        #        n.loads_t.p_set.columns.intersection(
-        #            n.buses.index[n.buses.country == cnt]
-        #        )
-        #    ]
-        #    .sum()
-        #    .sum()
-        # )
-
-        sbntw = subnetw_ac_monocountry_df.query(
-            "country == @cnt"
-        ).index.get_level_values("sub_network")
-
-        # power threshold should be accounted to identify the relevant networks
-        i_island_ntw = [
-            s
-            for s in sbntw
-            if (
-                n.loads_t.p_set[n.buses[n.buses.sub_network == s].index].mean().mean()
-                < threshold
-            )
-        ]
-        island_sbntw.extend(i_island_ntw)
-
-    i_islands = n.buses[n.buses.sub_network.isin(island_sbntw)].index
-
-    return i_islands
-
-
 def transform_to_gdf(n, network_crs):
     buses_df = n.buses.copy()
 

From b7f50c5582ffe4a16e7af042078ee53768eaa99c Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:00:38 +0300
Subject: [PATCH 52/67] Fix formatting

---
 scripts/simplify_network.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 3f0a6fb90..6dfb3ec1e 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -818,9 +818,9 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
         .query("sbntw_share_of_country_load < @threshold")
     )
 
-    ## return the original network if no isolated nodes are detected
-    # if len(gdf_islands) == 0:
-    #    return n, n.buses.index.to_series()
+    # return the original network if no isolated nodes are detected
+    if len(gdf_islands) == 0:
+        return n, n.buses.index.to_series()
 
     gdf_backbone_buses = (
         n_buses_gdf.query("is_backbone_sbntw")

From f2a78408ddf4d4f6617506890d95bce719f8dfdd Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:04:41 +0300
Subject: [PATCH 53/67] Keep the backbone block together

---
 scripts/simplify_network.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 6dfb3ec1e..99332d069 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -805,23 +805,22 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     n.determine_network_topology()
 
     n_buses_gdf = transform_to_gdf(n, network_crs=network_crs)
-    # do not merge sub-networks spanned through a number of countries
-    n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
-        n_buses_gdf.groupby(["sub_network"])["country"].apply(
-            lambda x: len(pd.unique(x)) > 1
-        )
-    )
 
     gdf_islands = (
         n_buses_gdf.query("~is_multicnt_subntw")
         .query("carrier=='AC'")
         .query("sbntw_share_of_country_load < @threshold")
     )
-
     # return the original network if no isolated nodes are detected
     if len(gdf_islands) == 0:
         return n, n.buses.index.to_series()
 
+    # do not merge sub-networks spanned through a number of countries
+    n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
+        n_buses_gdf.groupby(["sub_network"])["country"].apply(
+            lambda x: len(pd.unique(x)) > 1
+        )
+    )
     gdf_backbone_buses = (
         n_buses_gdf.query("is_backbone_sbntw")
         .query("carrier=='AC'")

From f00c04b454a84354c412429abc0e5c6ac3b7ee02 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:04:53 +0300
Subject: [PATCH 54/67] Add a comment

---
 scripts/simplify_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 99332d069..bf076ad0b 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -827,6 +827,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
         .query("load_in_subnetw>0")
     )
 
+    # find the closest buses of the backbone networks for each isolated network and each country
     islands_bcountry = {k: d for k, d in gdf_islands.groupby("country")}
     gdf_map = gdf_backbone_buses.groupby("country").apply(
         lambda d: gpd.sjoin_nearest(islands_bcountry[d["country"].values[0]], d)

From 1702c3ae35b40faa1b66da5b4d42c5723e559a44 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:11:29 +0300
Subject: [PATCH 55/67] Fix code structure

---
 scripts/simplify_network.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index bf076ad0b..177dddb23 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -806,6 +806,12 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
 
     n_buses_gdf = transform_to_gdf(n, network_crs=network_crs)
 
+    # do not merge sub-networks spanned through a number of countries
+    n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
+        n_buses_gdf.groupby(["sub_network"])["country"].apply(
+            lambda x: len(pd.unique(x)) > 1
+        )
+    )
     gdf_islands = (
         n_buses_gdf.query("~is_multicnt_subntw")
         .query("carrier=='AC'")
@@ -815,12 +821,6 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     if len(gdf_islands) == 0:
         return n, n.buses.index.to_series()
 
-    # do not merge sub-networks spanned through a number of countries
-    n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
-        n_buses_gdf.groupby(["sub_network"])["country"].apply(
-            lambda x: len(pd.unique(x)) > 1
-        )
-    )
     gdf_backbone_buses = (
         n_buses_gdf.query("is_backbone_sbntw")
         .query("carrier=='AC'")

From 9b725cb55fe0ed4aaef572d8c6fd10ee1d2deef6 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:11:50 +0300
Subject: [PATCH 56/67] Fix filtering for floats

---
 scripts/simplify_network.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 177dddb23..a54b8ac15 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -765,7 +765,7 @@ def transform_to_gdf(n, network_crs):
         buses_df.groupby(["country"], as_index=True)[
             "sbntw_share_of_country_load"
         ].transform("max")
-        == buses_df["sbntw_share_of_country_load"]
+        <= buses_df["sbntw_share_of_country_load"]
     )
 
     gdf_buses = gpd.GeoDataFrame(

From f1ec53bcf0179bf06de111a5456b5b14f0cc5512 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 21:14:44 +0300
Subject: [PATCH 57/67] Revise naming

---
 scripts/simplify_network.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index a54b8ac15..b65dd7d91 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -912,19 +912,19 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
 
     # isolated buses with load below than a specified threshold should be merged
     i_load_islands = n.loads_t.p_set.columns.intersection(i_islands)
-    i_islands_fetch = i_load_islands[
+    i_islands_merge = i_load_islands[
         n.loads_t.p_set[i_load_islands].mean(axis=0) <= threshold
     ]
 
     # all the nodes to be merged should be mapped into a single node
     map_isolated_node_by_country = (
         n.buses.assign(bus_id=n.buses.index)
-        .loc[i_islands_fetch]
+        .loc[i_islands_merge]
         .groupby("country")["bus_id"]
         .first()
         .to_dict()
     )
-    isolated_buses_mapping = n.buses.loc[i_islands_fetch, "country"].replace(
+    isolated_buses_mapping = n.buses.loc[i_islands_merge, "country"].replace(
         map_isolated_node_by_country
     )
     busmap = (
@@ -958,7 +958,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
     generators_mean_final = n.generators.p_nom.mean()
 
     logger.info(
-        f"Merged {len(i_islands_fetch)} buses. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
+        f"Merged {len(i_islands_merge)} buses. Load attached to a single bus with discrepancies of {(100 * ((load_mean_final - load_mean_origin)/load_mean_origin)):2.1E}% and {(100 * ((generators_mean_final - generators_mean_origin)/generators_mean_origin)):2.1E}% for load and generation capacity, respectively"
     )
 
     return clustering.network, busmap

From 6cf958b8737b3bc2df6ae0c29a2d4badc06155e2 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 22:40:00 +0300
Subject: [PATCH 58/67] Keep id column

---
 scripts/simplify_network.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index b65dd7d91..e73647301 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -748,6 +748,7 @@ def drop_isolated_nodes(n, threshold):
 
 def transform_to_gdf(n, network_crs):
     buses_df = n.buses.copy()
+    buses_df["bus_id"] = buses_df.index
 
     # load data are crucial to deal with sub-networks
     buses_df = buses_df.join(n.loads_t.p_set.sum().T.rename("load"))

From 3953b705523d62c7e508c13425ca9be849150c08 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 22:40:38 +0300
Subject: [PATCH 59/67] Add a safety filtering

---
 scripts/simplify_network.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index e73647301..e88618761 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -830,8 +830,10 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
 
     # find the closest buses of the backbone networks for each isolated network and each country
     islands_bcountry = {k: d for k, d in gdf_islands.groupby("country")}
-    gdf_map = gdf_backbone_buses.groupby("country").apply(
-        lambda d: gpd.sjoin_nearest(islands_bcountry[d["country"].values[0]], d)
+    gdf_map = (
+        gdf_backbone_buses.query("country in @islands_bcountry")
+        .groupby("country")
+        .apply(lambda d: gpd.sjoin_nearest(islands_bcountry[d["country"].values[0]], d))
     )
     nearest_bus_df = n.buses.loc[n.buses.index.isin(gdf_map.bus_id_right)]
 

From a3086f7565629042849450ece2fbc1c4415bb341 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 23:25:48 +0300
Subject: [PATCH 60/67] Revise config defaults

---
 config.default.yaml  | 2 +-
 config.tutorial.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 96dfeefdb..0aab98fb1 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,7 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 0.2 # [-] isolated buses are being merged into a backbone network if a network load is below the specified share of the overall national load
+    p_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index 758c2ea03..b520ccc20 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,7 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 0.2 # [-] isolated buses are being merged into a backbone network if a network load is below the specified share of the overall national load
+    p_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time

From 10cad77b27e82552e3c068f452d981ead06de365 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 23:35:13 +0300
Subject: [PATCH 61/67] Add an entry to the configtables

---
 doc/configtables/cluster_options.csv | 1 +
 1 file changed, 1 insertion(+)

diff --git a/doc/configtables/cluster_options.csv b/doc/configtables/cluster_options.csv
index c85d6dfb0..e9838367e 100644
--- a/doc/configtables/cluster_options.csv
+++ b/doc/configtables/cluster_options.csv
@@ -8,6 +8,7 @@ simplify_network,,,
 -- remove_stubs_across_borders, bool, "{True, False}", "True: Stub lines and links can be removed across borders."
 -- p_threshold_drop_isolated, MW, positive number, "Isolated buses are discarded if bus mean power is below the `p_threshold_drop_isolated`."
 -- p_threshold_merge_isolated, MW, positive number, "Isolated buses are merged into a single isolated bus if bus mean power is below `p_threshold_merge_isolated`."
+-- p_threshold_fetch_isolated, [-], positive number, "Isolated networks are merged into a backbone network of a respective country if the network load comprises a share of the national load less than p_threshold_fetch_isolated."
 cluster_network,,,
 -- algorithm,,"{hac, kmeans}", "Clustering algorithm used in the cluster_network rule. Options available are Hierarchical Agglomerative Clustering (HAC) or k-means."
 -- feature,,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}. Examples: solar+offwind-cap, solar-time", "Only for Hierarchical Agglomerative Clustering (HAC). Feature(s) used to do the clustering."

From 171bd009e7447cdb2382602dbc327f2673d7afea Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 23:35:22 +0300
Subject: [PATCH 62/67] Add a release note

---
 doc/release_notes.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 53eb6b7df..715adf8b2 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -18,6 +18,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 * Introduce flexible regional selection of the demand files of GEGIS. `PR #991 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/991>`__
 
+* Add an option to merge isolated networks into respective backbone networks by countries. `PR #903 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/903>`__
+
 **Minor Changes and bug-fixing**
 
 * Minor bug-fixing for GADM_ID format naming. `PR #980 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/980>`__, `PR #986 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/986>`__ and `PR #989 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/989>`__

From 6f20a31f0c6707f65f0f1aa4b6678747f6e09e65 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Sat, 30 Mar 2024 23:40:50 +0300
Subject: [PATCH 63/67] Revise naming

---
 config.default.yaml                  | 2 +-
 config.tutorial.yaml                 | 2 +-
 doc/configtables/cluster_options.csv | 2 +-
 scripts/simplify_network.py          | 6 +++---
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/config.default.yaml b/config.default.yaml
index 0aab98fb1..398113391 100644
--- a/config.default.yaml
+++ b/config.default.yaml
@@ -65,7 +65,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
+    s_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/config.tutorial.yaml b/config.tutorial.yaml
index b520ccc20..64b1867a1 100644
--- a/config.tutorial.yaml
+++ b/config.tutorial.yaml
@@ -78,7 +78,7 @@ cluster_options:
     remove_stubs_across_borders: true
     p_threshold_drop_isolated: 20 # [MW] isolated buses are being discarded if bus mean power is below the specified threshold
     p_threshold_merge_isolated: 300 # [MW] isolated buses are being merged into a single isolated bus if a bus mean power is below the specified threshold
-    p_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
+    s_threshold_fetch_isolated: 0.05 # [-] a share of the national load for merging an isolated network into a backbone network
   cluster_network:
     algorithm: kmeans
     feature: solar+onwind-time
diff --git a/doc/configtables/cluster_options.csv b/doc/configtables/cluster_options.csv
index e9838367e..fa30e7035 100644
--- a/doc/configtables/cluster_options.csv
+++ b/doc/configtables/cluster_options.csv
@@ -8,7 +8,7 @@ simplify_network,,,
 -- remove_stubs_across_borders, bool, "{True, False}", "True: Stub lines and links can be removed across borders."
 -- p_threshold_drop_isolated, MW, positive number, "Isolated buses are discarded if bus mean power is below the `p_threshold_drop_isolated`."
 -- p_threshold_merge_isolated, MW, positive number, "Isolated buses are merged into a single isolated bus if bus mean power is below `p_threshold_merge_isolated`."
--- p_threshold_fetch_isolated, [-], positive number, "Isolated networks are merged into a backbone network of a respective country if the network load comprises a share of the national load less than p_threshold_fetch_isolated."
+-- s_threshold_fetch_isolated, [-], positive number, "Isolated networks are merged into a backbone network of a respective country if the network load comprises a share of the national load less than p_threshold_fetch_isolated."
 cluster_network,,,
 -- algorithm,,"{hac, kmeans}", "Clustering algorithm used in the cluster_network rule. Options available are Hierarchical Agglomerative Clustering (HAC) or k-means."
 -- feature,,"Str in the format ‘carrier1+carrier2+...+carrierN-X’, where CarrierI can be from {‘solar’, ‘onwind’, ‘offwind’, ‘ror’} and X is one of {‘cap’, ‘time’}. Examples: solar+offwind-cap, solar-time", "Only for Hierarchical Agglomerative Clustering (HAC). Feature(s) used to do the clustering."
diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index e88618761..2efa615ee 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -1116,7 +1116,7 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         0.0, cluster_config.get("p_threshold_drop_isolated", 0.0)
     )
     p_threshold_merge_isolated = cluster_config.get("p_threshold_merge_isolated", False)
-    p_threshold_fetch_isolated = cluster_config.get("p_threshold_fetch_isolated", False)
+    s_threshold_fetch_isolated = cluster_config.get("s_threshold_fetch_isolated", False)
 
     n = drop_isolated_nodes(n, threshold=p_threshold_drop_isolated)
     if p_threshold_merge_isolated:
@@ -1127,10 +1127,10 @@ def merge_isolated_nodes(n, threshold, aggregation_strategies=dict()):
         )
         busmaps.append(merged_nodes_map)
 
-    if p_threshold_fetch_isolated:
+    if s_threshold_fetch_isolated:
         n, fetched_nodes_map = merge_into_network(
             n,
-            threshold=p_threshold_fetch_isolated,
+            threshold=s_threshold_fetch_isolated,
             aggregation_strategies=aggregation_strategies,
         )
         busmaps.append(fetched_nodes_map)

From ad8d08363078b88f00891692317be023c8147bb0 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 8 Apr 2024 00:22:02 +0300
Subject: [PATCH 64/67] Remove not needed imports

---
 scripts/simplify_network.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index 8a2a7624c..da3a709ff 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -110,8 +110,6 @@
 )
 from pypsa.io import import_components_from_dataframe, import_series_from_dataframe
 from scipy.sparse.csgraph import connected_components, dijkstra
-from scipy.spatial import cKDTree
-from shapely.geometry import Point
 
 sys.settrace
 

From 278bc8005e23eddcd57e5493a4f93f3f6c89e5c4 Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 8 Apr 2024 00:36:00 +0300
Subject: [PATCH 65/67] Implement Davide's suggestion

---
 scripts/simplify_network.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index da3a709ff..e3c0b3b44 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -807,10 +807,9 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
 
     # do not merge sub-networks spanned through a number of countries
     n_buses_gdf["is_multicnt_subntw"] = n_buses_gdf.sub_network.map(
-        n_buses_gdf.groupby(["sub_network"])["country"].apply(
-            lambda x: len(pd.unique(x)) > 1
-        )
+        n_buses_gdf.groupby(["sub_network"]).country.nunique() > 1
     )
+
     gdf_islands = (
         n_buses_gdf.query("~is_multicnt_subntw")
         .query("carrier=='AC'")

From cc97b02e4ac1a535b7eabc453cb7a683c1321ddb Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 8 Apr 2024 00:36:19 +0300
Subject: [PATCH 66/67] Use a safer filtering

---
 scripts/simplify_network.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/scripts/simplify_network.py b/scripts/simplify_network.py
index e3c0b3b44..2ad071f45 100644
--- a/scripts/simplify_network.py
+++ b/scripts/simplify_network.py
@@ -819,11 +819,7 @@ def merge_into_network(n, threshold, aggregation_strategies=dict()):
     if len(gdf_islands) == 0:
         return n, n.buses.index.to_series()
 
-    gdf_backbone_buses = (
-        n_buses_gdf.query("is_backbone_sbntw")
-        .query("carrier=='AC'")
-        .query("load_in_subnetw>0")
-    )
+    gdf_backbone_buses = n_buses_gdf.query("is_backbone_sbntw").query("carrier=='AC'")
 
     # find the closest buses of the backbone networks for each isolated network and each country
     islands_bcountry = {k: d for k, d in gdf_islands.groupby("country")}

From f92638e651edf619b1022bd000db49e42238916e Mon Sep 17 00:00:00 2001
From: ekatef <e.v.kasilova@gmail.com>
Date: Mon, 8 Apr 2024 14:09:57 +0300
Subject: [PATCH 67/67] Add a release note

---
 doc/release_notes.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/doc/release_notes.rst b/doc/release_notes.rst
index 2fb1f9a23..b2666095b 100644
--- a/doc/release_notes.rst
+++ b/doc/release_notes.rst
@@ -20,6 +20,8 @@ E.g. if a new rule becomes available describe how to use it `snakemake -j1 run_t
 
 * Generalize line types for AC and DC networks. `PR #999 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/999>`__
 
+* Add an option to merge isolated networks into respective backbone networks by countries. `PR #903 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/903>`__
+
 **Minor Changes and bug-fixing**
 
 * Minor bug-fixing for GADM_ID format naming. `PR #980 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/980>`__, `PR #986 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/986>`__ and `PR #989 <https://github.com/pypsa-meets-earth/pypsa-earth/pull/989>`__