Update cloud

notebooks/rijkswaterstaat/netwerk.py

@@ -0,0 +1,95 @@
+# %%
+import geopandas as gpd
+import pandas as pd
+from ribasim_nl import CloudStorage
+from ribasim_nl.geometry import cut_basin
+
+cloud = CloudStorage()
+
+# %% inlezen dataframes
+
+
+rws_opp_poly_gdf = gpd.read_file(
+    cloud.joinpath(
+        "Rijkswaterstaat", "aangeleverd", "oppervlaktewaterlichamen_rijk.gpkg"
+    )
+)
+
+
+# %% write it generic so it can b converted to 1 function
+krw_poly_gdf = gpd.read_file(
+    cloud.joinpath(
+        "Basisgegevens", "KRW", "krw_oppervlaktewaterlichamen_nederland_vlakken.gpkg"
+    )
+)
+
+krw_split_lines_gdf = gpd.read_file(
+    cloud.joinpath("Rijkswaterstaat", "verwerkt", "krw_split_lijnen.gpkg")
+)
+
+rws_krw_basins = cloud.joinpath(
+    "Rijkswaterstaat", "verwerkt", "krw_basins_vlakken.gpkg"
+)
+
+lines_gdf = krw_split_lines_gdf
+poly_gdf = krw_poly_gdf
+
+for line in lines_gdf.explode(index_parts=False).itertuples():
+    # filter by spatial index
+    idx = poly_gdf.sindex.intersection(line.geometry.bounds)
+    poly_select_gdf = poly_gdf.iloc[idx][poly_gdf.iloc[idx].intersects(line.geometry)]
+
+    ## filter by intersecting geometry
+    poly_select_gdf = poly_select_gdf[poly_select_gdf.intersects(line.geometry)]
+
+    ## filter polygons with two intersection-points only
+    poly_select_gdf = poly_select_gdf[
+        poly_select_gdf.geometry.boundary.intersection(line.geometry).apply(
+            lambda x: False if x.geom_type == "Point" else len(x.geoms) == 2
+        )
+    ]
+
+    ## if there are no polygon-candidates, something is wrong
+    if poly_select_gdf.empty:
+        print(
+            f"no intersect for {line}. Please make sure it is extended outside the basin on two sides"
+        )
+    else:
+        ## we create 2 new fatures in data
+        data = []
+        for basin in poly_select_gdf.itertuples():
+            kwargs = basin._asdict()
+            for geom in cut_basin(basin.geometry, line.geometry).geoms:
+                kwargs["geometry"] = geom
+                data += [{**kwargs}]
+
+    ## we update poly_gdf with new polygons
+    poly_gdf = poly_gdf[~poly_gdf.index.isin(poly_select_gdf.index)]
+    poly_gdf = pd.concat(
+        [poly_gdf, gpd.GeoDataFrame(data, crs=poly_gdf.crs).set_index("Index")],
+        ignore_index=True,
+    )
+
+poly_gdf.to_file(rws_krw_basins)
+
+# %% add name
+naming_poly_gdf = rws_opp_poly_gdf
+naming_column = "waterlichaam"
+
+# start function
+columns = list(poly_gdf.columns) + [naming_column]
+poly_gdf["left_index"] = poly_gdf.index
+
+overlay_gdf = gpd.overlay(poly_gdf, naming_poly_gdf, how="intersection")
+overlay_gdf["geom_area"] = overlay_gdf.geometry.area
+overlay_gdf.sort_values(by="geom_area", inplace=True)
+
+overlay_gdf.drop_duplicates(subset="left_index", keep="last", inplace=True)
+overlay_gdf.sort_values(by="left_index", inplace=True)
+overlay_gdf.index = poly_gdf.index
+overlay_gdf = overlay_gdf[columns]
+
+overlay_gdf.loc[:, ["geometry"]] = poly_gdf.geometry
+overlay_gdf.to_file(rws_krw_basins)
+
+# %%

src/ribasim_nl/ribasim_nl/cloud.py

@@ -111,7 +111,7 @@ def validate_authority(self, authority):
     def file_url(self, file_path: Union[str, Path]) -> str:
         relative_path = Path(file_path).relative_to(self.data_dir)
 
-        return f"{self.url}/{relative_path}"
+        return f"{self.url}/{relative_path.as_posix()}"
 
     def relative_url(self, file_url: str) -> str:
         return file_url[len(self.url) + 1 :]
@@ -130,7 +130,7 @@ def joinurl(self, *args: str):
             return self.url
 
     def joinpath(self, *args: str):
-        self.data_dir.joinpath(*args)
+        return self.data_dir.joinpath(*args)
 
     def upload_file(self, file_path: Path):
         # get url
@@ -265,6 +265,31 @@ def download_content(self, url, overwrite: bool = False):
                     logger.info(f"downloading file {path}")
                     self.download_file(file_url=item_url)
 
+    def upload_content(self, dir_path: Path, overwrite: bool = False):
+        """Upload content of a directory recursively."""
+
+        # get all remote content
+        content = self.content(self.joinurl(self.relative_path(dir_path).as_posix()))
+
+        # get al local directories and files.
+        dirs = [i for i in dir_path.glob("*") if i.is_dir()]
+        files = [i for i in dir_path.glob("*") if i.is_file()]
+
+        # add files
+        for file_path in files:
+            if (file_path.stem not in content) or overwrite:
+                logger.info(f"uploading file {file_path}")
+                self.upload_file(file_path)
+
+        # add dirs and upload their content recursively
+        for path in dirs:
+            if path.stem not in content:
+                remote_path = self.relative_path(path).as_posix()
+                self.create_dir(remote_path)
+                self.upload_content(
+                    dir_path=dir_path.joinpath(path.stem), overwrite=overwrite
+                )
+
     def download_aangeleverd(self, authority: str, overwrite: bool = False):
         """Download all files in folder 'aangeleverd'"""
         self.validate_authority(authority)
@@ -279,6 +304,26 @@ def download_verwerkt(self, authority: str, overwrite: bool = False):
         url = self.joinurl(authority, "verwerkt")
         self.download_content(url, overwrite=overwrite)
 
+    def download_basisgegevens(self, bronnen: List[str] = [], overwrite=True):
+        """Download sources in the folder 'Basisgegevens'"""
+        source_data = self.source_data
+        if not bronnen:
+            bronnen = source_data
+
+        for bron in bronnen:
+            if bron not in source_data:
+                raise ValueError(f"""{bron} not in {source_data}""")
+            else:
+                url = self.joinurl("Basisgegevens", bron)
+                self.download_content(url, overwrite=overwrite)
+
+    def upload_verwerkt(self, authority: str, overwrite: bool = False):
+        """Upload all files in folder 'verwerkt'"""
+        self.validate_authority(authority)
+
+        dir_path = self.joinpath(authority, "verwerkt")
+        self.upload_content(dir_path, overwrite=overwrite)
+
     def download_all(self, authority, overwrite: bool = False):
         """Download all files for authority."""
         url = self.joinurl(authority)