Integrate ribasim lumping vrij afwaterend (#118)

This work is all done by Sweco, @harm-nomden-sweco. Originally pushed to the branch https://github.com/Deltares/Ribasim-NL/tree/ribasim_lumping_vrij_afwaterend, I created this copy to integrate it into main without changing the original. --------- Co-authored-by: harm-nomden-sweco <[email protected]>
Deltares · Jul 30, 2024 · 9c5af32 · 9c5af32
1 parent 3479218
commit 9c5af32
Show file tree

Hide file tree

Showing 36 changed files with 9,488 additions and 2 deletions.
diff --git a/ruff.toml b/ruff.toml
@@ -1,12 +1,11 @@
 extend-include = ["*.ipynb"]
-exclude = ["scripts/*"]
 target-version = "py310"
 line-length = 120
 
 [lint]
 # See https://docs.astral.sh/ruff/rules/
 select = ["D", "E", "F", "NPY", "PD", "C4", "I", "UP"]
-ignore = ["D1", "D202", "D205", "D400", "D401", "D404", "E402", "E501", "E712", "E721", "F401", "F403", "F405", "PD002", "PD003", "PD008", "PD009", "PD011", "PD015", "PD901"]
+ignore = ["D1", "D202", "D205", "D400", "D401", "D404", "E402", "E501", "E712", "E721", "F401", "F403", "F405", "PD002", "PD003", "PD008", "PD009", "PD010", "PD011", "PD013", "PD015", "PD901"]
 fixable = ["ALL"]
 
 [lint.pydocstyle]

diff --git a/scripts/notebooks/hydamo_0_analyse_data_waterboard.ipynb b/scripts/notebooks/hydamo_0_analyse_data_waterboard.ipynb
@@ -0,0 +1,283 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "from pathlib import Path\n",
+    "\n",
+    "import geopandas as gpd\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from hydamo.datamodel import HyDAMO\n",
+    "from pandas_xlsx_tables import xlsx_tables_to_dfs\n",
+    "from ribasim_lumping_tools.LHM_data_bewerking_analyse_utils import (\n",
+    "    check_ids_hydamo_data,\n",
+    "    check_if_object_on_hydroobject,\n",
+    "    export_to_geopackage,\n",
+    "    read_original_data,\n",
+    "    translate_data_to_hydamo_format,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1",
+   "metadata": {},
+   "source": [
+    "Vertaal originele data naar Hydamo data zoals gedefinieerd in de tabel hydamo_data_format.xlsx"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "base_dir = \"..\\\\\"\n",
+    "\n",
+    "waterboard = \"AAenMaas\"\n",
+    "waterboard_code = 1"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "waterboard_dir = Path(base_dir, waterboard, \"verwerkt\")\n",
+    "path_hydamo_format = Path(waterboard_dir, \"HyDAMO_format_AAenMaas.xlsx\")\n",
+    "hydamo_format = xlsx_tables_to_dfs(path_hydamo_format)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# eerst inlezen hydroobject, vertalen naar hydamo\n",
+    "hydamo_object = \"hydroobject\"\n",
+    "hydamo_translate_table, data_original = read_original_data(waterboard_dir, hydamo_format, hydamo_object, waterboard)\n",
+    "hydroobject = translate_data_to_hydamo_format(hydamo_translate_table, data_original)\n",
+    "\n",
+    "# maak een created_date aan indien nodig\n",
+    "if \"created_date\" not in data_original.columns:\n",
+    "    hydroobject[\"created_date\"] = pd.NaT\n",
+    "# transformeer created_date waardes indien nodig\n",
+    "hydroobject[\"created_date\"] = hydroobject[\"created_date\"].replace(\"\", pd.NaT)\n",
+    "\n",
+    "# hydroobject.loc[hydroobject['code'].duplicated(keep=False), 'data_issue'] = 'duplicate_id'\n",
+    "data_hydamo_dict = {\"hydroobject\": hydroobject.set_crs(28992)}\n",
+    "\n",
+    "# geometry hydroobject bufferen met 10 cm voor de spatial join\n",
+    "hydroobject[\"buffer\"] = hydroobject.copy().buffer(5)  # 5 meter buffer omdat anders relevante gemalen wegvallen\n",
+    "hydroobject_buffered = hydroobject.set_geometry(\"buffer\").set_crs(28992)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5",
+   "metadata": {},
+   "source": [
+    "Specificeer welke HyDAMO data je wilt omzetten"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "6",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo_objects = [\n",
+    "    \"stuw\",\n",
+    "    \"gemaal\",\n",
+    "    \"afvoergebiedaanvoergebied\",\n",
+    "    \"pomp\",\n",
+    "    ##'peilgebiedvigerend',\n",
+    "    ##'peilgebiedpraktijk',\n",
+    "    ##'streefpeil',\n",
+    "    \"duikersifonhevel\",\n",
+    "    ##'afsluiter',\n",
+    "    ##'sluis',\n",
+    "]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "for hydamo_object in hydamo_objects:\n",
+    "    # lees aangeleverde data en hydamo tabel voor gegeven kunstwerk en waterschap\n",
+    "    table_hydamo, data_original = read_original_data(waterboard_dir, hydamo_format, hydamo_object, waterboard)\n",
+    "    if data_original is None:\n",
+    "        data_hydamo_dict[hydamo_object] = None\n",
+    "    else:\n",
+    "        # vertaal data naar hydamo-ribasim format\n",
+    "        data_hydamo = translate_data_to_hydamo_format(table_hydamo, data_original)\n",
+    "\n",
+    "        # maak een created_date aan indien nodig\n",
+    "        if \"created_date\" not in data_original.columns and hydamo_object != \"sluis\":\n",
+    "            hydroobject[\"created_date\"] = pd.NaT\n",
+    "        if \"last_edited_date\" not in data_original.columns and hydamo_object == \"afsluiter\":\n",
+    "            hydroobject[\"last_edited_date\"] = pd.NaT\n",
+    "        if \"lvpublicatiedatum\" not in data_original.columns and hydamo_object == \"afsluiter\":\n",
+    "            hydroobject[\"lvpublicatiedatum\"] = pd.NaT\n",
+    "\n",
+    "        # transformeer created_date waardes indien nodig\n",
+    "        if hydamo_object != \"sluis\":\n",
+    "            data_hydamo[\"created_date\"] = data_hydamo[\"created_date\"].replace(\"\", pd.NaT)\n",
+    "        if hydamo_object == \"afsluiter\":\n",
+    "            data_hydamo[\"last_edited_date\"] = data_hydamo[\"last_edited_date\"].replace(\"\", pd.NaT)\n",
+    "            data_hydamo[\"lvpublicatiedatum\"] = data_hydamo[\"lvpublicatiedatum\"].replace(\"\", pd.NaT)\n",
+    "\n",
+    "        # check dubbele id's\n",
+    "        if hydamo_object not in [\"streefpeil\"]:  # streefpeil heeft geen code, alleen globalid etc\n",
+    "            data_hydamo.loc[data_hydamo[\"code\"].duplicated(keep=False), \"data_issue\"] = \"duplicate_id\"\n",
+    "            # TODO check op 'code' lijkt met logischer want die kolom wordt vaker gebruikt. Maar bij WDOD bijv. is die niet ingevuld. Toch op globalid?\n",
+    "        # check of kuntstwerk op hydroobject ligt\n",
+    "        if hydamo_object in [\"stuw\", \"gemaal\", \"duikersifonhevel\", \"sluis\"]:\n",
+    "            data_hydamo = check_if_object_on_hydroobject(\n",
+    "                data_hydamo=data_hydamo, hydroobject_buffered=hydroobject_buffered\n",
+    "            )\n",
+    "            # verwijder kunstwerken die niet op hydroobject liggen\n",
+    "            data_hydamo = data_hydamo[data_hydamo[\"code_hydroobject\"] != \"niet op hydroobject\"]\n",
+    "            data_hydamo = data_hydamo.reset_index()\n",
+    "        # voeg toe aan de hydamo dataset\n",
+    "        data_hydamo_dict[hydamo_object] = data_hydamo"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
+   "source": [
+    "Waterschap specifieke acties"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "9",
+   "metadata": {},
+   "source": [
+    "Export normal"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "10",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "# for hydamo_object in ['hydroobject'] + hydamo_objects:\n",
+    "#     # export to geopackage\n",
+    "#     export_to_geopackage(\n",
+    "#         data_hydamo=data_hydamo_dict[hydamo_object],\n",
+    "#         hydamo_format=hydamo_format,\n",
+    "#         waterboard=waterboard,\n",
+    "#         hydamo_object=hydamo_object\n",
+    "#     )"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "11",
+   "metadata": {},
+   "source": [
+    "### ribasim-nl hydamo"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "12",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo = HyDAMO(version=\"2.2.1_sweco\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "13",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for hydamo_object in [\"hydroobject\"] + hydamo_objects:\n",
+    "    data_hydamo = data_hydamo_dict[hydamo_object]\n",
+    "    if hydamo_object == \"stuw\":\n",
+    "        data_hydamo = data_hydamo.drop(columns=[\"code_hydroobject\", \"data_issue\"])  # ,'index_right'\n",
+    "    data_hydamo = check_ids_hydamo_data(data_hydamo, waterboard_code, hydamo_object)\n",
+    "    setattr(hydamo, hydamo_object, data_hydamo)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "14",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "hydamo.to_geopackage(\"..\\\\hydamo.gpkg\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "15",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "ribasim_lumping_venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}