Add deleted_market_actors to data model

CHANGELOG.md

@@ -8,9 +8,13 @@ and the versioning aims to respect [Semantic Versioning](http://semver.org/spec/
 
 ## [v0.XX.X] unreleased - 2024-XX-XX
 ### Added
-- Add OFFIS eV as partner organization [#493](https://github.com/OpenEnergyPlatform/open-MaStR/pull/493)
+- Add `deleted_market_actors` to data model and prevent crash on unknown tables
+  [#575](https://github.com/OpenEnergyPlatform/open-MaStR/pull/575)
+- Add OFFIS eV as partner organization
+  [#493](https://github.com/OpenEnergyPlatform/open-MaStR/pull/493)
 ### Changed
-- Fix usercff workflow [#545](https://github.com/OpenEnergyPlatform/open-MaStR/issues/544)
+- Fix usercff workflow
+  [#545](https://github.com/OpenEnergyPlatform/open-MaStR/issues/544)
 - Fix docs on user-defined output path for csv, xml, database
   [#549](https://github.com/OpenEnergyPlatform/open-MaStR/issues/549)
 - Set pandas version to >=2.2.2 for compatibility with numpy v2.0

docs/dataset.md

@@ -81,6 +81,8 @@ After downloading the MaStR, you will find a database with a large number of tab
     | permit  |  |
     | storage_units  |  |
     | kwk  | *short for: Combined heat and power (CHP)* |
+    | deleted_units | Units from all technologies that were deleted or deactivated |
+    | deleted_market_actors | Market actors that were deleted or deactivated |
 
 
 ### MaStR data model

open_mastr/mastr.py

@@ -77,7 +77,6 @@ class Mastr:
     """
 
     def __init__(self, engine="sqlite", connect_to_translated_db=False) -> None:
-
         validate_parameter_format_for_mastr_init(engine)
 
         self.output_dir = get_output_dir()
@@ -150,6 +149,7 @@ def download(
             | "balancing_area"      | Yes  | No   |
             | "permit"              | Yes  | Yes  |
             | "deleted_units"       | Yes  | No   |
+            | "deleted_market_actors"| Yes | No   |
             | "retrofit_units"      | Yes  | No   |
         date : None or `datetime.datetime` or str, optional
 

open_mastr/utils/constants.py

@@ -16,6 +16,7 @@
     "balancing_area",
     "permit",
     "deleted_units",
+    "deleted_market_actors",
     "retrofit_units",
     "changed_dso_assignment",
     "storage_units",
@@ -63,6 +64,7 @@
     "market_roles",
     "permit",
     "deleted_units",
+    "deleted_market_actors",
     "retrofit_units",
     "changed_dso_assignment",
     "storage_units",
@@ -106,6 +108,7 @@
     "balancing_area": ["bilanzierungsgebiete"],
     "permit": ["einheitengenehmigung"],
     "deleted_units": ["geloeschteunddeaktivierteeinheiten"],
+    "deleted_market_actors": ["geloeschteunddeaktiviertemarktakteure"],
     "retrofit_units": ["ertuechtigungen"],
     "changed_dso_assignment": ["einheitenaenderungnetzbetreiberzuordnungen"],
 }
@@ -125,6 +128,7 @@
     "balancing_area": ["balancing_area"],
     "permit": ["permit"],
     "deleted_units": ["deleted_units"],
+    "deleted_market_actors": ["deleted_market_actors"],
     "retrofit_units": ["retrofit_units"],
     "changed_dso_assignment": ["changed_dso_assignment"],
 }
@@ -163,10 +167,7 @@
         "eeg_data": "HydroEeg",
         "permit_data": "Permit",
     },
-    "nuclear": {
-        "unit_data": "NuclearExtended",
-        "permit_data": "Permit"
-    },
+    "nuclear": {"unit_data": "NuclearExtended", "permit_data": "Permit"},
     "storage": {
         "unit_data": "StorageExtended",
         "eeg_data": "StorageEeg",
@@ -185,6 +186,7 @@
     "balancing_area": "BalancingArea",
     "permit": "Permit",
     "deleted_units": "DeletedUnits",
+    "deleted_market_actors": "DeletedMarketActors",
     "retrofit_units": "RetrofitUnits",
     "changed_dso_assignment": "ChangedDSOAssignment",
     "storage_units": "StorageUnits",

open_mastr/utils/orm.py

@@ -780,6 +780,14 @@ class DeletedUnits(ParentAllTables, Base):
     EinheitBetriebsstatus = Column(String)
 
 
+class DeletedMarketActors(ParentAllTables, Base):
+    __tablename__ = "deleted_market_actors"
+
+    MarktakteurMastrNummer = Column(String, primary_key=True)
+    MarktakteurStatus = Column(String)
+    DatumLetzteAktualisierung = Column(DateTime(timezone=True))
+
+
 class RetrofitUnits(ParentAllTables, Base):
     __tablename__ = "retrofit_units"
 
@@ -1006,6 +1014,11 @@ class ChangedDSOAssignment(ParentAllTables, Base):
         "__class__": DeletedUnits,
         "replace_column_names": None,
     },
+    "geloeschteunddeaktiviertemarktakteure": {
+        "__name__": DeletedMarketActors.__tablename__,
+        "__class__": DeletedMarketActors,
+        "replace_column_names": None,
+    },
     "marktrollen": {
         "__name__": MarketRoles.__tablename__,
         "__class__": MarketRoles,

open_mastr/xml_download/colums_to_replace.py

@@ -57,6 +57,8 @@
     "Pumpspeichertechnologie",
     "Einsatzort",
     # geloeschteunddeaktivierteEinheiten
+    # geloeschteunddeaktivierteMarktAkteure
+    "MarktakteurStatus",
     # lokationen
     # marktakteure
     "Personenart",

open_mastr/xml_download/utils_write_to_database.py

@@ -73,9 +73,16 @@ def is_table_relevant(xml_tablename: str, include_tables: list) -> bool:
     have it in the database."""
     # few tables are only needed for data cleansing of the xml files and contain no
     # information of relevance
-    boolean_write_table_to_sql_database = (
-        tablename_mapping[xml_tablename]["__class__"] is not None
-    )
+    try:
+        boolean_write_table_to_sql_database = (
+            tablename_mapping[xml_tablename]["__class__"] is not None
+        )
+    except KeyError:
+        print(
+            f"Table '{xml_tablename}' is not supported by your open-mastr version and "
+            f"will be skipped."
+        )
+        return False
     # check if the table should be written to sql database (depends on user input)
     include_count = include_tables.count(xml_tablename)
 

tests/test_helpers.py

@@ -66,6 +66,7 @@ def parameter_dict_working_list():
             "balancing_area",
             "permit",
             "deleted_units",
+            "deleted_market_actors",
             "retrofit_units",
             None,
             ["wind", "solar"],
@@ -250,7 +251,12 @@ def test_validate_parameter_format_for_mastr_init(db):
 
 
 def test_transform_data_parameter():
-    (data, api_data_types, api_location_types, harm_log,) = transform_data_parameter(
+    (
+        data,
+        api_data_types,
+        api_location_types,
+        harm_log,
+    ) = transform_data_parameter(
         method="API",
         data=["wind", "location"],
         api_data_types=["eeg_data"],
@@ -369,7 +375,6 @@ def test_db_query_to_csv(tmpdir, engine):
             os.remove(csv_path)
 
         for addit_table in addit_tables:
-
             csv_path = join(
                 get_data_version_dir(),
                 f"bnetza_mastr_{addit_table}_raw.csv",