Implement measurement field conversion from Vision

src/power_grid_model_io/converters/tabular_converter.py

@@ -178,7 +178,15 @@ def _convert_table_to_component(
         if table not in data:
             return None
 
-        n_records = len(data[table])
+        table_mask = np.arary(True)  * len(data[table])
+        if "filter" in attributes:
+            table_mask = self._parse_filter()
+            pass
+
+        n_records = np.sum(table_mask)
+
+        if n_records == 0:
+            return None
 
         try:
             pgm_data = initialize_array(data_type=data_type, component_type=component, shape=n_records)
@@ -193,7 +201,7 @@ def _convert_table_to_component(
 
         for attr, col_def in sorted_attributes:
             self._convert_col_def_to_attribute(
-                data=data,
+                data=data[table_mask],
                 pgm_data=pgm_data,
                 table=table,
                 component=component,
@@ -213,6 +221,7 @@ def _convert_col_def_to_attribute(
         component: str,
         attr: str,
         col_def: Any,
+        table_mask: np.ndarray,
         extra_info: Optional[ExtraInfo],
     ):
         """This function updates one of the attributes of pgm_data, based on the corresponding table/column in a tabular
@@ -242,26 +251,33 @@ def _convert_col_def_to_attribute(
         """
         # To avoid mistakes, the attributes in the mapping should exist. There is one extra attribute called
         # 'extra' in which extra information can be captured.
-        if attr not in pgm_data.dtype.names and attr != "extra":
+        if attr not in pgm_data.dtype.names and attr not in ["extra", "filter"]:
             attrs = ", ".join(pgm_data.dtype.names)
             raise KeyError(f"Could not find attribute '{attr}' for '{component}s'. (choose from: {attrs})")
 
         if attr == "extra":
             # Extra info must be linked to the object IDs, therefore the uuids should be known before extra info can
             # be parsed. Before this for loop, it is checked that "id" exists and it is placed at the front.
             self._handle_extra_info(
-                data=data, table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info
+                data=data[table_mask], table=table, col_def=col_def, uuids=pgm_data["id"], extra_info=extra_info
             )
             # Extra info should not be added to the numpy arrays, so let's continue to the next attribute
             return
 
-        attr_data = self._parse_col_def(data=data, table=table, col_def=col_def, extra_info=extra_info)
+        attr_data = self._parse_col_def(data=data[table_mask], table=table, col_def=col_def, extra_info=extra_info)
 
         if len(attr_data.columns) != 1:
             raise ValueError(f"DataFrame for {component}.{attr} should contain a single column ({attr_data.columns})")
 
         pgm_data[attr] = attr_data.iloc[:, 0]
 
+    def _parse_filters() -> pd.Series[bool]:
+        mask = True * len(data) * 483
+        for function, args in mapping["filter"]  {
+            mask &= data.apply(function, arg)
+        }
+        return pd.Series()
+
     def _handle_extra_info(
         self,
         data: TabularData,
@@ -517,7 +533,7 @@ def _parse_auto_id(
             key_col_def: A column definition which should be unique for each object within the current table
 
         Returns: A single column containing numerical ids
-
+    
         """
 
         # Handle reference table

src/power_grid_model_io/functions/_functions.py

@@ -10,8 +10,7 @@
 
 import numpy as np
 import structlog
-from power_grid_model import WindingType
-from power_grid_model import MeasuredTerminalType
+from power_grid_model import MeasuredTerminalType, WindingType
 
 T = TypeVar("T")
 
@@ -29,7 +28,7 @@
     "cable_from": MeasuredTerminalType.branch_from,
     "cable_to": MeasuredTerminalType.branch_to,
     "line_from": MeasuredTerminalType.branch_from,
-    "line_to": MeasuredTerminalType.branch_to, 
+    "line_to": MeasuredTerminalType.branch_to,
     "reactance_coil_from": MeasuredTerminalType.branch_from,
     "reactance_coil_to": MeasuredTerminalType.branch_to,
     "special_transformer_from": MeasuredTerminalType.branch_from,
@@ -48,7 +47,8 @@
     "wind_turbine": MeasuredTerminalType.generator,
     "load": MeasuredTerminalType.load,
 }
-
+
+
 def has_value(value: Any) -> bool:
     """
     Return True if the value is not None, NaN or empty string.
@@ -129,12 +129,26 @@ def both_zeros_to_nan(value: float, other_value: float) -> float:
         return float("nan")
     return value
 
+
 def find_terminal_type(**kwargs) -> MeasuredTerminalType:
     """
     Return the measured terminal type, based on the string representation
-    """ 
+    """
     for key, id in kwargs.items():
         if id is not None:
             return MEASURED_TERMINAL_TYPE_MAP[key]
     _LOG.warning("No measured terminal type is found!")
-    return float("nan")
+    return float("nan")
+
+
+def if_not_link(input_string) -> bool:
+    """
+    Check if the measurement field is applied on a link
+    """
+    return input_string != "link"
+
+def filter_if_object(object_name: str, excl_object: str) -> bool:
+    """
+    Return false if the measured object should be excluded.
+    """
+    return object_name != excl_object

src/power_grid_model_io/utils/auto_id.py

@@ -119,4 +119,4 @@ def __getitem__(self, idx: int) -> Any:
         Returns:
             The original item
         """
-        return getattr(self._items[idx], None)
+        return self._items.get(idx)

tests/data/vision/vision_9_7_en.yaml

@@ -804,10 +804,11 @@ grid:
       u_angle_measured: nan
       u_sigma: nan
       filter:
-        - power_grid_model_io.functions.filter_if_empty:
+        - power_grid_model_io.functions.has_value:
           - power_1
-        - power_grid_model_io.functions.filter_if_link:
-          - InObject.Sort
+        - power_grid_model_io.functions.filter_if_object:
+          object: InObject.Sort
+          excl_object: link
       extra:
         - ID
         - Name
@@ -931,10 +932,11 @@ grid:
       p_sigma: nan
       q_sigma: nan
       filter:
-        - power_grid_model_io.functions.filter_if_empty:
+        - power_grid_model_io.functions.has_value:
           - power_2
-        - power_grid_model_io.functions.filter_if_link:
-          - InObject.Sort
+        - power_grid_model_io.functions.filter_if_object:
+          object: InObject.Sort
+          excl_object: link
       extra:
         - ID
         - Name
@@ -1058,10 +1060,11 @@ grid:
       p_sigma: nan
       q_sigma: nan
       filter:
-        - power_grid_model_io.functions.filter_if_empty:
+        - power_grid_model_io.functions.has_value:
           - power_3
-        - power_grid_model_io.functions.filter_if_link:
-          - InObject.Sort
+        - power_grid_model_io.functions.filter_if_object:
+          object: InObject.Sort
+          excl_object: link
       extra:
         - ID
         - Name
@@ -1185,10 +1188,11 @@ grid:
       p_sigma: nan
       q_sigma: nan
       filter:
-        - power_grid_model_io.functions.filter_if_empty:
+        - power_grid_model_io.functions.has_value:
           - power_4
-        - power_grid_model_io.functions.filter_if_link:
-          - InObject.Sort
+        - power_grid_model_io.functions.filter_if_object:
+          object: InObject.Sort
+          excl_object: link
       extra:
         - ID
         - Name