Problem with join

[LucaMarconato]

CC @melonora

The recent changes on the join operations #813 introduced a bug (currently not covered by tests) which can be reproduced by running the Visium notebook from spatialdata-notebook.

This is the log of the error from the data-ci machine. Unfortunately this bug is blocking a release.

Traceback (most recent call last):
  File "/mnt/miniconda3/envs/ome/bin/jupyter-nbconvert", line 10, in <module>
    sys.exit(main())
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/jupyter_core/application.py", line 283, in launch_instance
    super().launch_instance(argv=argv, **kwargs)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/traitlets/config/application.py", line 1075, in launch_instance
    app.start()
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/nbconvertapp.py", line 420, in start
    self.convert_notebooks()
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/nbconvertapp.py", line 597, in convert_notebooks
    self.convert_single_notebook(notebook_filename)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/nbconvertapp.py", line 563, in convert_single_notebook
    output, resources = self.export_single_notebook(
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/nbconvertapp.py", line 487, in export_single_notebook
    output, resources = self.exporter.from_filename(
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/exporters/exporter.py", line 201, in from_filename
    return self.from_file(f, resources=resources, **kw)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/exporters/exporter.py", line 220, in from_file
    return self.from_notebook_node(
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/exporters/notebook.py", line 36, in from_notebook_node
    nb_copy, resources = super().from_notebook_node(nb, resources, **kw)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/exporters/exporter.py", line 154, in from_notebook_node
    nb_copy, resources = self._preprocess(nb_copy, resources)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/exporters/exporter.py", line 353, in _preprocess
    nbc, resc = preprocessor(nbc, resc)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/preprocessors/base.py", line 48, in __call__
    return self.preprocess(nb, resources)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/preprocessors/execute.py", line 103, in preprocess
    self.preprocess_cell(cell, resources, index)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbconvert/preprocessors/execute.py", line 124, in preprocess_cell
    cell = self.execute_cell(cell, index, store_history=True)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/jupyter_core/utils/__init__.py", line 165, in wrapped
    return loop.run_until_complete(inner)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/asyncio/base_events.py", line 641, in run_until_complete
    return future.result()
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbclient/client.py", line 1062, in async_execute_cell
    await self._check_raise_for_error(cell, cell_index, exec_reply)
  File "/mnt/miniconda3/envs/ome/lib/python3.10/site-packages/nbclient/client.py", line 918, in _check_raise_for_error
    raise CellExecutionError.from_cell_and_msg(cell, exec_reply_content)
nbclient.exceptions.CellExecutionError: An error occurred while executing the following cell:
------------------
(
    visium_sdata.pl.render_images(elements="ST8059050_hires_image")
    .pl.render_shapes(elements="ST8059050", color="mt-Co3")
    .pl.show()
)
------------------
----- stdout -----
INFO     Dropping coordinate system 'ST8059048' since it doesn't have relevant elements.
----- stdout -----
INFO     Rasterizing image for faster rendering.
----- stderr -----
/tmp/ipykernel_2637212/2752567425.py:2: DeprecationWarning: `elements` is being deprecated as an argument to `PlotAccessor.render_images.render_images` in spatialdata_plot version version 0.3.0, switch to `element` instead.
  visium_sdata.pl.render_images(elements="ST8059050_hires_image")
/tmp/ipykernel_2637212/2752567425.py:2: DeprecationWarning: `elements` is being deprecated as an argument to `PlotAccessor.render_shapes.render_shapes` in spatialdata_plot version 0.3.0, switch to `element` instead.
  visium_sdata.pl.render_images(elements="ST8059050_hires_image")
/mnt/repos/spatialdata/src/spatialdata/_core/spatialdata.py:167: UserWarning: The table is annotating 'ST8059048', which is not present in the SpatialData object.
  self.validate_table_in_spatialdata(v)
----- stderr -----
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers). Got range [-0.028571429..1.0].
------------------
---------------------------------------------------------------------------
InvalidIndexError                         Traceback (most recent call last)
Cell In[6], line 4
      1 (
      2     visium_sdata.pl.render_images(elements="ST8059050_hires_image")
      3     .pl.render_shapes(elements="ST8059050", color="mt-Co3")
----> 4     .pl.show()
      5 )
File /mnt/repos/spatialdata-plot/src/spatialdata_plot/pl/basic.py:920, in PlotAccessor.show(self, coordinate_systems, legend_fontsize, legend_fontweight, legend_loc, legend_fontoutline, na_in_legend, colorbar, wspace, hspace, ncols, frameon, figsize, dpi, fig, title, share_extent, pad_extent, ax, return_ax, save)
    915     wanted_elements, wanted_shapes_on_this_cs, wants_shapes = _get_wanted_render_elements(
    916         sdata, wanted_elements, params_copy, cs, "shapes"
    917     )
    919     if wanted_shapes_on_this_cs:
--> 920         _render_shapes(
    921             sdata=sdata,
    922             render_params=params_copy,
    923             coordinate_system=cs,
    924             ax=ax,
    925             fig_params=fig_params,
    926             scalebar_params=scalebar_params,
    927             legend_params=legend_params,
    928         )
    930 elif cmd == "render_points" and has_points:
    931     wanted_elements, wanted_points_on_this_cs, wants_points = _get_wanted_render_elements(
    932         sdata, wanted_elements, params_copy, cs, "points"
    933     )
File /mnt/repos/spatialdata-plot/src/spatialdata_plot/pl/render.py:83, in _render_shapes(sdata, render_params, coordinate_system, ax, fig_params, scalebar_params, legend_params)
     81     shapes = sdata_filt[element]
     82 else:
---> 83     element_dict, joined_table = join_spatialelement_table(
     84         sdata, spatial_element_names=element, table_name=table_name, how="inner"
     85     )
     86     sdata_filt[element] = shapes = element_dict[element]
     87     joined_table.uns["spatialdata_attrs"]["region"] = (
     88         joined_table.obs[joined_table.uns["spatialdata_attrs"]["region_key"]].unique().tolist()
     89     )
File /mnt/repos/spatialdata/src/spatialdata/_core/query/relational_query.py:677, in join_spatialelement_table(sdata, spatial_element_names, spatial_elements, table_name, table, how, match_rows)
    674         for name, element in getattr(derived_sdata, element_type).items():
    675             elements_dict[element_type][name] = element
--> 677 elements_dict_joined, table = _call_join(elements_dict, table, how, match_rows)
    678 return elements_dict_joined, table
File /mnt/repos/spatialdata/src/spatialdata/_core/query/relational_query.py:694, in _call_join(elements_dict, table, how, match_rows)
    690     raise TypeError(
    691         f"`{match_rows}` is an invalid argument for `match_rows`. Can be either `no`, ``'left'`` or ``'right'``"
    692     )
    693 if how in JoinTypes.__dict__["_member_names_"]:
--> 694     elements_dict, table = JoinTypes[how](elements_dict, table, match_rows)
    695 else:
    696     raise TypeError(f"`{how}` is not a valid type of join.")
File /mnt/repos/spatialdata/src/spatialdata/_core/query/relational_query.py:525, in JoinTypes.__call__(self, *args)
    524 def __call__(self, *args: Any) -> tuple[dict[str, Any], AnnData]:
--> 525     return self.value(*args)
File /mnt/repos/spatialdata/src/spatialdata/_core/query/relational_query.py:416, in _inner_join_spatialelement_table(element_dict, table, match_rows)
    413 if joined_indices is not None:
    414     joined_indices = joined_indices.dropna() if any(joined_indices.isna()) else joined_indices
--> 416 joined_table = table[joined_indices, :].copy() if joined_indices is not None else None
    417 _inplace_fix_subset_categorical_obs(subset_adata=joined_table, original_adata=table)
    418 return element_dict, joined_table
File /mnt/miniconda3/envs/ome/lib/python3.10/site-packages/anndata/_core/anndata.py:1022, in AnnData.__getitem__(self, index)
   1020 def __getitem__(self, index: Index) -> AnnData:
   1021     """Returns a sliced view of the object."""
-> 1022     oidx, vidx = self._normalize_indices(index)
   1023     return AnnData(self, oidx=oidx, vidx=vidx, asview=True)
File /mnt/miniconda3/envs/ome/lib/python3.10/site-packages/anndata/_core/anndata.py:1003, in AnnData._normalize_indices(self, index)
   1002 def _normalize_indices(self, index: Index | None) -> tuple[slice, slice]:
-> 1003     return _normalize_indices(index, self.obs_names, self.var_names)
File /mnt/miniconda3/envs/ome/lib/python3.10/site-packages/anndata/_core/index.py:32, in _normalize_indices(index, names0, names1)
     30     index = tuple(i.values if isinstance(i, pd.Series) else i for i in index)
     31 ax0, ax1 = unpack_index(index)
---> 32 ax0 = _normalize_index(ax0, names0)
     33 ax1 = _normalize_index(ax1, names1)
     34 return ax0, ax1
File /mnt/miniconda3/envs/ome/lib/python3.10/site-packages/anndata/_core/index.py:96, in _normalize_index(indexer, index)
     94     return indexer
     95 else:  # indexer should be string array
---> 96     positions = index.get_indexer(indexer)
     97     if np.any(positions < 0):
     98         not_found = indexer[positions < 0]
File /mnt/miniconda3/envs/ome/lib/python3.10/site-packages/pandas/core/indexes/base.py:3885, in Index.get_indexer(self, target, method, limit, tolerance)
   3882 self._check_indexing_method(method, limit, tolerance)
   3884 if not self._index_as_unique:
-> 3885     raise InvalidIndexError(self._requires_unique_msg)
   3887 if len(target) == 0:
   3888     return np.array([], dtype=np.intp)
InvalidIndexError: Reindexing only valid with uniquely valued Index objects