diff --git a/crates/polars-core/src/chunked_array/logical/categorical/mod.rs b/crates/polars-core/src/chunked_array/logical/categorical/mod.rs
index 7eaec38f7201..7b286e476e09 100644
--- a/crates/polars-core/src/chunked_array/logical/categorical/mod.rs
+++ b/crates/polars-core/src/chunked_array/logical/categorical/mod.rs
@@ -361,20 +361,16 @@ impl LogicalType for CategoricalChunked {
                     StringChunked::with_chunk("", self.get_rev_map().get_categories().clone());
                 let casted_series = categories.cast(dtype)?;
 
-                macro_rules! get_elements {
-                    ($ca:expr) => {{
-                        Ok(self
-                            .physical()
-                            .into_iter()
-                            .map(|opt_el| {
-                                opt_el.map(|el: u32| unsafe {
-                                    $ca.get_unchecked(el as usize).unwrap()
-                                })
-                            })
-                            .collect())
-                    }};
+                #[cfg(feature = "bigidx")]
+                {
+                    let s = self.physical.cast(&DataType::UInt64)?;
+                    Ok(unsafe { casted_series.take_unchecked(s.u64()?) })
+                }
+                #[cfg(not(feature = "bigidx"))]
+                {
+                    // Safety: Invariant of categorical means indices are in bound
+                    Ok(unsafe { casted_series.take_unchecked(&self.physical) })
                 }
-                downcast_as_macro_arg_physical!(casted_series, get_elements)
             },
             _ => self.physical.cast(dtype),
         }