From ad9fd00dd811f94ddcc798649a2d59daa3672bfd Mon Sep 17 00:00:00 2001 From: hammadb Date: Tue, 17 Sep 2024 13:40:16 -0700 Subject: [PATCH] [BUG] Fix I32Array BW compat --- .../src/arrow/block/value/int32array_value.rs | 38 +++++++++++++++---- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/rust/blockstore/src/arrow/block/value/int32array_value.rs b/rust/blockstore/src/arrow/block/value/int32array_value.rs index da3de68f33f..d165eefe134 100644 --- a/rust/blockstore/src/arrow/block/value/int32array_value.rs +++ b/rust/blockstore/src/arrow/block/value/int32array_value.rs @@ -6,7 +6,7 @@ use crate::{ key::KeyWrapper, }; use arrow::{ - array::{Array, ListArray, UInt32Array}, + array::{Array, Int32Array, ListArray, UInt32Array}, util::bit_util, }; use std::{mem::size_of, sync::Arc}; @@ -50,12 +50,36 @@ impl<'referred_data> ArrowReadableValue<'referred_data> for &'referred_data [u32 let list_array = array.as_any().downcast_ref::().unwrap(); let start = list_array.value_offsets()[index] as usize; let end = list_array.value_offsets()[index + 1] as usize; - let u32array = list_array - .values() - .as_any() - .downcast_ref::() - .unwrap(); - &u32array.values()[start..end] + + // 9/17 In order to support backwards compatability before #2729 (https://github.com/chroma-core/chroma/pull/2729) + // which had this stored as a Int32Array, we first try to downcast to a UInt32Array and then if that fails + // we downcast to a Int32Array, if that fails we panic. + let u32array = list_array.values().as_any().downcast_ref::(); + match u32array { + Some(u32array) => &u32array.values()[start..end], + None => { + let i32array = list_array.values().as_any().downcast_ref::(); + match i32array { + Some(i32array) => { + // &i32array.values()[start..end] as &[u32] + // We are forced to use unsafe here because we are casting a &[i32] to a &[u32] + // this is safe as of 9/17 ONLY because we use exclusively positive values here, + // we should introduce versioning to the blockfile + // to ensure that this sort of behavior is only done when needed. + // (Yes this is not great :( ) + return unsafe { + std::slice::from_raw_parts( + i32array.values()[start..end].as_ptr() as *const u32, + i32array.values()[start..end].len(), + ) + }; + } + None => panic!( + "Expected UInt32Array or Int32Array (for legacy reasons) got neither" + ), + } + } + } } fn add_to_delta(