Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: Int96 and Decimal96 support #21

Merged
merged 2 commits into from
Sep 13, 2023
Merged

feat: Int96 and Decimal96 support #21

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6f2958d
chore: Int96 and Decimal96 support
waralexrom Jul 26, 2023
185bca6
bump rust version in tests
waralexrom Aug 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/miri.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
steps:
- uses: actions/checkout@v2
with:
Expand Down
14 changes: 7 additions & 7 deletions .github/workflows/rust.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
container:
image: ${{ matrix.arch }}/rust
env:
Expand Down Expand Up @@ -73,7 +73,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
container:
image: ${{ matrix.arch }}/rust
env:
Expand Down Expand Up @@ -174,7 +174,7 @@ jobs:
strategy:
matrix:
os: [windows-latest, macos-latest]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
steps:
- uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -202,7 +202,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
container:
image: ${{ matrix.arch }}/rust
env:
Expand Down Expand Up @@ -257,7 +257,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
steps:
- uses: actions/checkout@v2
with:
Expand Down Expand Up @@ -297,7 +297,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
container:
image: ${{ matrix.arch }}/rust
env:
Expand Down Expand Up @@ -341,7 +341,7 @@ jobs:
strategy:
matrix:
arch: [amd64]
rust: [nightly-2021-07-04]
rust: [nightly-2022-06-22]
container:
image: ${{ matrix.arch }}/rust
env:
Expand Down
33 changes: 33 additions & 0 deletions arrow/src/array/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,7 @@ pub fn make_array(data: ArrayData) -> ArrayRef {
DataType::Int16 => Arc::new(Int16Array::from(data)) as ArrayRef,
DataType::Int32 => Arc::new(Int32Array::from(data)) as ArrayRef,
DataType::Int64 => Arc::new(Int64Array::from(data)) as ArrayRef,
DataType::Int96 => Arc::new(Int96Array::from(data)) as ArrayRef,
DataType::Int64Decimal(0) => Arc::new(Int64Decimal0Array::from(data)) as ArrayRef,
DataType::Int64Decimal(1) => Arc::new(Int64Decimal1Array::from(data)) as ArrayRef,
DataType::Int64Decimal(2) => Arc::new(Int64Decimal2Array::from(data)) as ArrayRef,
Expand All @@ -244,6 +245,15 @@ pub fn make_array(data: ArrayData) -> ArrayRef {
DataType::Int64Decimal(10) => {
Arc::new(Int64Decimal10Array::from(data)) as ArrayRef
}
DataType::Int96Decimal(0) => Arc::new(Int96Decimal0Array::from(data)) as ArrayRef,
DataType::Int96Decimal(1) => Arc::new(Int96Decimal1Array::from(data)) as ArrayRef,
DataType::Int96Decimal(2) => Arc::new(Int96Decimal2Array::from(data)) as ArrayRef,
DataType::Int96Decimal(3) => Arc::new(Int96Decimal3Array::from(data)) as ArrayRef,
DataType::Int96Decimal(4) => Arc::new(Int96Decimal4Array::from(data)) as ArrayRef,
DataType::Int96Decimal(5) => Arc::new(Int96Decimal5Array::from(data)) as ArrayRef,
DataType::Int96Decimal(10) => {
Arc::new(Int96Decimal10Array::from(data)) as ArrayRef
}
DataType::UInt8 => Arc::new(UInt8Array::from(data)) as ArrayRef,
DataType::UInt16 => Arc::new(UInt16Array::from(data)) as ArrayRef,
DataType::UInt32 => Arc::new(UInt32Array::from(data)) as ArrayRef,
Expand Down Expand Up @@ -401,6 +411,7 @@ pub fn new_null_array(data_type: &DataType, length: usize) -> ArrayRef {
// expanding this into Date23{unit}Type results in needless branching
DataType::Time32(_) => new_null_sized_array::<Int32Type>(data_type, length),
DataType::Int64 => new_null_sized_array::<Int64Type>(data_type, length),
DataType::Int96 => new_null_sized_array::<Int96Type>(data_type, length),
DataType::Int64Decimal(0) => {
new_null_sized_array::<Int64Decimal0Type>(data_type, length)
}
Expand All @@ -423,6 +434,28 @@ pub fn new_null_array(data_type: &DataType, length: usize) -> ArrayRef {
new_null_sized_array::<Int64Decimal10Type>(data_type, length)
}
DataType::Int64Decimal(_) => panic!("invalid scale"),
DataType::Int96Decimal(0) => {
new_null_sized_array::<Int96Decimal0Type>(data_type, length)
}
DataType::Int96Decimal(1) => {
new_null_sized_array::<Int96Decimal1Type>(data_type, length)
}
DataType::Int96Decimal(2) => {
new_null_sized_array::<Int96Decimal2Type>(data_type, length)
}
DataType::Int96Decimal(3) => {
new_null_sized_array::<Int96Decimal3Type>(data_type, length)
}
DataType::Int96Decimal(4) => {
new_null_sized_array::<Int96Decimal4Type>(data_type, length)
}
DataType::Int96Decimal(5) => {
new_null_sized_array::<Int96Decimal5Type>(data_type, length)
}
DataType::Int96Decimal(10) => {
new_null_sized_array::<Int96Decimal10Type>(data_type, length)
}
DataType::Int96Decimal(_) => panic!("invalid scale"),
DataType::UInt64 => new_null_sized_array::<UInt64Type>(data_type, length),
DataType::Float64 => new_null_sized_array::<Float64Type>(data_type, length),
DataType::Date64 => new_null_sized_array::<Date64Type>(data_type, length),
Expand Down
8 changes: 8 additions & 0 deletions arrow/src/array/array_primitive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,13 +393,21 @@ def_numeric_from_vec!(Int8Type);
def_numeric_from_vec!(Int16Type);
def_numeric_from_vec!(Int32Type);
def_numeric_from_vec!(Int64Type);
def_numeric_from_vec!(Int96Type);
def_numeric_from_vec!(Int64Decimal0Type);
def_numeric_from_vec!(Int64Decimal1Type);
def_numeric_from_vec!(Int64Decimal2Type);
def_numeric_from_vec!(Int64Decimal3Type);
def_numeric_from_vec!(Int64Decimal4Type);
def_numeric_from_vec!(Int64Decimal5Type);
def_numeric_from_vec!(Int64Decimal10Type);
def_numeric_from_vec!(Int96Decimal0Type);
def_numeric_from_vec!(Int96Decimal1Type);
def_numeric_from_vec!(Int96Decimal2Type);
def_numeric_from_vec!(Int96Decimal3Type);
def_numeric_from_vec!(Int96Decimal4Type);
def_numeric_from_vec!(Int96Decimal5Type);
def_numeric_from_vec!(Int96Decimal10Type);
def_numeric_from_vec!(UInt8Type);
def_numeric_from_vec!(UInt16Type);
def_numeric_from_vec!(UInt32Type);
Expand Down
8 changes: 8 additions & 0 deletions arrow/src/array/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1451,13 +1451,21 @@ pub fn make_builder(datatype: &DataType, capacity: usize) -> Box<ArrayBuilder> {
DataType::Int16 => Box::new(Int16Builder::new(capacity)),
DataType::Int32 => Box::new(Int32Builder::new(capacity)),
DataType::Int64 => Box::new(Int64Builder::new(capacity)),
DataType::Int96 => Box::new(Int96Builder::new(capacity)),
DataType::Int64Decimal(0) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int64Decimal(1) => Box::new(Int64Decimal1Builder::new(capacity)),
DataType::Int64Decimal(2) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int64Decimal(3) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int64Decimal(4) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int64Decimal(5) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int64Decimal(10) => Box::new(Int64Decimal0Builder::new(capacity)),
DataType::Int96Decimal(0) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::Int96Decimal(1) => Box::new(Int96Decimal1Builder::new(capacity)),
DataType::Int96Decimal(2) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::Int96Decimal(3) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::Int96Decimal(4) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::Int96Decimal(5) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::Int96Decimal(10) => Box::new(Int96Decimal0Builder::new(capacity)),
DataType::UInt8 => Box::new(UInt8Builder::new(capacity)),
DataType::UInt16 => Box::new(UInt16Builder::new(capacity)),
DataType::UInt32 => Box::new(UInt32Builder::new(capacity)),
Expand Down
10 changes: 10 additions & 0 deletions arrow/src/array/data.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ pub(crate) fn new_buffers(data_type: &DataType, capacity: usize) -> [MutableBuff
MutableBuffer::new(capacity * mem::size_of::<u64>()),
empty_buffer,
],
DataType::Int96Decimal(_) => [
MutableBuffer::new(capacity * mem::size_of::<u128>()),
empty_buffer,
],
DataType::Int8 => [
MutableBuffer::new(capacity * mem::size_of::<i8>()),
empty_buffer,
Expand All @@ -91,6 +95,10 @@ pub(crate) fn new_buffers(data_type: &DataType, capacity: usize) -> [MutableBuff
MutableBuffer::new(capacity * mem::size_of::<i64>()),
empty_buffer,
],
DataType::Int96 => [
MutableBuffer::new(capacity * mem::size_of::<i128>()),
empty_buffer,
],
DataType::Float32 => [
MutableBuffer::new(capacity * mem::size_of::<f32>()),
empty_buffer,
Expand Down Expand Up @@ -430,7 +438,9 @@ impl ArrayData {
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::Int96
| DataType::Int64Decimal(_)
| DataType::Int96Decimal(_)
| DataType::Float32
| DataType::Float64
| DataType::Date32
Expand Down
6 changes: 6 additions & 0 deletions arrow/src/array/equal/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,9 +175,15 @@ fn equal_values(
DataType::Int64 => primitive_equal::<i64>(
lhs, rhs, lhs_nulls, rhs_nulls, lhs_start, rhs_start, len,
),
DataType::Int96 => primitive_equal::<i128>(
lhs, rhs, lhs_nulls, rhs_nulls, lhs_start, rhs_start, len,
),
DataType::Int64Decimal(_) => primitive_equal::<i64>(
lhs, rhs, lhs_nulls, rhs_nulls, lhs_start, rhs_start, len,
),
DataType::Int96Decimal(_) => primitive_equal::<i64>(
lhs, rhs, lhs_nulls, rhs_nulls, lhs_start, rhs_start, len,
),
DataType::Float32 => primitive_equal::<f32>(
lhs, rhs, lhs_nulls, rhs_nulls, lhs_start, rhs_start, len,
),
Expand Down
23 changes: 23 additions & 0 deletions arrow/src/array/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,13 +137,21 @@ pub type Int8Array = PrimitiveArray<Int8Type>;
pub type Int16Array = PrimitiveArray<Int16Type>;
pub type Int32Array = PrimitiveArray<Int32Type>;
pub type Int64Array = PrimitiveArray<Int64Type>;
pub type Int96Array = PrimitiveArray<Int96Type>;
pub type Int64Decimal0Array = PrimitiveArray<Int64Decimal0Type>;
pub type Int64Decimal1Array = PrimitiveArray<Int64Decimal1Type>;
pub type Int64Decimal2Array = PrimitiveArray<Int64Decimal2Type>;
pub type Int64Decimal3Array = PrimitiveArray<Int64Decimal3Type>;
pub type Int64Decimal4Array = PrimitiveArray<Int64Decimal4Type>;
pub type Int64Decimal5Array = PrimitiveArray<Int64Decimal5Type>;
pub type Int64Decimal10Array = PrimitiveArray<Int64Decimal10Type>;
pub type Int96Decimal0Array = PrimitiveArray<Int96Decimal0Type>;
pub type Int96Decimal1Array = PrimitiveArray<Int96Decimal1Type>;
pub type Int96Decimal2Array = PrimitiveArray<Int96Decimal2Type>;
pub type Int96Decimal3Array = PrimitiveArray<Int96Decimal3Type>;
pub type Int96Decimal4Array = PrimitiveArray<Int96Decimal4Type>;
pub type Int96Decimal5Array = PrimitiveArray<Int96Decimal5Type>;
pub type Int96Decimal10Array = PrimitiveArray<Int96Decimal10Type>;
pub type UInt8Array = PrimitiveArray<UInt8Type>;
pub type UInt16Array = PrimitiveArray<UInt16Type>;
pub type UInt32Array = PrimitiveArray<UInt32Type>;
Expand Down Expand Up @@ -200,6 +208,13 @@ pub type Int64Decimal3BufferBuilder = BufferBuilder<Int64Decimal3Type>;
pub type Int64Decimal4BufferBuilder = BufferBuilder<Int64Decimal4Type>;
pub type Int64Decimal5BufferBuilder = BufferBuilder<Int64Decimal5Type>;
pub type Int64Decimal10BufferBuilder = BufferBuilder<Int64Decimal10Type>;
pub type Int96Decimal0BufferBuilder = BufferBuilder<Int96Decimal0Type>;
pub type Int96Decimal1BufferBuilder = BufferBuilder<Int96Decimal1Type>;
pub type Int96Decimal2BufferBuilder = BufferBuilder<Int96Decimal2Type>;
pub type Int96Decimal3BufferBuilder = BufferBuilder<Int96Decimal3Type>;
pub type Int96Decimal4BufferBuilder = BufferBuilder<Int96Decimal4Type>;
pub type Int96Decimal5BufferBuilder = BufferBuilder<Int96Decimal5Type>;
pub type Int96Decimal10BufferBuilder = BufferBuilder<Int96Decimal10Type>;
pub type UInt8BufferBuilder = BufferBuilder<u8>;
pub type UInt16BufferBuilder = BufferBuilder<u16>;
pub type UInt32BufferBuilder = BufferBuilder<u32>;
Expand Down Expand Up @@ -246,13 +261,21 @@ pub type Int8Builder = PrimitiveBuilder<Int8Type>;
pub type Int16Builder = PrimitiveBuilder<Int16Type>;
pub type Int32Builder = PrimitiveBuilder<Int32Type>;
pub type Int64Builder = PrimitiveBuilder<Int64Type>;
pub type Int96Builder = PrimitiveBuilder<Int96Type>;
pub type Int64Decimal0Builder = PrimitiveBuilder<Int64Decimal0Type>;
pub type Int64Decimal1Builder = PrimitiveBuilder<Int64Decimal1Type>;
pub type Int64Decimal2Builder = PrimitiveBuilder<Int64Decimal2Type>;
pub type Int64Decimal3Builder = PrimitiveBuilder<Int64Decimal3Type>;
pub type Int64Decimal4Builder = PrimitiveBuilder<Int64Decimal4Type>;
pub type Int64Decimal5Builder = PrimitiveBuilder<Int64Decimal5Type>;
pub type Int64Decimal10Builder = PrimitiveBuilder<Int64Decimal10Type>;
pub type Int96Decimal0Builder = PrimitiveBuilder<Int96Decimal0Type>;
pub type Int96Decimal1Builder = PrimitiveBuilder<Int96Decimal1Type>;
pub type Int96Decimal2Builder = PrimitiveBuilder<Int96Decimal2Type>;
pub type Int96Decimal3Builder = PrimitiveBuilder<Int96Decimal3Type>;
pub type Int96Decimal4Builder = PrimitiveBuilder<Int96Decimal4Type>;
pub type Int96Decimal5Builder = PrimitiveBuilder<Int96Decimal5Type>;
pub type Int96Decimal10Builder = PrimitiveBuilder<Int96Decimal10Type>;
pub type UInt8Builder = PrimitiveBuilder<UInt8Type>;
pub type UInt16Builder = PrimitiveBuilder<UInt16Type>;
pub type UInt32Builder = PrimitiveBuilder<UInt32Type>;
Expand Down
37 changes: 32 additions & 5 deletions arrow/src/array/transform/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,9 @@ fn build_extend(array: &ArrayData) -> Extend {
DataType::Int16 => primitive::build_extend::<i16>(array),
DataType::Int32 => primitive::build_extend::<i32>(array),
DataType::Int64 => primitive::build_extend::<i64>(array),
DataType::Int96 => primitive::build_extend::<i128>(array),
DataType::Int64Decimal(_) => primitive::build_extend::<i64>(array),
DataType::Int96Decimal(_) => primitive::build_extend::<i128>(array),
DataType::Float32 => primitive::build_extend::<f32>(array),
DataType::Float64 => primitive::build_extend::<f64>(array),
DataType::Date32
Expand Down Expand Up @@ -291,7 +293,9 @@ fn build_extend_nulls(data_type: &DataType) -> ExtendNulls {
DataType::Int16 => primitive::extend_nulls::<i16>,
DataType::Int32 => primitive::extend_nulls::<i32>,
DataType::Int64 => primitive::extend_nulls::<i64>,
DataType::Int96 => primitive::extend_nulls::<i128>,
DataType::Int64Decimal(_) => primitive::extend_nulls::<i64>,
DataType::Int96Decimal(_) => primitive::extend_nulls::<i128>,
DataType::Float32 => primitive::extend_nulls::<f32>,
DataType::Float64 => primitive::extend_nulls::<f64>,
DataType::Date32
Expand Down Expand Up @@ -434,7 +438,9 @@ impl<'a> MutableArrayData<'a> {
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::Int96
| DataType::Int64Decimal(_)
| DataType::Int96Decimal(_)
| DataType::Float32
| DataType::Float64
| DataType::Date32
Expand Down Expand Up @@ -616,7 +622,7 @@ impl<'a> MutableArrayData<'a> {
pub fn extend_nulls(&mut self, len: usize) {
self.data.null_count += len;

let null_bytes_count = bit_util::ceil(self.data.len + len, 8);
let null_bytes_count = bit_util::ceil(self.data.len + len, 8);
if null_bytes_count > self.data.null_buffer.len() {
self.data.null_buffer.resize(null_bytes_count, 0x00);
}
Expand Down Expand Up @@ -744,7 +750,17 @@ mod tests {
let result = a.freeze();
let array = UInt8Array::from(result);
assert_eq!(array.data().null_buffer().unwrap().len(), 2);
let expected = UInt8Array::from(vec![Some(1), Some(2), Some(3), None, None, None, None, None, None]);
let expected = UInt8Array::from(vec![
Some(1),
Some(2),
Some(3),
None,
None,
None,
None,
None,
None,
]);
assert_eq!(array, expected);

let b = UInt8Array::from(vec![Some(1), Some(2), Some(3)]);
Expand All @@ -759,12 +775,23 @@ mod tests {
let result = a.freeze();
let array = UInt8Array::from(result);
assert_eq!(array.data().null_buffer().unwrap().len(), 2);
let expected = UInt8Array::from(vec![Some(1), Some(2), Some(3), None, None, None, None, None, None, Some(1), Some(2), Some(3)]);
let expected = UInt8Array::from(vec![
Some(1),
Some(2),
Some(3),
None,
None,
None,
None,
None,
None,
Some(1),
Some(2),
Some(3),
]);
assert_eq!(array, expected);

}


#[test]
fn test_list_null_offset() -> Result<()> {
let int_builder = Int64Builder::new(24);
Expand Down
Loading
Loading