Skip to content

Commit

Permalink
Minor refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
Jefffrey committed Sep 28, 2024
1 parent 9219071 commit 0f848c3
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 15 deletions.
10 changes: 0 additions & 10 deletions src/array_decoder/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@
use std::sync::Arc;

use arrow::array::{ArrayRef, BooleanArray, PrimitiveArray};
use arrow::buffer::NullBuffer;
use arrow::datatypes::ArrowNativeTypeOp;
use arrow::datatypes::{ArrowPrimitiveType, Decimal128Type};
use arrow::datatypes::{DataType as ArrowDataType, Field};
Expand Down Expand Up @@ -221,15 +220,6 @@ fn derive_present_vec(
}
}

fn create_null_buffer(present: Option<Vec<bool>>) -> Option<NullBuffer> {
match present {
// Edge case where keys of map cannot have a null buffer
Some(present) if present.iter().all(|&p| p) => None,
Some(present) => Some(NullBuffer::from(present)),
None => None,
}
}

pub struct NaiveStripeDecoder {
stripe: Stripe,
schema_ref: SchemaRef,
Expand Down
16 changes: 11 additions & 5 deletions src/array_decoder/string.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,12 @@ use std::marker::PhantomData;
use std::sync::Arc;

use arrow::array::{ArrayRef, DictionaryArray, GenericByteArray, StringArray};
use arrow::buffer::{Buffer, OffsetBuffer};
use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer};
use arrow::compute::kernels::cast;
use arrow::datatypes::{ByteArrayType, DataType, GenericBinaryType, GenericStringType};
use snafu::ResultExt;

use crate::array_decoder::{create_null_buffer, derive_present_vec};
use crate::array_decoder::derive_present_vec;
use crate::column::{get_present_vec, Column};
use crate::compression::Decompressor;
use crate::encoding::{get_unsigned_rle_reader, PrimitiveValueDecoder};
Expand Down Expand Up @@ -117,11 +117,18 @@ impl<T: ByteArrayType> GenericByteArrayDecoder<T> {
let present = derive_present_vec(&mut self.present, parent_present, batch_size);

let mut lengths = vec![0; batch_size];
if let Some(present) = &present {
let null_buffer = if let Some(present) = &present {
self.lengths.decode_spaced(&mut lengths, present)?;
if present.iter().all(|&p| p) {
// Edge case where keys of map cannot have a null buffer
None
} else {
Some(NullBuffer::from(present.as_slice()))
}
} else {
self.lengths.decode(&mut lengths)?;
}
None
};
let total_length: i64 = lengths.iter().sum();
// Fetch all data bytes at once
let mut bytes = Vec::with_capacity(total_length as usize);
Expand All @@ -133,7 +140,6 @@ impl<T: ByteArrayType> GenericByteArrayDecoder<T> {
let bytes = Buffer::from(bytes);
let offsets =
OffsetBuffer::<T::Offset>::from_lengths(lengths.into_iter().map(|l| l as usize));
let null_buffer = create_null_buffer(present);

let array =
GenericByteArray::<T>::try_new(offsets, bytes, null_buffer).context(ArrowSnafu)?;
Expand Down

0 comments on commit 0f848c3

Please sign in to comment.