Skip to content

Commit

Permalink
refactor(interactive): revert encoding of groot (#3471)
Browse files Browse the repository at this point in the history 
As titled.
  • Loading branch information
@longbinlai
longbinlai authored Jan 5, 2024
1 parent faa7230 commit f40ee9a
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 22 deletions.
38 changes: 19 additions & 19 deletions interactive_engine/executor/store/groot/src/db/api/property.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use super::error::*;
use super::GraphResult;
use crate::db::api::PropertyId;
use crate::db::common::bytes::transform;
use crate::db::common::bytes::util::{UnsafeBytesReader, UnsafeBytesWriter, LEN32_SIZE, LEN_SIZE};
use crate::db::common::bytes::util::{UnsafeBytesReader, UnsafeBytesWriter, LEN_SIZE};
use crate::db::common::numeric::*;
use crate::db::proto::schema_common::{DataTypePb, PropertyValuePb};

Expand Down Expand Up @@ -298,7 +298,7 @@ impl<'a> ValueRef<'a> {
ValueType::LongList | ValueType::DoubleList => ::std::mem::size_of::<u64>(),
_ => unreachable!(),
};
if reader.read_u64(0).to_be() as usize * unit_size + LEN_SIZE == self.data.len() {
if reader.read_u32(0).to_be() as usize * unit_size + LEN_SIZE == self.data.len() {
return Ok(reader);
}
let msg = format!("invalid {:?} bytes, data len is {}", value_type, self.data.len());
Expand All @@ -310,11 +310,11 @@ impl<'a> ValueRef<'a> {
/// in valid utf8 and when user extract the str, the process will be panic
fn weak_check_str_list(&self) -> GraphResult<UnsafeBytesReader> {
let reader = UnsafeBytesReader::new(self.data);
let len = reader.read_u64(0).to_be() as usize;
let len = reader.read_u32(0).to_be() as usize;
let total_len = reader
.read_u32(LEN_SIZE + LEN32_SIZE * (len - 1))
.read_u32(LEN_SIZE + LEN_SIZE * (len - 1))
.to_be() as usize;
if total_len == self.data.len() - (LEN_SIZE + LEN32_SIZE * len) {
if total_len == self.data.len() - (LEN_SIZE + LEN_SIZE * len) {
return Ok(reader);
}
let msg = format!("invalid str array bytes");
Expand Down Expand Up @@ -524,31 +524,31 @@ fn get_double(data: &[u8]) -> f64 {
/// +-----+----+----+-----+----+
/// | len | x1 | x2 | ... | xn |
/// +-----+----+----+-----+----+
/// | 8B | 4B | 4B | ... | 4B |
/// | 4B | 4B | 4B | ... | 4B |
/// +-----+----+----+-----+----+ len and every xi is in int format above
/// long array:
/// +-----+----+----+-----+----+
/// | len | x1 | x2 | ... | xn |
/// +-----+----+----+-----+----+
/// | 8B | 8B | 8B | ... | 8B |
/// | 4B | 8B | 8B | ... | 8B |
/// +-----+----+----+-----+----+ len is in int format above and every xi is in long format above
/// float array:
/// +-----+----+----+-----+----+
/// | len | x1 | x2 | ... | xn |
/// +-----+----+----+-----+----+
/// | 8B | 4B | 4B | ... | 4B |
/// | 4B | 4B | 4B | ... | 4B |
/// +-----+----+----+-----+----+ len is in int format above and every xi is in float format above
/// double array:
/// +-----+----+----+-----+----+
/// | len | x1 | x2 | ... | xn |
/// +-----+----+----+-----+----+
/// | 8B | 8B | 8B | ... | 8B |
/// | 4B | 8B | 8B | ... | 8B |
/// +-----+----+----+-----+----+ len is in int format above and every xi is in double format above
/// string array:
/// +-----+------+------+-----+------+------+------+-----+------+
/// | len | off1 | off2 | ... | offn | str1 | str2 | ... | strn |
/// +-----+------+------+-----+------+------+------+-----+------+
/// | 8B | 4B | 4B | ... | 4B | x1 B | x2 B | ... | xn B |
/// | 4B | 4B | 4B | ... | 4B | x1 B | x2 B | ... | xn B |
/// +-----+------+------+-----+------+------+------+-----+------+
/// len and offi is in int format above, stri is in string format above
/// off1 == x1 means it's str1's end offset
Expand All @@ -570,7 +570,7 @@ macro_rules! gen_array {
data.set_len(total_len);
}
let mut writer = UnsafeBytesWriter::new(&mut data);
writer.write_u64(0, ($arr.len() as u64).to_be());
writer.write_u32(0, ($arr.len() as u32).to_be());
for i in 0..$arr.len() {
writer.$func(LEN_SIZE + size * i, $arr[i].to_big_endian());
}
Expand Down Expand Up @@ -663,7 +663,7 @@ impl Value {
}

pub fn string_list(v: &[String]) -> Self {
let mut size = LEN_SIZE + LEN32_SIZE * v.len();
let mut size = LEN_SIZE + LEN_SIZE * v.len();
for s in v {
size += s.len();
}
Expand All @@ -672,13 +672,13 @@ impl Value {
data.set_len(size);
}
let mut writer = UnsafeBytesWriter::new(&mut data);
writer.write_u64(0, (v.len() as u64).to_be());
writer.write_u32(0, (v.len() as u32).to_be());
let mut off = 0;
let mut pos = LEN_SIZE;
for s in v {
off += s.len() as u32;
writer.write_u32(pos, off.to_be());
pos += LEN32_SIZE;
pos += LEN_SIZE;
}
for s in v {
writer.write_bytes(pos, s.as_bytes());
Expand Down Expand Up @@ -876,7 +876,7 @@ impl<'a, T> NumericArray<'a, T> {

impl<'a, T: ToBigEndian> NumericArray<'a, T> {
fn new(reader: UnsafeBytesReader<'a>) -> Self {
let len = reader.read_u64(0).to_be() as usize;
let len = reader.read_u32(0).to_be() as usize;
NumericArray { reader, len, _phantom: Default::default() }
}

Expand Down Expand Up @@ -980,23 +980,23 @@ pub struct StrArray<'a> {

impl<'a> StrArray<'a> {
fn new(reader: UnsafeBytesReader<'a>) -> Self {
let len = reader.read_u64(0).to_be() as usize;
let len = reader.read_u32(0).to_be() as usize;
StrArray { reader, len }
}

pub fn get(&self, idx: usize) -> Option<&str> {
if idx < self.len {
let str_start_off = LEN_SIZE + LEN32_SIZE * self.len;
let str_start_off = LEN_SIZE + LEN_SIZE * self.len;
let start_off = if idx == 0 {
0
} else {
self.reader
.read_u32(LEN_SIZE + LEN32_SIZE * (idx - 1))
.read_u32(LEN_SIZE + LEN_SIZE * (idx - 1))
.to_be() as usize
};
let end_off = self
.reader
.read_u32(LEN_SIZE + LEN32_SIZE * idx)
.read_u32(LEN_SIZE + LEN_SIZE * idx)
.to_be() as usize;
let len = end_off - start_off;
let offset = str_start_off + start_off;
Expand Down
4 changes: 1 addition & 3 deletions interactive_engine/executor/store/groot/src/db/common/bytes/util.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,7 @@ use crate::db::api::GraphErrorCode::InvalidData;
use crate::db::api::{GraphError, GraphResult};

/// define the size that a length field takes in encoding an array
pub const LEN_SIZE: usize = ::std::mem::size_of::<u64>();
/// half of LEN_SIZE
pub const LEN32_SIZE: usize = ::std::mem::size_of::<u32>();
pub const LEN_SIZE: usize = ::std::mem::size_of::<u32>();

/// This reader won't check whether the offset is overflow when read bytes.
/// It's for performance purpose. Be careful to use it.
Expand Down

0 comments on commit f40ee9a

Please sign in to comment.