From 247ac349a286dffca931e732f7a369eac02f6166 Mon Sep 17 00:00:00 2001 From: Zack Slayton Date: Tue, 20 Jun 2023 16:32:06 -0400 Subject: [PATCH] Adds `"experimental-reader"` feature (#577) This PR is the first of a short series that will move the streaming types behind an experimental feature. It is part of issue #576. In this PR, we move the public re-exports of: * `IonReader` * `Reader`/`StreamItem` * `SystemReader`/`SystemStreamItem` * `RawReader`/`RawStreamItem` * `BlockingRawReader` behind the feature flag `"experimental-reader"`. No behavior is changed, and internal modules can continue to access these types as desired. --- Cargo.toml | 6 +- examples/read_all_values.rs | 309 ++++++++++--------- src/binary/binary_writer.rs | 4 +- src/binary/non_blocking/raw_binary_reader.rs | 5 +- src/binary/raw_binary_writer.rs | 6 +- src/binary/timestamp.rs | 6 +- src/blocking_reader.rs | 2 +- src/element/element_stream_reader.rs | 8 +- src/element/mod.rs | 5 +- src/element/reader.rs | 4 +- src/{stream_reader.rs => ion_reader.rs} | 0 src/lib.rs | 34 +- src/raw_reader.rs | 2 +- src/reader.rs | 18 +- src/system_reader.rs | 3 +- src/text/non_blocking/raw_text_reader.rs | 6 +- src/text/text_writer.rs | 5 +- tests/element_test_vectors.rs | 3 +- tests/good_test_vectors.rs | 176 ----------- 19 files changed, 226 insertions(+), 376 deletions(-) rename src/{stream_reader.rs => ion_reader.rs} (100%) delete mode 100644 tests/good_test_vectors.rs diff --git a/Cargo.toml b/Cargo.toml index 9a05ebb3..617d6348 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,11 +34,15 @@ ion-hash = ["digest"] # Feature for indicating particularly bleeding edge APIs or functionality in the library. # These are not guaranteed any sort of API stability and may also have non-standard # Ion behavior (e.g., draft Ion 1.1 capabilities). -experimental = ["experimental-streaming", "experimental-lazy-reader"] +experimental = ["experimental-streaming", "experimental-reader", "experimental-lazy-reader"] # Experimental streaming APIs experimental-streaming = [] +# Access to the `IonReader` trait, its implementations, and the `ReaderBuilder` requires enabling this feature. +# These APIs are functional and well-tested, but are not yet stable. +experimental-reader = [] + # A reader API that parks on a top-level value and yields 'lazy' handles to the data # contained therein. These lazy handles can be read multiple times, allowing for # an inexpensive form of 'rewind' functionality. diff --git a/examples/read_all_values.rs b/examples/read_all_values.rs index 57ec51d4..bd489056 100644 --- a/examples/read_all_values.rs +++ b/examples/read_all_values.rs @@ -1,166 +1,179 @@ -use ion_rs::binary::non_blocking::raw_binary_reader::RawBinaryReader; -use ion_rs::raw_reader::RawStreamItem; -use ion_rs::result::IonResult; -use ion_rs::{BlockingRawBinaryReader, IonReader, IonType, RawReader, StreamItem, UserReader}; -use memmap::MmapOptions; -use std::fs::File; -use std::process::exit; +#[cfg(not(feature = "experimental-reader"))] +fn main() { + println!("This example requires the 'experimental-reader' feature to work."); +} -fn main() -> IonResult<()> { - let args: Vec = std::env::args().collect(); - let mode = args.get(1).unwrap_or_else(|| { - eprintln!( - "USAGE:\n\n {} [raw-blocking|raw-nonblocking|user] [Binary Ion file]\n", - args.get(0).unwrap() - ); - eprintln!("No mode was specified."); - exit(1); - }); - let path = args.get(2).unwrap_or_else(|| { - eprintln!( - "USAGE:\n\n {} [raw-blocking|raw-nonblocking|user] [Binary Ion file]\n", - args.get(0).unwrap() - ); - eprintln!("No input file was specified."); - exit(2); - }); - let file = File::open(path).unwrap(); +#[cfg(feature = "experimental-reader")] +fn main() { + example::main().unwrap(); +} - // This example uses `mmap` so we can use either the blocking reader (which reads from an - // io::BufRead) or the non-blocking reader (which reads from an AsRef<[u8]>). - let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; +#[cfg(feature = "experimental-reader")] +mod example { + use ion_rs::binary::non_blocking::raw_binary_reader::RawBinaryReader; + use ion_rs::result::IonResult; + use ion_rs::RawStreamItem; + use ion_rs::{BlockingRawBinaryReader, IonReader, IonType, RawReader, StreamItem, UserReader}; + use memmap::MmapOptions; + use std::fs::File; + use std::process::exit; - // Treat the mmap as a byte array. - let ion_data: &[u8] = &mmap[..]; + pub fn main() -> IonResult<()> { + let args: Vec = std::env::args().collect(); + let mode = args.get(1).unwrap_or_else(|| { + eprintln!( + "USAGE:\n\n {} [raw-blocking|raw-nonblocking|user] [Binary Ion file]\n", + args.get(0).unwrap() + ); + eprintln!("No mode was specified."); + exit(1); + }); + let path = args.get(2).unwrap_or_else(|| { + eprintln!( + "USAGE:\n\n {} [raw-blocking|raw-nonblocking|user] [Binary Ion file]\n", + args.get(0).unwrap() + ); + eprintln!("No input file was specified."); + exit(2); + }); + let file = File::open(path).unwrap(); - if mode == "raw-blocking" { - let mut reader = BlockingRawBinaryReader::new(ion_data)?; - let number_of_values = read_all_values_raw(&mut reader)?; - println!("Blocking: read {} values", number_of_values); - } else if mode == "raw-nonblocking" { - let mut reader = RawBinaryReader::new(ion_data); - let number_of_values = read_all_values_raw(&mut reader)?; - println!("Non-blocking: read {} values", number_of_values); - } else if mode == "user" { - let raw_reader = RawBinaryReader::new(ion_data); - let mut reader = UserReader::new(raw_reader); - let number_of_values = read_all_values(&mut reader)?; - println!("Non-blocking: read {} values", number_of_values); - } else { - eprintln!("Unsupported `mode`: {}.", mode); - exit(3); - } + // This example uses `mmap` so we can use either the blocking reader (which reads from an + // io::BufRead) or the non-blocking reader (which reads from an AsRef<[u8]>). + let mmap = unsafe { MmapOptions::new().map(&file).unwrap() }; - Ok(()) -} + // Treat the mmap as a byte array. + let ion_data: &[u8] = &mmap[..]; -// Visits each value in the stream recursively, reading each scalar into a native Rust type. -// Prints the total number of values read upon completion. -fn read_all_values_raw(reader: &mut R) -> IonResult { - use IonType::*; - use RawStreamItem::{Nothing, Null as NullValue, Value, VersionMarker}; - let mut count: usize = 0; - loop { - match reader.next()? { - VersionMarker(_major, _minor) => {} - NullValue(_ion_type) => { - count += 1; - continue; - } - Value(ion_type) => { - count += 1; - match ion_type { - Struct | List | SExp => reader.step_in()?, - String => { - let _string = reader.read_str()?; - } - Symbol => { - let _symbol_id = reader.read_symbol()?; - } - Int => { - let _int = reader.read_i64()?; - } - Float => { - let _float = reader.read_f64()?; - } - Decimal => { - let _decimal = reader.read_decimal()?; - } - Timestamp => { - let _timestamp = reader.read_timestamp()?; - } - Bool => { - let _boolean = reader.read_bool()?; - } - Blob => { - let _blob = reader.read_blob()?; - } - Clob => { - let _clob = reader.read_clob()?; + if mode == "raw-blocking" { + let mut reader = BlockingRawBinaryReader::new(ion_data)?; + let number_of_values = read_all_values_raw(&mut reader)?; + println!("Blocking: read {} values", number_of_values); + } else if mode == "raw-nonblocking" { + let mut reader = RawBinaryReader::new(ion_data); + let number_of_values = read_all_values_raw(&mut reader)?; + println!("Non-blocking: read {} values", number_of_values); + } else if mode == "user" { + let raw_reader = RawBinaryReader::new(ion_data); + let mut reader = UserReader::new(raw_reader); + let number_of_values = read_all_values(&mut reader)?; + println!("Non-blocking: read {} values", number_of_values); + } else { + eprintln!("Unsupported `mode`: {}.", mode); + exit(3); + } + + Ok(()) + } + + // Visits each value in the stream recursively, reading each scalar into a native Rust type. + // Prints the total number of values read upon completion. + fn read_all_values_raw(reader: &mut R) -> IonResult { + use IonType::*; + use RawStreamItem::{Nothing, Null as NullValue, Value, VersionMarker}; + let mut count: usize = 0; + loop { + match reader.next()? { + VersionMarker(_major, _minor) => {} + NullValue(_ion_type) => { + count += 1; + continue; + } + Value(ion_type) => { + count += 1; + match ion_type { + Struct | List | SExp => reader.step_in()?, + String => { + let _string = reader.read_str()?; + } + Symbol => { + let _symbol_id = reader.read_symbol()?; + } + Int => { + let _int = reader.read_i64()?; + } + Float => { + let _float = reader.read_f64()?; + } + Decimal => { + let _decimal = reader.read_decimal()?; + } + Timestamp => { + let _timestamp = reader.read_timestamp()?; + } + Bool => { + let _boolean = reader.read_bool()?; + } + Blob => { + let _blob = reader.read_blob()?; + } + Clob => { + let _clob = reader.read_clob()?; + } + Null => {} } - Null => {} } + Nothing if reader.depth() > 0 => { + reader.step_out()?; + } + _ => break, } - Nothing if reader.depth() > 0 => { - reader.step_out()?; - } - _ => break, } + Ok(count) } - Ok(count) -} -// Visits each value in the stream recursively, reading each scalar into a native Rust type. -// Prints the total number of values read upon completion. -fn read_all_values>(reader: &mut R) -> IonResult { - use IonType::*; - use StreamItem::{Nothing, Null as NullValue, Value}; - let mut count: usize = 0; - loop { - match reader.next()? { - NullValue(_ion_type) => { - count += 1; - continue; - } - Value(ion_type) => { - count += 1; - match ion_type { - Struct | List | SExp => reader.step_in()?, - String => { - let _string = reader.read_str()?; - } - Symbol => { - let _symbol_id = reader.read_symbol()?; - } - Int => { - let _int = reader.read_i64()?; - } - Float => { - let _float = reader.read_f64()?; - } - Decimal => { - let _decimal = reader.read_decimal()?; - } - Timestamp => { - let _timestamp = reader.read_timestamp()?; - } - Bool => { - let _boolean = reader.read_bool()?; - } - Blob => { - let _blob = reader.read_blob()?; - } - Clob => { - let _clob = reader.read_clob()?; + // Visits each value in the stream recursively, reading each scalar into a native Rust type. + // Prints the total number of values read upon completion. + fn read_all_values>(reader: &mut R) -> IonResult { + use IonType::*; + use StreamItem::{Nothing, Null as NullValue, Value}; + let mut count: usize = 0; + loop { + match reader.next()? { + NullValue(_ion_type) => { + count += 1; + continue; + } + Value(ion_type) => { + count += 1; + match ion_type { + Struct | List | SExp => reader.step_in()?, + String => { + let _string = reader.read_str()?; + } + Symbol => { + let _symbol_id = reader.read_symbol()?; + } + Int => { + let _int = reader.read_i64()?; + } + Float => { + let _float = reader.read_f64()?; + } + Decimal => { + let _decimal = reader.read_decimal()?; + } + Timestamp => { + let _timestamp = reader.read_timestamp()?; + } + Bool => { + let _boolean = reader.read_bool()?; + } + Blob => { + let _blob = reader.read_blob()?; + } + Clob => { + let _clob = reader.read_clob()?; + } + Null => {} } - Null => {} } + Nothing if reader.depth() > 0 => { + reader.step_out()?; + } + _ => break, } - Nothing if reader.depth() > 0 => { - reader.step_out()?; - } - _ => break, } + Ok(count) } - Ok(count) } diff --git a/src/binary/binary_writer.rs b/src/binary/binary_writer.rs index 1f1f1ceb..c1e7b216 100644 --- a/src/binary/binary_writer.rs +++ b/src/binary/binary_writer.rs @@ -201,10 +201,10 @@ impl IonWriter for BinaryWriter { #[cfg(test)] mod tests { use super::*; + use crate::ion_reader::IonReader; use crate::reader::ReaderBuilder; - use crate::stream_reader::IonReader; - use crate::StreamItem::Value; + use crate::reader::StreamItem::Value; #[test] fn intern_field_names() -> IonResult<()> { diff --git a/src/binary/non_blocking/raw_binary_reader.rs b/src/binary/non_blocking/raw_binary_reader.rs index a0757ee5..8db590ed 100644 --- a/src/binary/non_blocking/raw_binary_reader.rs +++ b/src/binary/non_blocking/raw_binary_reader.rs @@ -5,13 +5,14 @@ use crate::binary::non_blocking::type_descriptor::{Header, TypeDescriptor}; use crate::binary::uint::DecodedUInt; use crate::binary::var_uint::VarUInt; use crate::binary::IonTypeCode; -use crate::raw_reader::{BufferedRawReader, Expandable}; +use crate::ion_reader::IonReader; +use crate::raw_reader::{BufferedRawReader, Expandable, RawStreamItem}; use crate::result::{ decoding_error, decoding_error_raw, illegal_operation, illegal_operation_raw, incomplete_data_error, }; use crate::types::{Blob, Clob, Decimal, IntAccess, Str, SymbolId}; -use crate::{Int, IonReader, IonResult, IonType, RawStreamItem, RawSymbolToken, Timestamp}; +use crate::{Int, IonResult, IonType, RawSymbolToken, Timestamp}; use bytes::{BigEndian, Buf, ByteOrder}; use num_bigint::BigUint; use num_traits::Zero; diff --git a/src/binary/raw_binary_writer.rs b/src/binary/raw_binary_writer.rs index 3598d900..af4fcd9d 100644 --- a/src/binary/raw_binary_writer.rs +++ b/src/binary/raw_binary_writer.rs @@ -904,15 +904,13 @@ impl IonWriter for RawBinaryWriter { mod writer_tests { use std::fmt::Debug; - use crate::StreamItem; - use rstest::*; use super::*; + use crate::ion_reader::IonReader; use crate::raw_symbol_token::{local_sid_token, RawSymbolToken}; - use crate::reader::{Reader, ReaderBuilder}; + use crate::reader::{Reader, ReaderBuilder, StreamItem}; use crate::types::{Blob, Clob, Symbol}; - use crate::IonReader; use num_bigint::BigInt; use num_traits::Float; use std::convert::TryInto; diff --git a/src/binary/timestamp.rs b/src/binary/timestamp.rs index 5784352d..56fe9b48 100644 --- a/src/binary/timestamp.rs +++ b/src/binary/timestamp.rs @@ -130,7 +130,9 @@ where #[cfg(test)] mod binary_timestamp_tests { use super::*; - use crate::{reader, IonReader, IonType, ReaderBuilder}; + use crate::ion_reader::IonReader; + use crate::reader::{ReaderBuilder, StreamItem}; + use crate::IonType; use rstest::*; // These tests show how varying levels of precision affects number of bytes @@ -148,7 +150,7 @@ mod binary_timestamp_tests { ) -> IonResult<()> { let mut reader = ReaderBuilder::new().build(input).unwrap(); match reader.next().unwrap() { - reader::StreamItem::Value(IonType::Timestamp) => { + StreamItem::Value(IonType::Timestamp) => { let timestamp = reader.read_timestamp().unwrap(); let mut buf = vec![]; let written = buf.encode_timestamp_value(×tamp)?; diff --git a/src/blocking_reader.rs b/src/blocking_reader.rs index 38b4a3c1..92ead92c 100644 --- a/src/blocking_reader.rs +++ b/src/blocking_reader.rs @@ -4,9 +4,9 @@ use std::ops::Range; use crate::binary::non_blocking::raw_binary_reader::RawBinaryReader; use crate::data_source::ToIonDataSource; use crate::element::{Blob, Clob}; +use crate::ion_reader::IonReader; use crate::raw_reader::BufferedRawReader; use crate::result::IonResult; -use crate::stream_reader::IonReader; use crate::text::non_blocking::raw_text_reader::RawTextReader; use crate::types::Timestamp; use crate::{Decimal, Int, IonError, IonType, Str}; diff --git a/src/element/element_stream_reader.rs b/src/element/element_stream_reader.rs index d3a9fab1..84802d92 100644 --- a/src/element/element_stream_reader.rs +++ b/src/element/element_stream_reader.rs @@ -3,9 +3,9 @@ use crate::text::parent_container::ParentContainer; use crate::element::iterators::SymbolsIterator; use crate::element::{Blob, Clob, Element}; -use crate::{ - Decimal, Int, IonError, IonReader, IonResult, IonType, Str, StreamItem, Symbol, Timestamp, -}; +use crate::ion_reader::IonReader; +use crate::reader::StreamItem; +use crate::{Decimal, Int, IonError, IonResult, IonType, Str, Symbol, Timestamp}; use std::fmt::Display; use std::mem; @@ -354,8 +354,8 @@ mod reader_tests { use rstest::*; use super::*; + use crate::ion_reader::IonReader; use crate::result::IonResult; - use crate::stream_reader::IonReader; use crate::types::{Decimal, Timestamp}; use crate::IonType; diff --git a/src/element/mod.rs b/src/element/mod.rs index 84623901..025e6e15 100644 --- a/src/element/mod.rs +++ b/src/element/mod.rs @@ -17,8 +17,8 @@ use crate::element::reader::ElementReader; use crate::ion_data::{IonEq, IonOrd}; use crate::text::text_formatter::IonValueFormatter; use crate::{ - ion_data, BinaryWriterBuilder, Decimal, Int, IonResult, IonType, IonWriter, ReaderBuilder, Str, - Symbol, TextWriterBuilder, Timestamp, + ion_data, BinaryWriterBuilder, Decimal, Int, IonResult, IonType, IonWriter, Str, Symbol, + TextWriterBuilder, Timestamp, }; use std::cmp::Ordering; use std::fmt::{Display, Formatter}; @@ -35,6 +35,7 @@ pub mod writer; // Re-export the Value variant types and traits so they can be accessed directly from this module. use crate::data_source::ToIonDataSource; use crate::element::writer::{ElementWriter, Format, TextKind}; +use crate::reader::ReaderBuilder; pub use crate::types::{Blob, Bytes, Clob}; pub use annotations::{Annotations, IntoAnnotations}; diff --git a/src/element/reader.rs b/src/element/reader.rs index 8a04376d..15886dbf 100644 --- a/src/element/reader.rs +++ b/src/element/reader.rs @@ -4,8 +4,10 @@ //! as slices or files. use crate::element::{Annotations, Element, Sequence, Struct, Value}; +use crate::ion_reader::IonReader; +use crate::reader::StreamItem; use crate::result::{decoding_error, IonResult}; -use crate::{IonReader, StreamItem, Symbol}; +use crate::Symbol; /// Reads Ion data into [`Element`] instances. /// diff --git a/src/stream_reader.rs b/src/ion_reader.rs similarity index 100% rename from src/stream_reader.rs rename to src/ion_reader.rs diff --git a/src/lib.rs b/src/lib.rs index ee37ecba..aa766e69 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -93,14 +93,11 @@ //! ```no_run //! # use ion_rs::IonResult; //! # fn main() -> IonResult<()> { -//! use ion_rs::element::reader::ElementReader; //! use ion_rs::element::Element; -//! use ion_rs::ReaderBuilder; //! use std::fs::File; //! let ion_file = File::open("/foo/bar/baz.ion").unwrap(); -//! let mut reader = ReaderBuilder::default().build(ion_file)?; //! // A simple pretty-printer -//! for element in reader.elements() { +//! for element in Element::iter(ion_file)? { //! println!("{}", element?) //! } //! # Ok(()) @@ -177,9 +174,10 @@ use element::Element; pub mod result; pub mod binary; +pub mod constants; pub mod data_source; pub mod element; -pub mod raw_reader; +pub(crate) mod raw_reader; pub mod text; pub mod types; @@ -187,16 +185,15 @@ mod ion_data; #[cfg(feature = "ion-hash")] pub mod ion_hash; -mod blocking_reader; +pub(crate) mod blocking_reader; mod catalog; -// Public as a workaround for: https://github.com/amazon-ion/ion-rust/issues/484 -pub mod constants; + +mod ion_reader; mod raw_symbol_token; mod raw_symbol_token_ref; // Public as a workaround for: https://github.com/amazon-ion/ion-rust/issues/484 -pub mod reader; +pub(crate) mod reader; mod shared_symbol_table; -mod stream_reader; mod symbol_ref; mod symbol_table; mod system_reader; @@ -229,13 +226,20 @@ pub use text::text_writer::{TextWriter, TextWriterBuilder}; pub use writer::IonWriter; pub use binary::raw_binary_writer::RawBinaryWriter; -pub use blocking_reader::{BlockingRawBinaryReader, BlockingRawReader, BlockingRawTextReader}; -pub use raw_reader::{RawReader, RawStreamItem}; -pub use reader::{Reader, ReaderBuilder, StreamItem, UserReader}; -pub use stream_reader::IonReader; -pub use system_reader::{SystemReader, SystemStreamItem}; pub use text::raw_text_writer::{RawTextWriter, RawTextWriterBuilder}; +// These re-exports are only visible if the "experimental-reader" feature is enabled. +#[cfg(feature = "experimental-reader")] +pub use { + blocking_reader::{BlockingRawBinaryReader, BlockingRawReader, BlockingRawTextReader}, + ion_reader::IonReader, + raw_reader::{BufferedRawReader, RawReader, RawStreamItem}, + // Public as a workaround for: https://github.com/amazon-ion/ion-rust/issues/484 + reader::integration_testing, + reader::{Reader, ReaderBuilder, StreamItem, UserReader}, + system_reader::{SystemReader, SystemStreamItem}, +}; + pub use result::{IonError, IonResult}; /// Re-exports of third party dependencies that are part of our public API. diff --git a/src/raw_reader.rs b/src/raw_reader.rs index 74fb94e8..4d0e3be9 100644 --- a/src/raw_reader.rs +++ b/src/raw_reader.rs @@ -1,6 +1,6 @@ use crate::element::{Blob, Clob}; +use crate::ion_reader::IonReader; use crate::raw_symbol_token::RawSymbolToken; -use crate::stream_reader::IonReader; use crate::types::{IonType, Str}; use crate::{Decimal, Int, IonResult, Timestamp}; use std::fmt::{Display, Formatter}; diff --git a/src/reader.rs b/src/reader.rs index 35427bea..08902f4d 100644 --- a/src/reader.rs +++ b/src/reader.rs @@ -6,17 +6,17 @@ use delegate::delegate; use crate::binary::constants::v1_0::IVM; use crate::binary::non_blocking::raw_binary_reader::RawBinaryReader; +use crate::blocking_reader::{BlockingRawBinaryReader, BlockingRawTextReader}; use crate::data_source::ToIonDataSource; use crate::element::{Blob, Clob}; +use crate::ion_reader::IonReader; use crate::raw_reader::{Expandable, RawReader}; use crate::raw_symbol_token::RawSymbolToken; use crate::result::{decoding_error, IonResult}; -use crate::stream_reader::IonReader; use crate::symbol_table::SymbolTable; +use crate::system_reader::SystemReader; use crate::types::{Decimal, Int, Symbol, Timestamp}; -use crate::{ - BlockingRawBinaryReader, BlockingRawTextReader, IonType, SystemReader, SystemStreamItem, -}; +use crate::IonType; use std::fmt::{Display, Formatter}; use crate::types::Str; @@ -129,7 +129,8 @@ impl UserReader { // See: https://github.com/amazon-ion/ion-rust/issues/484 #[doc(hidden)] pub mod integration_testing { - use crate::{RawReader, Reader, UserReader}; + use crate::raw_reader::RawReader; + use crate::reader::{Reader, UserReader}; pub fn new_reader<'a, R: 'a + RawReader>(raw_reader: R) -> Reader<'a> { UserReader::new(Box::new(raw_reader)) @@ -201,7 +202,7 @@ impl IonReader for UserReader { // v-- Clippy complains that `next` resembles `Iterator::next()` #[allow(clippy::should_implement_trait)] fn next(&mut self) -> IonResult { - use SystemStreamItem::*; + use crate::system_reader::SystemStreamItem::*; loop { // If the system reader encounters an encoding artifact like a symbol table or IVM, // keep going until we find a value or exhaust the stream. @@ -290,11 +291,10 @@ mod tests { use super::*; use crate::binary::constants::v1_0::IVM; - use crate::BlockingRawBinaryReader; + use crate::reader::{BlockingRawBinaryReader, StreamItem::Value}; use crate::result::IonResult; use crate::types::IonType; - use crate::StreamItem::Value; type TestDataSource = io::Cursor>; @@ -314,7 +314,7 @@ mod tests { // Prepends an Ion 1.0 IVM to the provided data and then creates a BinaryIonCursor over it fn raw_binary_reader_for(bytes: &[u8]) -> BlockingRawBinaryReader { - use crate::RawStreamItem::*; + use crate::raw_reader::RawStreamItem::*; let mut raw_reader = BlockingRawBinaryReader::new(data_source_for(bytes)).expect("unable to create reader"); assert_eq!(raw_reader.ion_type(), None); diff --git a/src/system_reader.rs b/src/system_reader.rs index 716d16d1..2670fddb 100644 --- a/src/system_reader.rs +++ b/src/system_reader.rs @@ -4,12 +4,13 @@ use std::ops::Range; use crate::binary::non_blocking::raw_binary_reader::RawBinaryReader; use crate::constants::v1_0::{system_symbol_ids, SYSTEM_SYMBOLS}; use crate::element::{Blob, Clob}; +use crate::ion_reader::IonReader; use crate::raw_reader::{Expandable, RawReader, RawStreamItem}; use crate::raw_symbol_token::RawSymbolToken; use crate::result::{decoding_error, decoding_error_raw, illegal_operation, IonError, IonResult}; use crate::system_reader::LstPosition::*; use crate::types::{Decimal, Int, Str, Symbol, Timestamp}; -use crate::{IonReader, IonType, SymbolTable}; +use crate::{IonType, SymbolTable}; /// Tracks where the [SystemReader] is in the process of reading a local symbol table. #[derive(Debug, Copy, Clone, PartialEq, Eq)] diff --git a/src/text/non_blocking/raw_text_reader.rs b/src/text/non_blocking/raw_text_reader.rs index 6921bab1..c6ffc356 100644 --- a/src/text/non_blocking/raw_text_reader.rs +++ b/src/text/non_blocking/raw_text_reader.rs @@ -4,13 +4,13 @@ use crate::element::{Blob, Clob}; use crate::types::Str; use nom::Err::{Error, Failure, Incomplete}; +use crate::ion_reader::IonReader; use crate::raw_reader::{BufferedRawReader, Expandable, RawStreamItem}; use crate::raw_symbol_token::RawSymbolToken; use crate::result::{ decoding_error, illegal_operation, illegal_operation_raw, incomplete_text_error, IonError, IonResult, Position, }; -use crate::stream_reader::IonReader; use crate::text::non_blocking::text_buffer::TextBuffer; use crate::text::parent_container::ParentContainer; use crate::text::parse_result::IonParseResult; @@ -944,15 +944,15 @@ mod reader_tests { use rstest::*; use super::*; + use crate::ion_reader::IonReader; use crate::raw_reader::RawStreamItem; + use crate::raw_reader::RawStreamItem::Nothing; use crate::raw_symbol_token::{local_sid_token, text_token, RawSymbolToken}; use crate::result::{IonResult, Position}; - use crate::stream_reader::IonReader; use crate::text::non_blocking::raw_text_reader::RawTextReader; use crate::text::text_value::{IntoRawAnnotations, TextValue}; use crate::types::{Decimal, Timestamp}; use crate::IonType; - use crate::RawStreamItem::Nothing; fn next_type + Expandable>( reader: &mut RawTextReader, diff --git a/src/text/text_writer.rs b/src/text/text_writer.rs index 0882504a..4c409676 100644 --- a/src/text/text_writer.rs +++ b/src/text/text_writer.rs @@ -188,9 +188,8 @@ impl IonWriter for TextWriter { #[cfg(test)] mod tests { use super::*; - use crate::reader::ReaderBuilder; - use crate::IonReader; - use crate::StreamItem::Value; + use crate::ion_reader::IonReader; + use crate::reader::{ReaderBuilder, StreamItem::Value}; #[test] fn resolve_symbol_ids() -> IonResult<()> { diff --git a/tests/element_test_vectors.rs b/tests/element_test_vectors.rs index 0f79ac13..c6244fdd 100644 --- a/tests/element_test_vectors.rs +++ b/tests/element_test_vectors.rs @@ -1,4 +1,5 @@ // Copyright Amazon.com, Inc. or its affiliates. +#![cfg(feature = "experimental-reader")] use ion_rs::element::reader::ElementReader; use ion_rs::element::writer::{ElementWriter, Format, TextKind}; @@ -558,7 +559,7 @@ mod non_blocking_native_element_tests { // These imports are visible as a temporary workaround. // See: https://github.com/amazon-ion/ion-rust/issues/484 use ion_rs::binary::constants::v1_0::IVM; - use ion_rs::reader::integration_testing::new_reader; + use ion_rs::integration_testing::new_reader; // If the data is binary, create a non-blocking binary reader. if data.starts_with(&IVM) { let raw_reader = RawBinaryReader::new(data); diff --git a/tests/good_test_vectors.rs b/tests/good_test_vectors.rs deleted file mode 100644 index ea5b5f9a..00000000 --- a/tests/good_test_vectors.rs +++ /dev/null @@ -1,176 +0,0 @@ -use std::collections::BTreeSet; -use std::ffi::OsStr; -use std::fs::File; -use std::io::BufReader; -use std::iter::FromIterator; -use std::path::{Path, PathBuf}; -use std::str::FromStr; - -use walkdir::WalkDir; - -use ion_rs::result::{decoding_error, IonResult}; -use ion_rs::{IonReader, IonType, Reader, ReaderBuilder}; - -const GOOD_TEST_FILES_PATH: &str = "ion-tests/iontestdata/good/"; - -// TODO: Populate skip list -const GOOD_TEST_FILES_SKIP_LIST: &[&str] = &[ - // NOP pad support - "ion-tests/iontestdata/good/emptyThreeByteNopPad.10n", - "ion-tests/iontestdata/good/equivs/nopPadEmptyStruct.10n", - "ion-tests/iontestdata/good/equivs/nopPadNonEmptyStruct.10n", - "ion-tests/iontestdata/good/nopPad16Bytes.10n", - "ion-tests/iontestdata/good/nopPadInsideEmptyStructNonZeroSymbolId.10n", - "ion-tests/iontestdata/good/nopPadInsideEmptyStructZeroSymbolId.10n", - "ion-tests/iontestdata/good/nopPadInsideStructWithNopPadThenValueNonZeroSymbolId.10n", - "ion-tests/iontestdata/good/nopPadInsideStructWithNopPadThenValueZeroSymbolId.10n", - "ion-tests/iontestdata/good/nopPadInsideStructWithValueThenNopPad.10n", - "ion-tests/iontestdata/good/nopPadOneByte.10n", - "ion-tests/iontestdata/good/valueFollowedByNopPad.10n", - "ion-tests/iontestdata/good/valuePrecededByNopPad.10n", - "ion-tests/iontestdata/good/valueBetweenNopPads.10n", - // Timestamp support - "ion-tests/iontestdata/good/equivs/timestampSuperfluousOffset.10n", - "ion-tests/iontestdata/good/timestamp/timestamp2011-02.10n", - "ion-tests/iontestdata/good/timestamp/timestamp2011.10n", - // Integer representation limits - "ion-tests/iontestdata/good/equivs/intsLargeNegative1.10n", - "ion-tests/iontestdata/good/equivs/intsLargeNegative2.10n", - "ion-tests/iontestdata/good/equivs/intsLargeNegative3.10n", - "ion-tests/iontestdata/good/equivs/intsLargePositive1.10n", - "ion-tests/iontestdata/good/equivs/intsLargePositive2.10n", - "ion-tests/iontestdata/good/equivs/intsLargePositive3.10n", - "ion-tests/iontestdata/good/intBigSize1201.10n", - "ion-tests/iontestdata/good/intBigSize13.10n", - "ion-tests/iontestdata/good/intBigSize14.10n", - "ion-tests/iontestdata/good/intBigSize16.10n", - "ion-tests/iontestdata/good/intBigSize256.10n", - "ion-tests/iontestdata/good/intLongMaxValuePlusOne.10n", - "ion-tests/iontestdata/good/intLongMinValue.10n", - "ion-tests/iontestdata/good/equivs/paddedInts.10n", - "ion-tests/iontestdata/good/item1.10n", - // Typecode validation - "ion-tests/iontestdata/good/typecodes/T0.10n", - "ion-tests/iontestdata/good/typecodes/T11.10n", - "ion-tests/iontestdata/good/typecodes/T12.10n", - "ion-tests/iontestdata/good/typecodes/T2.10n", - "ion-tests/iontestdata/good/typecodes/T3.10n", - "ion-tests/iontestdata/good/typecodes/T5.10n", - "ion-tests/iontestdata/good/typecodes/T6-small.10n", - "ion-tests/iontestdata/good/typecodes/T7-large.10n", -]; - -// Iterates over all of the Ion files in GOOD_TEST_FILES_PATH and tries reading each in full. -// If reading completes without an error, the test succeeds. -#[test] -fn read_good_files() -> IonResult<()> { - // TODO: This is binary-specific because don't have a text reader yet. - let binary_file_extension: &OsStr = OsStr::new("10n"); - let good_files = all_files_in(GOOD_TEST_FILES_PATH); - let paths_to_skip = skip_list_as_set(GOOD_TEST_FILES_SKIP_LIST); - let binary_good_files: Vec<_> = good_files - .iter() - .filter(|f| f.extension() == Some(binary_file_extension)) - .filter(|f| !paths_to_skip.contains(<&&PathBuf as AsRef>::as_ref(&f))) - .collect(); - let mut failure_count: usize = 0; - println!(); - for entry in &binary_good_files { - print!("Reading {}... ", entry.display()); - if let Err(error) = read_file(entry.as_ref()) { - print!("ERROR: {error:?}"); - failure_count += 1; - } else { - print!("OK"); - } - println!(); - } - if failure_count > 0 { - return decoding_error(format!( - "{failure_count} good test files could not be read successfully." - )); - } - Ok(()) -} - -// Converts the provided slice of strings to a HashSet of paths -fn skip_list_as_set(files_to_skip: &[&str]) -> BTreeSet { - let mut skip_set = BTreeSet::new(); - for file in files_to_skip { - skip_set.insert(PathBuf::from_str(file).unwrap()); - } - skip_set -} - -// Collects all of the files in the provided path into a BTreeSet for easy iteration/filtering. -fn all_files_in(path: &str) -> BTreeSet { - let binary_file_iterator = WalkDir::new(path) - .into_iter() - .map(|entry| { - entry.unwrap_or_else(|error| panic!("Failure during dir traversal: {error:?}")) - }) - .filter(|entry| entry.path().is_file()) - .map(|entry| entry.path().to_owned()); - BTreeSet::from_iter(binary_file_iterator) -} - -// Reads all of the Ion values found in the provided file, reporting any errors. -fn read_file(path: &Path) -> IonResult<()> { - let file = File::open(path).unwrap_or_else(|error| panic!("Failed to open file: {error:?}")); - let file_reader = BufReader::new(file); - let mut reader = ReaderBuilder::new().build(file_reader)?; - read_all_values(&mut reader)?; - Ok(()) -} - -// Recursively reads all of the values in the provided Reader, surfacing any errors. -fn read_all_values(reader: &mut Reader) -> IonResult<()> { - // StreamItem::Null conflicts with IonType::Null, so we give them aliases for clarity. - use ion_rs::StreamItem::{Null as NullValue, *}; - use IonType::{Null as NullType, *}; - - while let Value(ion_type) | NullValue(ion_type) = reader.next()? { - if reader.is_null() { - continue; - } - match ion_type { - Struct | List | SExp => { - reader.step_in()?; - read_all_values(reader)?; - reader.step_out()?; - } - String => { - let _text = reader.read_string()?; - } - Symbol => { - // The binary reader's tokens are always SIDs - let _symbol_id = reader.read_symbol()?; - } - Int => { - let _int = reader.read_i64()?; - } - Float => { - let _float = reader.read_f64()?; - } - Decimal => { - let _decimal = reader.read_decimal()?; - } - Timestamp => { - let _timestamp = reader.read_timestamp()?; - } - Bool => { - let _boolean = reader.read_bool()?; - } - Blob => { - let _blob = reader.read_blob()?; - } - Clob => { - let _clob = reader.read_clob()?; - } - NullType => { - unreachable!("Value with IonType::Null returned is_null=false."); - } - } - } - Ok(()) -}