Incomplete text detection tests for v1.0 (#823)

Cargo.toml

@@ -18,10 +18,7 @@ exclude = [
 ]
 version = "1.0.0-rc.6"
 edition = "2021"
-# We need at least 1.65 for GATs[1] and 1.67 for `ilog`[2]
-# [1] https://blog.rust-lang.org/2022/11/03/Rust-1.65.0.html
-# [2] https://blog.rust-lang.org/2023/01/26/Rust-1.67.0.html#stabilized-apis
-rust-version = "1.77"
+rust-version = "1.80"
 
 [features]
 default = []

tests/detect_incomplete_text.rs

@@ -0,0 +1,74 @@
+#![cfg(feature = "experimental-reader-writer")]
+
+use crate::ion_tests::{DataStraw, ELEMENT_GLOBAL_SKIP_LIST};
+use ion_rs::{AnyEncoding, ElementReader, IonError, IonResult, IonStream, Reader};
+use std::collections::HashSet;
+use std::fs;
+use std::io::BufReader;
+use std::iter::Iterator;
+use std::sync::LazyLock;
+use test_generator::test_resources;
+
+mod ion_tests;
+
+// A copy of the `ELEMENT_GLOBAL_SKIP_LIST` in which each file name has been canonicalized for the
+// current host machine. This makes it possible to compare names in the list with names of files
+// on the host without worrying about differences in (for example) path separators.
+static CANONICAL_FILE_NAMES: LazyLock<Vec<String>> = LazyLock::new(|| {
+    ELEMENT_GLOBAL_SKIP_LIST
+        .iter()
+        .filter_map(|filename| {
+            // Canonicalize the skip list file names so they're in the host OS' preferred format.
+            // This involves looking up the actual file; if canonicalization fails, the file could
+            // not be found/read which could mean the skip list is outdated.
+            fs::canonicalize(filename).ok()
+        })
+        .map(|filename| filename.to_string_lossy().into())
+        .collect()
+});
+
+static SKIP_LIST_1_0: LazyLock<HashSet<String>> =
+    LazyLock::new(|| CANONICAL_FILE_NAMES.iter().cloned().collect());
+
+static SKIP_LIST_1_1: LazyLock<HashSet<String>> = LazyLock::new(|| {
+    CANONICAL_FILE_NAMES
+        .iter()
+        .map(|file_1_0| file_1_0.replace("_1_0", "_1_1"))
+        .collect()
+});
+
+#[test_resources("ion-tests/iontestdata_1_0/good/**/*.ion")]
+fn detect_incomplete_input_1_0(file_name: &str) {
+    incomplete_text_detection_test(&SKIP_LIST_1_0, file_name).unwrap()
+}
+
+#[test_resources("ion-tests/iontestdata_1_1/good/**/*.ion")]
+fn detect_incomplete_input_1_1(file_name: &str) {
+    incomplete_text_detection_test(&SKIP_LIST_1_1, file_name).unwrap()
+}
+
+fn incomplete_text_detection_test(skip_list: &HashSet<String>, file_name: &str) -> IonResult<()> {
+    // Canonicalize the file name so it can be compared to skip list file names without worrying
+    // about path separators.
+    let file_name: String = fs::canonicalize(file_name)?.to_string_lossy().into();
+    if skip_list.contains(&file_name) {
+        return Ok(());
+    }
+    println!("testing {file_name}");
+    let file = fs::File::open(file_name)?;
+    let buf_reader = BufReader::new(file);
+    let input = DataStraw::new(buf_reader);
+    let ion_stream = IonStream::new(input);
+    let mut reader = Reader::new(AnyEncoding, ion_stream)?;
+    // Manually destructure to allow for pretty-printing of errors
+    match reader.read_all_elements() {
+        Ok(_) => {}
+        Err(IonError::Decoding(e)) => {
+            panic!("{:?}: {}", e.position(), e);
+        }
+        Err(other) => {
+            panic!("{other:#?}");
+        }
+    }
+    Ok(())
+}

tests/ion_tests/mod.rs

@@ -14,7 +14,6 @@ use ion_rs::{
     Symbol, Value,
 };
 
-mod detect_incomplete_text;
 pub mod lazy_element_ion_tests;
 
 /// Concatenates two slices of string slices together.
@@ -443,7 +442,7 @@ pub const ELEMENT_EQUIVS_SKIP_LIST: SkipList = &[
 /// call to `read()`. This is used in tests to confirm that the reader's
 /// data-incompleteness and retry logic will properly handle all corner
 /// cases.
-pub(crate) struct DataStraw<R> {
+pub struct DataStraw<R> {
     input: R,
 }
 

tests/ion_tests_1_1.rs

@@ -1,5 +1,4 @@
 #![cfg(feature = "experimental-reader-writer")]
-
 /// TODO: When the Ion 1.1 binary reader is complete, update this module to include binary tests
 mod ion_tests;