From 6778051e855e39b0e9a5efb7c05593d804e7d4fc Mon Sep 17 00:00:00 2001
From: Douglas Raillard <douglas.raillard@arm.com>
Date: Fri, 5 Jan 2024 13:32:36 +0000
Subject: [PATCH] WIP

---
 tools/analysis/trace-tools/src/lib/parquet.rs | 43 ++++++++++---------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/tools/analysis/trace-tools/src/lib/parquet.rs b/tools/analysis/trace-tools/src/lib/parquet.rs
index 8c39680797..97336061ee 100644
--- a/tools/analysis/trace-tools/src/lib/parquet.rs
+++ b/tools/analysis/trace-tools/src/lib/parquet.rs
@@ -385,9 +385,9 @@ where
             }
         )?;
 
-        // If there was any error, some columns might have been appended to but others not, so we
-        // cannot finalize the files and we should just bail out.
-        events.collect::<Result<_, MainError>>()?;
+        // Even if there were some errors, we should have pushed some None values so that all
+        // columns are of the same length, so the file can be finalized.
+        let res = last_err(events);
         eprintln!("COUNT {count}");
 
         let mut handles = Vec::new();
@@ -421,7 +421,7 @@ where
         for handle in handles {
             handle.join().expect("Writer thread panicked")?;
         }
-        Ok(())
+        res
     })
     .unwrap()?;
     Ok(())
@@ -673,27 +673,15 @@ where
             }
         };
 
-        // We want to go through all the columns so that we have a chance to append None values in
-        // places we had an error, and when we are done we return the last error. This way, all
-        // columns should have the same length and we will still be able to dump to parquet.
-        fn last_err<E, I: Iterator<Item=Result<(), E>>>(iter: I) -> Result<(), E> {
-            let mut res = Ok(());
-            for x in iter {
-                match x {
-                    Err(err) => {
-                        res = Err(err);
-                    }
-                    _ => {}
-                }
-            }
-            res
-        }
-
         macro_rules! generic_iter {
             ($table_state:expr, $visitor:expr) => {{
                 let table_state = $table_state;
                 let visitor = $visitor;
 
+                // We want to go through all the columns so that we have a chance to append None
+                // values in places we had an error, and when we are done we return the last error.
+                // This way, all columns should have the same length and we will still be able to
+                // dump to parquet.
                 let res = last_err(visitor
                     .fields()?
                     .into_iter()
@@ -1171,6 +1159,21 @@ impl EventWriteState {
     }
 }
 
+
+fn last_err<E, I: Iterator<Item=Result<(), E>>>(iter: I) -> Result<(), E> {
+    let mut res = Ok(());
+    for x in iter {
+        match x {
+            Err(err) => {
+                res = Err(err);
+            }
+            _ => {}
+        }
+    }
+    res
+}
+
+
 // fn main2() -> Result<(), ArrowError> {
 // // declare arrays
 // let a = Int8Array::from(&[Some(1), None, Some(3)]);