increase size of compression buffer on very large arrays

closes #12
brentp · Feb 26, 2022 · 8f7596f · 8f7596f
1 parent c7cc03f
commit 8f7596f
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 1 deletion.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,3 +1,9 @@
+v0.1.3
+======
++ exit with error on multi-allelics (previously, only first allele was used) ( #11)
++ more docs on conf file (thanks @m-pauper)
++ fix compression error for very dense regions (#12)
+
 v0.1.2
 ======
 + fix #8. bug introduced for string values.

diff --git a/src/commands/encoder_cmd.rs b/src/commands/encoder_cmd.rs
@@ -240,7 +240,7 @@ pub fn encoder_main(vpaths: Vec<&str>, opath: &str, jpath: &str) {
     )
     .ok();
 
-    let mut compressed = vec![0u8; 50_000_000]; // TODO: set this based on values.len
+    let mut compressed = vec![0u8; 50_000_000];
     let mut last_rid: i32 = -1;
     let mut last_mod: i64 = 0;
 
@@ -284,6 +284,13 @@ pub fn encoder_main(vpaths: Vec<&str>, opath: &str, jpath: &str) {
                             zipf.start_file(fname, options)
                                 .expect("error starting file");
                             sort_by_indices(values, indexes.clone());
+                            if compressed.len() < 5 * values.len() {
+                                eprintln!(
+                                    "[echtvar] resizing compressed array to: {}",
+                                    5 * values.len()
+                                );
+                                compressed.resize(5 * values.len(), 0x0);
+                            }
                             write_bits(values, false, &mut zipf, &mut compressed);
                             values.clear();
                         }
@@ -396,6 +403,13 @@ pub fn encoder_main(vpaths: Vec<&str>, opath: &str, jpath: &str) {
                 zipf.start_file(fname, options)
                     .expect("error starting file");
                 sort_by_indices(values, indexes.clone());
+                if compressed.len() < 5 * values.len() {
+                    eprintln!(
+                        "[echtvar] resizing compressed array to: {}",
+                        5 * values.len()
+                    );
+                    compressed.resize(5 * values.len(), 0x0);
+                }
                 write_bits(values, false, &mut zipf, &mut compressed);
                 values.clear();
             }