Add support to unzip stream (#7713)

- Add support to unzip a stream. Adding unit test
QuantConnect · Jan 23, 2024 · 3a8a3dd · 3a8a3dd
1 parent 3dab2e4
commit 3a8a3dd
Show file tree

Hide file tree

Showing 2 changed files with 56 additions and 22 deletions.
diff --git a/Compression/Compression.cs b/Compression/Compression.cs
@@ -20,6 +20,7 @@
 using System.IO.Compression;
 using System.Linq;
 using System.Text;
+using System.Threading.Tasks;
 using ICSharpCode.SharpZipLib.Core;
 using ICSharpCode.SharpZipLib.GZip;
 using ICSharpCode.SharpZipLib.Tar;
@@ -237,39 +238,48 @@ public static bool ZipCreateAppendData(string path, string entry, byte[] data, b
         /// <param name="encoding">Specifies the encoding used to read the bytes. If not specified, defaults to ASCII</param>
         /// <returns>Uncompressed dictionary string-sting of files in the zip</returns>
         public static Dictionary<string, string> UnzipData(byte[] zipData, Encoding encoding = null)
+        {
+            using var stream = new MemoryStream(zipData);
+            return UnzipDataAsync(stream, encoding).ConfigureAwait(false).GetAwaiter().GetResult();
+        }
+
+        /// <summary>
+        /// Uncompress zip data byte array into a dictionary string array of filename-contents.
+        /// </summary>
+        /// <param name="stream">Stream data of zip compressed information</param>
+        /// <param name="encoding">Specifies the encoding used to read the bytes. If not specified, defaults to ASCII</param>
+        /// <returns>Uncompressed dictionary string-sting of files in the zip</returns>
+        public static async Task<Dictionary<string, string>> UnzipDataAsync(Stream stream, Encoding encoding = null)
         {
             // Initialize:
             var data = new Dictionary<string, string>();
 
             try
             {
-                using (var ms = new MemoryStream(zipData))
+                //Read out the zipped data into a string, save in array:
+                using (var zipStream = new ZipInputStream(stream))
                 {
-                    //Read out the zipped data into a string, save in array:
-                    using (var zipStream = new ZipInputStream(ms))
+                    while (true)
                     {
-                        while (true)
-                        {
-                            //Get the next file
-                            var entry = zipStream.GetNextEntry();
+                        //Get the next file
+                        var entry = zipStream.GetNextEntry();
 
-                            if (entry != null)
-                            {
-                                //Read the file into buffer:
-                                var buffer = new byte[entry.Size];
-                                zipStream.Read(buffer, 0, (int)entry.Size);
+                        if (entry != null)
+                        {
+                            // Read the file into buffer:
+                            var buffer = new byte[entry.Size];
+                            await zipStream.ReadAsync(buffer, 0, (int)entry.Size).ConfigureAwait(false);
 
-                                //Save into array:
-                                var str = (encoding ?? Encoding.ASCII).GetString(buffer);
-                                data.Add(entry.Name, str);
-                            }
-                            else
-                            {
-                                break;
-                            }
+                            //Save into array:
+                            var str = (encoding ?? Encoding.ASCII).GetString(buffer);
+                            data[entry.Name] = str;
+                        }
+                        else
+                        {
+                            break;
                         }
-                    } // End Zip Stream.
-                } // End Using Memory Stream
+                    }
+                } // End Zip Stream.
 
             }
             catch (Exception err)

diff --git a/Tests/Compression/CompressionTests.cs b/Tests/Compression/CompressionTests.cs
@@ -210,6 +210,30 @@ public void UnzipDataSupportsEncoding()
             Assert.AreEqual(expected.Value, actual.Value);
         }
 
+        [Test]
+        public void UnzipDataStream()
+        {
+            var data = new Dictionary<string, string>
+            {
+                {"Ł", "The key is unicode"}
+            };
+
+            var encoding = Encoding.UTF8;
+            var bytes = encoding.GetBytes(JsonConvert.SerializeObject(data));
+            var compressed = QuantConnect.Compression.ZipBytes(bytes, "entry.json");
+
+            using var stream = new MemoryStream(compressed);
+            var decompressed = QuantConnect.Compression.UnzipDataAsync(stream, encoding).Result;
+            var redata = JsonConvert.DeserializeObject<Dictionary<string, string>>(
+                decompressed.Single().Value
+            );
+
+            var expected = data.Single();
+            var actual = redata.Single();
+            Assert.AreEqual(expected.Key, actual.Key);
+            Assert.AreEqual(expected.Value, actual.Value);
+        }
+
         [TestCase(true)]
         [TestCase(false)]
         public void ZipCreateAppendData(bool overrideEntry)