Java: forFile throws OutOfMemoryError for files >2GB #383

Sha256: Stream file
GDATASoftwareAG · Mar 8, 2024 · d612b3b · d612b3b
1 parent d985a76
commit d612b3b
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 14 deletions.
diff --git a/java/src/main/java/de/gdata/vaas/Sha256.java b/java/src/main/java/de/gdata/vaas/Sha256.java
@@ -3,14 +3,13 @@
 import lombok.Getter;
 import lombok.NonNull;
 
+import java.io.FileInputStream;
 import java.io.IOException;
-import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.regex.Pattern;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
 
 public class Sha256 {
     private static Pattern pattern = Pattern.compile("^[A-Fa-f0-9]{64}$");
@@ -30,14 +29,21 @@ public Sha256(String sha256) {
     }
 
     public Sha256(Path file) throws IOException, NoSuchAlgorithmException {
-        var bytes = Files.readAllBytes(file);
-        var digest = MessageDigest.getInstance("SHA-256").digest(bytes);
-
-        String sb = IntStream
-                .range(0, digest.length)
-                .mapToObj(i -> Integer.toString((digest[i] & 0xff) + 0x100, 16)
-                        .substring(1)).collect(Collectors.joining());
-
-        this.value = sb;
+        MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
+        try (FileInputStream fileInputStream = new FileInputStream(file.toString());
+            DigestInputStream digestInputStream = new DigestInputStream(fileInputStream, sha256)) {
+
+            byte[] buffer = new byte[8192];
+            while (digestInputStream.read(buffer) != -1) {
+                // Read file in chunks and update the digest
+            }
+
+            byte[] digest = sha256.digest();
+            StringBuilder hexString = new StringBuilder();
+            for (byte b : digest) {
+                hexString.append(String.format("%02x", b));
+            }
+            this.value = hexString.toString();
+        }
     }
 }
diff --git a/java/src/main/java/de/gdata/vaas/Vaas.java b/java/src/main/java/de/gdata/vaas/Vaas.java
@@ -531,7 +531,7 @@ private CompletableFuture<VerdictResponse> forFileAsync(Path file, UUID guid,
                     try {
                         var uploadResponseFuture = this.client.waitForVerdict(verdictRequest.getGuid());
 
-                        return UploadFile(file, verdictResponse.getUploadUrl(), verdictResponse.getUploadToken())
+                        return uploadFile(file, verdictResponse.getUploadUrl(), verdictResponse.getUploadToken())
                                 .thenCompose((v) -> uploadResponseFuture);
                     } catch (Exception e) {
                         throwAsUnchecked(e);
@@ -540,7 +540,7 @@ private CompletableFuture<VerdictResponse> forFileAsync(Path file, UUID guid,
                 });
     }
 
-    private CompletableFuture<Void> UploadFile(Path file, String url, String authToken)
+    private CompletableFuture<Void> uploadFile(Path file, String url, String authToken)
             throws IOException, URISyntaxException {
 
         var builder = HttpRequest