change logic to use streams for package data attributes instead of lo…

…ading into ram
haiku · Feb 9, 2021 · 13a66a1 · 13a66a1
1 parent 2109840
commit 13a66a1
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 26 deletions.
diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HeapCoordinates.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HeapCoordinates.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2019, Andrew Lindesay
+ * Copyright 2013-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
@@ -36,6 +36,10 @@ public long getLength() {
         return length;
     }
 
+    public boolean isEmpty() {
+        return 0L == length;
+    }
+
     @SuppressWarnings("RedundantIfStatement") // was auto-generated!
     @Override
     public boolean equals(Object o) {

diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HeapInputStream.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/heap/HeapInputStream.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2021, Andrew Lindesay
+ * Distributed under the terms of the MIT License.
+ */
+
+package org.haiku.pkg.heap;
+
+import com.google.common.base.Preconditions;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class HeapInputStream extends InputStream {
+
+    private final HeapReader reader;
+
+    private final HeapCoordinates coordinates;
+
+    private long offsetInCoordinates = 0L;
+
+    public HeapInputStream(HeapReader reader, HeapCoordinates coordinates) {
+        this.reader = Preconditions.checkNotNull(reader);
+        this.coordinates = Preconditions.checkNotNull(coordinates);
+    }
+
+    @Override
+    public int read() throws IOException {
+        if (offsetInCoordinates < coordinates.getLength()) {
+            int result = reader.readHeap(coordinates.getOffset() + offsetInCoordinates);
+            offsetInCoordinates++;
+            return result;
+        }
+
+        return -1;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException {
+        Preconditions.checkArgument(null != b, "buffer required");
+        Preconditions.checkArgument(off >= 0, "bad offset supplied");
+        Preconditions.checkArgument(len >= 0, "bad length supplied");
+
+        if (len + offsetInCoordinates >= coordinates.getLength()) {
+            len = (int) (coordinates.getLength() - offsetInCoordinates);
+        }
+
+        if (0 == len) {
+            return -1;
+        }
+
+        HeapCoordinates readCoordinates = new HeapCoordinates(
+                coordinates.getOffset() + offsetInCoordinates, len);
+
+        reader.readHeap(b, off, readCoordinates);
+        offsetInCoordinates += len;
+
+        return len;
+    }
+}
diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/Attribute.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/Attribute.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2018-2019, Andrew Lindesay
+ * Copyright 2018-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
@@ -8,9 +8,7 @@
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import org.haiku.pkg.AttributeContext;
-import org.haiku.pkg.HpkException;
 
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
@@ -22,7 +20,7 @@
 
 public abstract class Attribute {
 
-    private AttributeId attributeId;
+    private final AttributeId attributeId;
 
     private List<Attribute> childAttributes = Collections.emptyList();
 

diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/PkgArchitecture.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/PkgArchitecture.java
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2014, Andrew Lindesay
+ * Copyright 2013-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
@@ -19,5 +19,7 @@ public enum PkgArchitecture {
     PPC,        // 5
     ARM,        // 6
     M68K,       // 7
-    SPARC       // 8
+    SPARC,      // 8
+    ARM64,      // 9
+    RISCV64     // 10
 }
diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/RawHeapAttribute.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/RawHeapAttribute.java
@@ -1,14 +1,19 @@
 /*
- * Copyright 2018, Andrew Lindesay
+ * Copyright 2018-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
 package org.haiku.pkg.model;
 
 import com.google.common.base.Preconditions;
+import com.google.common.io.ByteSource;
 import org.haiku.pkg.AttributeContext;
-import org.haiku.pkg.HpkException;
 import org.haiku.pkg.heap.HeapCoordinates;
+import org.haiku.pkg.heap.HeapInputStream;
+import org.haiku.pkg.heap.HeapReader;
+
+import java.io.IOException;
+import java.io.InputStream;
 
 /**
  * <p>This type of attribute refers to raw data.  It uses coordinates into the heap to provide a source for the
@@ -44,10 +49,8 @@ public int hashCode() {
     }
 
     @Override
-    public byte[] getValue(AttributeContext context) {
-        byte[] buffer = new byte[(int) heapCoordinates.getLength()];
-        context.getHeapReader().readHeap(buffer, 0, heapCoordinates);
-        return buffer;
+    public ByteSource getValue(AttributeContext context) {
+        return new HeapByteSource(context.getHeapReader(), heapCoordinates);
     }
 
     @Override
@@ -60,4 +63,26 @@ public String toString() {
         return String.format("%s : @%s",super.toString(),heapCoordinates.toString());
     }
 
+    private static class HeapByteSource extends ByteSource {
+
+        private final HeapReader heapReader;
+        private final HeapCoordinates heapCoordinates;
+
+        public HeapByteSource(HeapReader heapReader, HeapCoordinates heapCoordinates) {
+            this.heapCoordinates = heapCoordinates;
+            this.heapReader = heapReader;
+        }
+
+        @Override
+        public InputStream openStream() throws IOException {
+            return new HeapInputStream(heapReader, heapCoordinates);
+        }
+
+        @Override
+        public com.google.common.base.Optional<Long> sizeIfKnown() {
+            return com.google.common.base.Optional.of(heapCoordinates.getLength());
+        }
+
+    }
+
 }
diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/RawInlineAttribute.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/model/RawInlineAttribute.java
@@ -1,18 +1,19 @@
 /*
- * Copyright 2018, Andrew Lindesay
+ * Copyright 2018-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
 package org.haiku.pkg.model;
 
 import com.google.common.base.Preconditions;
+import com.google.common.io.ByteSource;
 import org.haiku.pkg.AttributeContext;
 
 import java.util.Arrays;
 
 public class RawInlineAttribute extends RawAttribute {
 
-    private byte[] rawValue;
+    private final byte[] rawValue;
 
     public RawInlineAttribute(AttributeId attributeId, byte[] rawValue) {
         super(attributeId);
@@ -39,8 +40,8 @@ public int hashCode() {
     }
 
     @Override
-    public byte[] getValue(AttributeContext context) {
-        return rawValue;
+    public ByteSource getValue(AttributeContext context) {
+        return ByteSource.wrap(rawValue);
     }
 
     @Override

diff --git a/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/output/AttributeWriter.java b/haikudepotserver-packagefile/src/main/java/org/haiku/pkg/output/AttributeWriter.java
@@ -1,11 +1,12 @@
 /*
- * Copyright 2018, Andrew Lindesay
+ * Copyright 2018-2021, Andrew Lindesay
  * Distributed under the terms of the MIT License.
  */
 
 package org.haiku.pkg.output;
 
 import com.google.common.base.Preconditions;
+import com.google.common.io.ByteSource;
 import org.haiku.pkg.AttributeContext;
 import org.haiku.pkg.HpkException;
 import org.haiku.pkg.model.Attribute;
@@ -43,24 +44,19 @@ private void write(int indent, AttributeContext context, Attribute attribute) th
 
         try {
             switch (attribute.getAttributeType()) {
-
                 case RAW:
-                    byte[] data = (byte[]) attribute.getValue(context);
-                    write(String.format("%d bytes",data.length));
+                    ByteSource byteSource = (ByteSource) attribute.getValue(context);
+                    write(String.format("%d bytes", byteSource.size()));
                     break;
-
                 case INT:
                     write(attribute.getValue(context).toString());
                     break;
-
                 case STRING:
                     write(attribute.getValue(context).toString());
                     break;
-
                 default:
                     write("???");
                     break;
-
             }
         }
         catch (HpkException e) {

diff --git a/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java b/haikudepotserver-packagefile/src/test/java/org/haiku/pkg/HpkgFileExtractorAttributeTest.java
@@ -1,12 +1,26 @@
+/*
+ * Copyright 2021, Andrew Lindesay
+ * Distributed under the terms of the MIT License.
+ */
+
 package org.haiku.pkg;
 
+import com.google.common.base.Preconditions;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.Hashing;
+import com.google.common.io.ByteSource;
 import org.fest.assertions.Assertions;
-import org.haiku.pkg.model.*;
+import org.haiku.pkg.model.Attribute;
+import org.haiku.pkg.model.AttributeId;
+import org.haiku.pkg.model.AttributeType;
+import org.haiku.pkg.model.IntAttribute;
 import org.junit.Test;
 
 import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
 
 public class HpkgFileExtractorAttributeTest extends AbstractHpkTest {
 
@@ -32,8 +46,39 @@ public void testReadFile() throws Exception {
             Assertions.assertThat(summaryAttribute.getAttributeId()).isEqualTo(AttributeId.PACKAGE_SUMMARY);
             Assertions.assertThat(summaryAttribute.getAttributeType()).isEqualTo(AttributeType.STRING);
             Assertions.assertThat(summaryAttribute.getValue(packageAttributeContext)).isEqualTo("An application to display Haiku usage tips");
+
+            // Pull out the actual binary to check.  The expected data results were obtained
+            // from a Haiku host with the package installed.
+            Attribute binaryDirectoryEntry = findByDirectoryEntries(tocAttributes, tocContext, List.of("apps", "Tipster"));
+            Attribute binaryData = binaryDirectoryEntry.getChildAttribute(AttributeId.DATA);
+            ByteSource binaryDataByteSource = (ByteSource) binaryData.getValue(tocContext);
+            Assertions.assertThat(binaryDataByteSource.size()).isEqualTo(153840L);
+            HashCode hashCode = binaryDataByteSource.hash(Hashing.md5());
+            Assertions.assertThat(hashCode.toString().toLowerCase(Locale.ROOT)).isEqualTo("13b16cd7d035ddda09a744c49a8ebdf2");
+        }
+    }
+
+    private Attribute findByDirectoryEntries(
+            List<Attribute> attributes,
+            AttributeContext context,
+            List<String> pathComponents) {
+        Preconditions.checkArgument(!pathComponents.isEmpty());
+        Optional<Attribute> resultOptional = attributes.stream()
+                .filter(a -> a.getAttributeId() == AttributeId.DIRECTORY_ENTRY)
+                .filter(a -> a.getValue(context).equals(pathComponents.get(0)))
+                .findFirst();
+
+        if (resultOptional.isPresent()) {
+            if (1 == pathComponents.size()) {
+                return resultOptional.get();
+            }
+            return findByDirectoryEntries(
+                    resultOptional.get().getChildAttributes(),
+                    context,
+                    pathComponents.subList(1, pathComponents.size()));
         }
 
+        throw new AssertionError("unable to find the diretory entry [" + pathComponents.get(0) + "]");
     }
 
     private List<Attribute> toList(AttributeIterator attributeIterator) {