From ac57a43fc0cfaae554ae302110f9bc81934abd70 Mon Sep 17 00:00:00 2001 From: Martin Junghanns Date: Wed, 3 Apr 2024 12:00:17 +0200 Subject: [PATCH] Fallback to ArrayUtil.oversize if we overvflow The default way is to always grow the byte[] to the next power of two of the requested length. This can overflow even if there might still be enough space. We now fall back to Lucene's oversize implementation in case of an overflow and try if we can make it fit. --- .../core/loading/ChunkedAdjacencyLists.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java index 84d32dd552..a20eac3413 100644 --- a/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java +++ b/core/src/main/java/org/neo4j/gds/core/loading/ChunkedAdjacencyLists.java @@ -19,9 +19,10 @@ */ package org.neo4j.gds.core.loading; +import org.neo4j.gds.collections.ArrayUtil; import org.neo4j.gds.collections.DrainingIterator; -import org.neo4j.gds.collections.hsl.HugeSparseByteArrayList; import org.neo4j.gds.collections.hsa.HugeSparseCollections; +import org.neo4j.gds.collections.hsl.HugeSparseByteArrayList; import org.neo4j.gds.collections.hsl.HugeSparseIntList; import org.neo4j.gds.collections.hsl.HugeSparseLongArrayList; import org.neo4j.gds.collections.hsl.HugeSparseLongList; @@ -220,8 +221,7 @@ private long[] ensurePropertyCapacity(long index, int pos, int required, int pro ) ); } else if (currentProperties.length <= pos + required) { -// int newLength = ArrayUtil.oversize(pos + required, Long.BYTES); - int newLength = BitUtil.nextHighestPowerOfTwo(pos + required); + var newLength = getNewLength(pos + required); currentProperties = Arrays.copyOf(currentProperties, newLength); this.properties[propertyIndex].set(index, currentProperties); } @@ -229,6 +229,23 @@ private long[] ensurePropertyCapacity(long index, int pos, int required, int pro return currentProperties; } + static int getNewLength(int minLength) { + int newLength = BitUtil.nextHighestPowerOfTwo(minLength); + if (newLength < 0) { + // If we overflow, we try to grow by ~1/8th and :fingers_crossed: it's enough. + newLength = ArrayUtil.oversize(minLength, Long.BYTES); + } + if (newLength < 0) { + throw new IllegalArgumentException( + formatWithLocale( + "Encountered numeric overflow in compressed buffer. Required a minimum length of %d.", + minLength + ) + ); + } + return newLength; + } + public long capacity() { return targetLists.capacity(); }