From 43d5f5e20c4c0d0f618260c641cae87590f607ea Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Wed, 13 Dec 2023 09:20:12 +0100 Subject: [PATCH] Cut over stored fields to ZSTD for compression. This cuts over stored fields with `index.codec: best_speed` (default) to ZSTD with level 0 and blocks of at most 128 documents or 16kB, and `index.codec: best_compression` to ZSTD with level 9 and blocks of at most 4,096 documents or 512kB. Compared with the current codecs, this would yield ~ equal indexing speed, much better space efficiency and somewhat slower retrieval speed. This is deemed acceptable as we are currently quite conservative when it comes to trading retrieval speed for space efficiency. The Lucene codec infrastructure records the codec on a per-segment basis and ensures that this change is backward-compatible. Segments will get progressively migrated to ZSTD as they get merged in the background. Bindings for ZSTD are provided by JNA, which we are already relying on to have access to the standard library. This change is not complete yet. TODO: - Ship ZSTD as part of Elasticsearch instead of relying on it being installed on the system. - Figure out SecurityManager permissions. --- server/src/main/java/module-info.java | 1 + .../index/codec/CodecService.java | 13 +- .../index/codec/Elasticsearch813Codec.java | 129 ++++++++++++++++ .../index/codec/PerFieldMapperCodec.java | 15 +- .../elasticsearch/index/codec/zstd/Zstd.java | 86 +++++++++++ .../codec/zstd/Zstd813StoredFieldsFormat.java | 143 ++++++++++++++++++ .../services/org.apache.lucene.codecs.Codec | 1 + .../elasticsearch/index/codec/CodecTests.java | 38 ++--- .../index/codec/PerFieldMapperCodecTests.java | 6 +- ...estCompressionStoredFieldsFormatTests.java | 23 +++ ...td813BestSpeedStoredFieldsFormatTests.java | 23 +++ .../index/codec/zstd/ZstdTests.java | 90 +++++++++++ .../index/mapper/MapperServiceTestCase.java | 4 +- 13 files changed, 525 insertions(+), 47 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/codec/Elasticsearch813Codec.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd.java create mode 100644 server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd813StoredFieldsFormat.java create mode 100644 server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec create mode 100644 server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestCompressionStoredFieldsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestSpeedStoredFieldsFormatTests.java create mode 100644 server/src/test/java/org/elasticsearch/index/codec/zstd/ZstdTests.java diff --git a/server/src/main/java/module-info.java b/server/src/main/java/module-info.java index 613e6868b8e9f..74da123e3f426 100644 --- a/server/src/main/java/module-info.java +++ b/server/src/main/java/module-info.java @@ -241,6 +241,7 @@ exports org.elasticsearch.index.codec; exports org.elasticsearch.index.codec.tsdb; exports org.elasticsearch.index.codec.bloomfilter; + exports org.elasticsearch.index.codec.zstd; exports org.elasticsearch.index.engine; exports org.elasticsearch.index.fielddata; exports org.elasticsearch.index.fielddata.fieldcomparator; diff --git a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java index d4771ba74e0fb..c04665aec07a8 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/CodecService.java +++ b/server/src/main/java/org/elasticsearch/index/codec/CodecService.java @@ -9,9 +9,9 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.core.Nullable; +import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; import org.elasticsearch.index.mapper.MapperService; import java.util.HashMap; @@ -35,11 +35,14 @@ public class CodecService { public CodecService(@Nullable MapperService mapperService, BigArrays bigArrays) { final var codecs = new HashMap(); if (mapperService == null) { - codecs.put(DEFAULT_CODEC, new Lucene99Codec()); - codecs.put(BEST_COMPRESSION_CODEC, new Lucene99Codec(Lucene99Codec.Mode.BEST_COMPRESSION)); + codecs.put(DEFAULT_CODEC, new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED)); + codecs.put(BEST_COMPRESSION_CODEC, new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_COMPRESSION)); } else { - codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, bigArrays)); - codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_COMPRESSION, mapperService, bigArrays)); + codecs.put(DEFAULT_CODEC, new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, bigArrays)); + codecs.put( + BEST_COMPRESSION_CODEC, + new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_COMPRESSION, mapperService, bigArrays) + ); } codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault()); for (String codec : Codec.availableCodecs()) { diff --git a/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch813Codec.java b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch813Codec.java new file mode 100644 index 0000000000000..9196c98d5f0ea --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/Elasticsearch813Codec.java @@ -0,0 +1,129 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec; + +import org.apache.lucene.codecs.DocValuesFormat; +import org.apache.lucene.codecs.FilterCodec; +import org.apache.lucene.codecs.KnnVectorsFormat; +import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.StoredFieldsFormat; +import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; +import org.apache.lucene.codecs.lucene99.Lucene99Codec; +import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; +import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; +import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; +import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; +import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; +import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; + +/** + * Elasticsearch codec as of 8.13. This extends the Lucene 9.9 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See + * {@link Zstd813StoredFieldsFormat}. + */ +public class Elasticsearch813Codec extends FilterCodec { + + private final StoredFieldsFormat storedFieldsFormat; + + private final PostingsFormat defaultPostingsFormat; + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Elasticsearch813Codec.this.getPostingsFormatForField(field); + } + }; + + private final DocValuesFormat defaultDVFormat; + private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return Elasticsearch813Codec.this.getDocValuesFormatForField(field); + } + }; + + private final KnnVectorsFormat defaultKnnVectorsFormat; + private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() { + @Override + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return Elasticsearch813Codec.this.getKnnVectorsFormatForField(field); + } + }; + + /** Public no-arg constructor, needed for SPI loading at read-time. */ + public Elasticsearch813Codec() { + this(Zstd813StoredFieldsFormat.Mode.BEST_SPEED); + } + + /** + * Constructor. Takes a {@link Zstd813StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of worse space-efficiency or vice-versa. + */ + public Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode mode) { + super("Elasticsearch813", new Lucene99Codec()); + this.storedFieldsFormat = new Zstd813StoredFieldsFormat(mode); + this.defaultPostingsFormat = new Lucene99PostingsFormat(); + this.defaultDVFormat = new Lucene90DocValuesFormat(); + this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); + } + + @Override + public StoredFieldsFormat storedFieldsFormat() { + return storedFieldsFormat; + } + + @Override + public final PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public final KnnVectorsFormat knnVectorsFormat() { + return knnVectorsFormat; + } + + /** + * Returns the postings format that should be used for writing new segments of field. + * + *

The default implementation always returns "Lucene99". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation, + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultPostingsFormat; + } + + /** + * Returns the docvalues format that should be used for writing new segments of field + * . + * + *

The default implementation always returns "Lucene99". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + /** + * Returns the vectors format that should be used for writing new segments of field + * + *

The default implementation always returns "Lucene95". + * + *

WARNING: if you subclass, you are responsible for index backwards compatibility: + * future version of Lucene are only guaranteed to be able to read the default implementation. + */ + public KnnVectorsFormat getKnnVectorsFormatForField(String field) { + return defaultKnnVectorsFormat; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java index d2ca31fe6a197..8bf6e69dcfb7e 100644 --- a/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java +++ b/server/src/main/java/org/elasticsearch/index/codec/PerFieldMapperCodec.java @@ -13,13 +13,13 @@ import org.apache.lucene.codecs.KnnVectorsFormat; import org.apache.lucene.codecs.PostingsFormat; import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.codec.bloomfilter.ES87BloomFilterPostingsFormat; import org.elasticsearch.index.codec.tsdb.ES87TSDBDocValuesFormat; +import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.Mapper; @@ -37,20 +37,19 @@ * per index in real time via the mapping API. If no specific postings format or vector format is * configured for a specific field the default postings or vector format is used. */ -public final class PerFieldMapperCodec extends Lucene99Codec { +public final class PerFieldMapperCodec extends Elasticsearch813Codec { private final MapperService mapperService; private final DocValuesFormat docValuesFormat = new Lucene90DocValuesFormat(); private final ES87BloomFilterPostingsFormat bloomFilterPostingsFormat; private final ES87TSDBDocValuesFormat tsdbDocValuesFormat; - static { - assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMapperCodec.class) - : "PerFieldMapperCodec must subclass the latest lucene codec: " + Lucene.LATEST_CODEC; - } - - public PerFieldMapperCodec(Mode compressionMode, MapperService mapperService, BigArrays bigArrays) { + public PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode compressionMode, MapperService mapperService, BigArrays bigArrays) { super(compressionMode); + // If the below assertion fails, it is a sign that Lucene released a new codec. You must create a copy of the current Elasticsearch + // codec that delegates to this new Lucene codec, and make PerFieldMapperCodec extend this new Elasticsearch codec. + assert Codec.forName(Lucene.LATEST_CODEC).getClass() == delegate.getClass() + : "PerFieldMapperCodec must be on the latest lucene codec: " + Lucene.LATEST_CODEC; this.mapperService = mapperService; this.bloomFilterPostingsFormat = new ES87BloomFilterPostingsFormat(bigArrays, this::internalGetPostingsFormatForField); this.tsdbDocValuesFormat = new ES87TSDBDocValuesFormat(); diff --git a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd.java b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd.java new file mode 100644 index 0000000000000..3ed786bcfff1c --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd.java @@ -0,0 +1,86 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.zstd; + +import com.sun.jna.Library; +import com.sun.jna.Native; + +import java.nio.ByteBuffer; + +/** JNA bindings for ZSTD. */ +final class Zstd { + + // TODO: Move this under libs/ and make it public so that we can progressively replace all our usage of DEFLATE with ZSTD? + + private static final ZstdLibrary LIBRARY; + + static { + LIBRARY = Native.load("zstd", ZstdLibrary.class); + } + + private interface ZstdLibrary extends Library { + + long ZSTD_compressBound(int scrLen); + + long ZSTD_compress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen, int compressionLevel); + + boolean ZSTD_isError(long code); + + String ZSTD_getErrorName(long code); + + long ZSTD_decompress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen); + + } + + /** + * Compress the content of {@code src} into {@code dst} at compression level {@code level}, and return the number of compressed bytes. + * {@link ByteBuffer#position()} and {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. + */ + public static int compress(ByteBuffer dst, ByteBuffer src, int level) { + long ret = LIBRARY.ZSTD_compress(dst, dst.remaining(), src, src.remaining(), level); + if (LIBRARY.ZSTD_isError(ret)) { + throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalStateException("Integer overflow? ret=" + ret); + } + return (int) ret; + } + + /** + * Compress the content of {@code src} into {@code dst}, and return the number of decompressed bytes. {@link ByteBuffer#position()} and + * {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. + */ + public static int decompress(ByteBuffer dst, ByteBuffer src) { + long ret = LIBRARY.ZSTD_decompress(dst, dst.remaining(), src, src.remaining()); + if (LIBRARY.ZSTD_isError(ret)) { + throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalStateException("Integer overflow? ret=" + ret); + } + return (int) ret; + } + + /** + * Return the maximum number of compressed bytes given an input length. + */ + public static int getMaxCompressedLen(int srcLen) { + long ret = LIBRARY.ZSTD_compressBound(srcLen); + if (LIBRARY.ZSTD_isError(ret)) { + throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); + } else if (ret < 0 || ret > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + srcLen + + " bytes may require up to " + + Long.toUnsignedString(ret) + + " bytes, which overflows the maximum capacity of a ByteBuffer" + ); + } + return (int) ret; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd813StoredFieldsFormat.java b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd813StoredFieldsFormat.java new file mode 100644 index 0000000000000..06afeecc04ab0 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd813StoredFieldsFormat.java @@ -0,0 +1,143 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.zstd; + +import org.apache.lucene.codecs.compressing.CompressionMode; +import org.apache.lucene.codecs.compressing.Compressor; +import org.apache.lucene.codecs.compressing.Decompressor; +import org.apache.lucene.codecs.lucene90.compressing.Lucene90CompressingStoredFieldsFormat; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.store.ByteBuffersDataInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; + +import java.io.IOException; +import java.nio.ByteBuffer; + +/** + * {@link org.apache.lucene.codecs.StoredFieldsFormat} that compresses blocks of data using ZStandard. + * + * Unlike Lucene's default stored fields format, this format does not make use of dictionaries (even though ZStandard has great support for + * dictionaries!). This is mostly due to the fact that LZ4/DEFLATE have short sliding windows that they can use to find duplicate strings + * (64kB and 32kB respectively). In contrast, ZSTD doesn't have such a limitation and can better take advantage of large compression + * buffers. + */ +public final class Zstd813StoredFieldsFormat extends Lucene90CompressingStoredFieldsFormat { + + public enum Mode { + BEST_SPEED(0, 16 * 1024, 128), + BEST_COMPRESSION(9, 512 * 1024, 4096); + + final int level, blockSizeInBytes, blockDocCount; + + private Mode(int level, int blockSizeInBytes, int blockDocCount) { + this.level = level; + this.blockSizeInBytes = blockSizeInBytes; + this.blockDocCount = blockDocCount; + } + } + + public Zstd813StoredFieldsFormat(Mode mode) { + this(mode.level, mode.blockSizeInBytes, mode.blockDocCount); + } + + Zstd813StoredFieldsFormat(int level, int blockSizeInBytes, int blockDocCount) { + super("ZstdStoredFields813", new ZstdCompressionMode(level), blockSizeInBytes, blockDocCount, 10); + } + + private static class ZstdCompressionMode extends CompressionMode { + private final int level; + + ZstdCompressionMode(int level) { + this.level = level; + } + + @Override + public Compressor newCompressor() { + return new ZstdCompressor(level); + } + + @Override + public Decompressor newDecompressor() { + return new ZstdDecompressor(); + } + + @Override + public String toString() { + return "ZSTD(level=" + level + ")"; + } + } + + private static final class ZstdDecompressor extends Decompressor { + + byte[] compressed; + + ZstdDecompressor() { + compressed = BytesRef.EMPTY_BYTES; + } + + @Override + public void decompress(DataInput in, int originalLength, int offset, int length, BytesRef bytes) throws IOException { + final int compressedLength = in.readVInt(); + compressed = ArrayUtil.growNoCopy(compressed, compressedLength); + in.readBytes(compressed, 0, compressedLength); + bytes.bytes = ArrayUtil.growNoCopy(bytes.bytes, originalLength); + + final int l = Zstd.decompress(ByteBuffer.wrap(bytes.bytes), ByteBuffer.wrap(compressed, 0, compressedLength)); + if (l != originalLength) { + throw new CorruptIndexException("Corrupt", in); + } + bytes.offset = offset; + bytes.length = length; + } + + @Override + public Decompressor clone() { + return new ZstdDecompressor(); + } + } + + private static class ZstdCompressor extends Compressor { + + final int level; + byte[] buffer; + byte[] compressed; + + ZstdCompressor(int level) { + this.level = level; + compressed = BytesRef.EMPTY_BYTES; + buffer = BytesRef.EMPTY_BYTES; + } + + @Override + public void compress(ByteBuffersDataInput buffersInput, DataOutput out) throws IOException { + final int len = Math.toIntExact(buffersInput.length()); + + buffer = ArrayUtil.growNoCopy(buffer, len); + buffersInput.readBytes(buffer, 0, len); + + final int maxCompressedLength = Zstd.getMaxCompressedLen(len); + compressed = ArrayUtil.growNoCopy(compressed, maxCompressedLength); + + final int compressedLen = Zstd.compress( + ByteBuffer.wrap(compressed, 0, compressed.length), + ByteBuffer.wrap(buffer, 0, len), + level + ); + + out.writeVInt(compressedLen); + out.writeBytes(compressed, compressedLen); + } + + @Override + public void close() throws IOException {} + } +} diff --git a/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec new file mode 100644 index 0000000000000..2d2ff58a493d5 --- /dev/null +++ b/server/src/main/resources/META-INF/services/org.apache.lucene.codecs.Codec @@ -0,0 +1 @@ +org.elasticsearch.index.codec.Elasticsearch813Codec diff --git a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java index 625c536a1c0d5..65c73fd69e0c3 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/CodecTests.java @@ -9,14 +9,6 @@ package org.elasticsearch.index.codec; import org.apache.lucene.codecs.Codec; -import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DirectoryReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.SegmentReader; -import org.apache.lucene.store.Directory; import org.apache.lucene.tests.util.LuceneTestCase.SuppressCodecs; import org.elasticsearch.TransportVersion; import org.elasticsearch.common.settings.Settings; @@ -44,35 +36,23 @@ public class CodecTests extends ESTestCase { public void testResolveDefaultCodecs() throws Exception { CodecService codecService = createCodecService(); assertThat(codecService.codec("default"), instanceOf(PerFieldMapperCodec.class)); - assertThat(codecService.codec("default"), instanceOf(Lucene99Codec.class)); + assertThat(codecService.codec("default"), instanceOf(Elasticsearch813Codec.class)); } public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_SPEED, codec); + assertEquals( + "Zstd813StoredFieldsFormat(compressionMode=ZSTD(level=0), chunkSize=16384, maxDocsPerChunk=128, blockShift=10)", + codec.storedFieldsFormat().toString() + ); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertStoredFieldsCompressionEquals(Lucene99Codec.Mode.BEST_COMPRESSION, codec); - } - - // write some docs with it, inspect .si to see this was the used compression - private void assertStoredFieldsCompressionEquals(Lucene99Codec.Mode expected, Codec actual) throws Exception { - Directory dir = newDirectory(); - IndexWriterConfig iwc = newIndexWriterConfig(null); - iwc.setCodec(actual); - IndexWriter iw = new IndexWriter(dir, iwc); - iw.addDocument(new Document()); - iw.commit(); - iw.close(); - DirectoryReader ir = DirectoryReader.open(dir); - SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); - String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY); - assertNotNull(v); - assertEquals(expected, Lucene99Codec.Mode.valueOf(v)); - ir.close(); - dir.close(); + assertEquals( + "Zstd813StoredFieldsFormat(compressionMode=ZSTD(level=9), chunkSize=524288, maxDocsPerChunk=4096, blockShift=10)", + codec.storedFieldsFormat().toString() + ); } private CodecService createCodecService() throws IOException { diff --git a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java index e2a2c72d3eae3..ac084c5bc7784 100644 --- a/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java +++ b/server/src/test/java/org/elasticsearch/index/codec/PerFieldMapperCodecTests.java @@ -8,13 +8,13 @@ package org.elasticsearch.index.codec; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.compress.CompressedXContent; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.MapperTestUtils; +import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.test.ESTestCase; @@ -168,7 +168,7 @@ private PerFieldMapperCodec createCodec(boolean timestampField, boolean timeSeri """; mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); } - return new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); + return new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); } public void testUseES87TSDBEncodingSettingDisabled() throws IOException { @@ -207,7 +207,7 @@ private PerFieldMapperCodec createCodec(boolean enableES87TSDBCodec, boolean tim settings.put(IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING.getKey(), enableES87TSDBCodec); MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(), settings.build(), "test"); mapperService.merge("type", new CompressedXContent(mapping), MapperService.MergeReason.MAPPING_UPDATE); - return new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); + return new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE); } } diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestCompressionStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestCompressionStoredFieldsFormatTests.java new file mode 100644 index 0000000000000..72df3b2ed1788 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestCompressionStoredFieldsFormatTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.zstd; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase; +import org.elasticsearch.index.codec.Elasticsearch813Codec; + +public class Zstd813BestCompressionStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase { + + private final Codec codec = new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_COMPRESSION); + + @Override + protected Codec getCodec() { + return codec; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestSpeedStoredFieldsFormatTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestSpeedStoredFieldsFormatTests.java new file mode 100644 index 0000000000000..1c1b143c7edfa --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/Zstd813BestSpeedStoredFieldsFormatTests.java @@ -0,0 +1,23 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.zstd; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.tests.index.BaseStoredFieldsFormatTestCase; +import org.elasticsearch.index.codec.Elasticsearch813Codec; + +public class Zstd813BestSpeedStoredFieldsFormatTests extends BaseStoredFieldsFormatTestCase { + + private final Codec codec = new Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED); + + @Override + protected Codec getCodec() { + return codec; + } +} diff --git a/server/src/test/java/org/elasticsearch/index/codec/zstd/ZstdTests.java b/server/src/test/java/org/elasticsearch/index/codec/zstd/ZstdTests.java new file mode 100644 index 0000000000000..aea0da66cf1d6 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/codec/zstd/ZstdTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.codec.zstd; + +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.nio.ByteBuffer; + +public class ZstdTests extends ESTestCase { + + public void testMaxCompressedLength() { + assertThat(Zstd.getMaxCompressedLen(0), Matchers.greaterThanOrEqualTo(1)); + assertThat(Zstd.getMaxCompressedLen(100), Matchers.greaterThanOrEqualTo(100)); + expectThrows(IllegalArgumentException.class, () -> Zstd.getMaxCompressedLen(Integer.MAX_VALUE)); + expectThrows(IllegalArgumentException.class, () -> Zstd.getMaxCompressedLen(-1)); + expectThrows(IllegalArgumentException.class, () -> Zstd.getMaxCompressedLen(-100)); + expectThrows(IllegalArgumentException.class, () -> Zstd.getMaxCompressedLen(Integer.MIN_VALUE)); + } + + public void testEmpty() { + ByteBuffer uncompressed = ByteBuffer.allocate(0); + ByteBuffer compressed = ByteBuffer.allocate(Zstd.getMaxCompressedLen(0)); + int compressedLen = Zstd.compress(compressed, uncompressed, randomIntBetween(0, 10)); + assertThat(compressedLen, Matchers.greaterThan(0)); + compressed.limit(compressedLen); + ByteBuffer decompressed = ByteBuffer.allocate(0); + assertEquals(0, Zstd.decompress(decompressed, compressed)); + } + + public void testOneByte() { + ByteBuffer uncompressed = ByteBuffer.wrap(new byte[] { 'z' }); + ByteBuffer compressed = ByteBuffer.allocate(Zstd.getMaxCompressedLen(1)); + int compressedLen = Zstd.compress(compressed, uncompressed, randomIntBetween(0, 10)); + assertThat(compressedLen, Matchers.greaterThan(0)); + compressed.limit(compressedLen); + ByteBuffer decompressed = ByteBuffer.allocate(1); + assertEquals(1, Zstd.decompress(decompressed, compressed)); + assertEquals(uncompressed, decompressed); + } + + public void testBufferOverflowOnCompression() { + ByteBuffer uncompressed = ByteBuffer.wrap(new byte[] { 'z' }); + ByteBuffer compressed = ByteBuffer.allocate(0); + expectThrows(IllegalArgumentException.class, () -> Zstd.compress(compressed, uncompressed, randomIntBetween(0, 10))); + } + + public void testBufferOverflowOnDecompression() { + ByteBuffer uncompressed = ByteBuffer.wrap(new byte[] { 'z' }); + ByteBuffer compressed = ByteBuffer.allocate(Zstd.getMaxCompressedLen(1)); + int compressedLen = Zstd.compress(compressed, uncompressed, randomIntBetween(0, 10)); + assertThat(compressedLen, Matchers.greaterThan(0)); + compressed.limit(compressedLen); + ByteBuffer decompressed = ByteBuffer.allocate(0); + expectThrows(IllegalArgumentException.class, () -> Zstd.decompress(decompressed, compressed)); + } + + public void testNonZeroOffsets() { + byte[] b = new byte[100]; + b[60] = 'z'; + ByteBuffer uncompressed = ByteBuffer.wrap(b, 60, 1); + ByteBuffer compressed = ByteBuffer.allocate(42 + Zstd.getMaxCompressedLen(uncompressed.remaining())); + compressed.position(42); + int compressedLen = Zstd.compress(compressed, uncompressed, randomIntBetween(0, 10)); + compressed.limit(compressed.position() + compressedLen); + for (int i = 0; i < compressed.capacity(); ++i) { + if (i < compressed.position() || i > compressed.limit()) { + // Bytes outside of the range have not been updated + assertEquals(0, compressed.array()[i]); + } + } + ByteBuffer decompressed = ByteBuffer.allocate(80); + decompressed.position(10); + assertEquals(1, Zstd.decompress(decompressed, compressed)); + decompressed.limit(decompressed.position() + 1); + assertEquals(uncompressed, decompressed); + for (int i = 0; i < decompressed.capacity(); ++i) { + if (i < compressed.position() || i > compressed.limit()) { + // Bytes outside of the range have not been updated + assertEquals(0, compressed.array()[i]); + } + } + } +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java index 63a726d83f79e..9e0d0052ce897 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/mapper/MapperServiceTestCase.java @@ -10,7 +10,6 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; -import org.apache.lucene.codecs.lucene99.Lucene99Codec; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriterConfig; @@ -43,6 +42,7 @@ import org.elasticsearch.index.analysis.NamedAnalyzer; import org.elasticsearch.index.cache.bitset.BitsetFilterCache; import org.elasticsearch.index.codec.PerFieldMapperCodec; +import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; import org.elasticsearch.index.fielddata.FieldDataContext; import org.elasticsearch.index.fielddata.IndexFieldData; import org.elasticsearch.index.fielddata.IndexFieldDataCache; @@ -245,7 +245,7 @@ protected static void withLuceneIndex( CheckedConsumer test ) throws IOException { IndexWriterConfig iwc = new IndexWriterConfig(IndexShard.buildIndexAnalyzer(mapperService)).setCodec( - new PerFieldMapperCodec(Lucene99Codec.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE) + new PerFieldMapperCodec(Zstd813StoredFieldsFormat.Mode.BEST_SPEED, mapperService, BigArrays.NON_RECYCLING_INSTANCE) ); try (Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc)) { builder.accept(iw);