forked from elastic/elasticsearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Cut over stored fields to ZSTD for compression.
This cuts over stored fields with `index.codec: best_speed` (default) to ZSTD with level 0 and blocks of at most 128 documents or 16kB, and `index.codec: best_compression` to ZSTD with level 9 and blocks of at most 4,096 documents or 512kB. Compared with the current codecs, this would yield ~ equal indexing speed, much better space efficiency and somewhat slower retrieval speed. This is deemed acceptable as we are currently quite conservative when it comes to trading retrieval speed for space efficiency. The Lucene codec infrastructure records the codec on a per-segment basis and ensures that this change is backward-compatible. Segments will get progressively migrated to ZSTD as they get merged in the background. Bindings for ZSTD are provided by JNA, which we are already relying on to have access to the standard library. This change is not complete yet. TODO: - Ship ZSTD as part of Elasticsearch instead of relying on it being installed on the system. - Figure out SecurityManager permissions.
- Loading branch information
Showing
13 changed files
with
525 additions
and
47 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
129 changes: 129 additions & 0 deletions
129
server/src/main/java/org/elasticsearch/index/codec/Elasticsearch813Codec.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,129 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.codec; | ||
|
|
||
| import org.apache.lucene.codecs.DocValuesFormat; | ||
| import org.apache.lucene.codecs.FilterCodec; | ||
| import org.apache.lucene.codecs.KnnVectorsFormat; | ||
| import org.apache.lucene.codecs.PostingsFormat; | ||
| import org.apache.lucene.codecs.StoredFieldsFormat; | ||
| import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat; | ||
| import org.apache.lucene.codecs.lucene99.Lucene99Codec; | ||
| import org.apache.lucene.codecs.lucene99.Lucene99HnswVectorsFormat; | ||
| import org.apache.lucene.codecs.lucene99.Lucene99PostingsFormat; | ||
| import org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat; | ||
| import org.apache.lucene.codecs.perfield.PerFieldKnnVectorsFormat; | ||
| import org.apache.lucene.codecs.perfield.PerFieldPostingsFormat; | ||
| import org.elasticsearch.index.codec.zstd.Zstd813StoredFieldsFormat; | ||
|
|
||
| /** | ||
| * Elasticsearch codec as of 8.13. This extends the Lucene 9.9 codec to compressed stored fields with ZSTD instead of LZ4/DEFLATE. See | ||
| * {@link Zstd813StoredFieldsFormat}. | ||
| */ | ||
| public class Elasticsearch813Codec extends FilterCodec { | ||
|
|
||
| private final StoredFieldsFormat storedFieldsFormat; | ||
|
|
||
| private final PostingsFormat defaultPostingsFormat; | ||
| private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { | ||
| @Override | ||
| public PostingsFormat getPostingsFormatForField(String field) { | ||
| return Elasticsearch813Codec.this.getPostingsFormatForField(field); | ||
| } | ||
| }; | ||
|
|
||
| private final DocValuesFormat defaultDVFormat; | ||
| private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() { | ||
| @Override | ||
| public DocValuesFormat getDocValuesFormatForField(String field) { | ||
| return Elasticsearch813Codec.this.getDocValuesFormatForField(field); | ||
| } | ||
| }; | ||
|
|
||
| private final KnnVectorsFormat defaultKnnVectorsFormat; | ||
| private final KnnVectorsFormat knnVectorsFormat = new PerFieldKnnVectorsFormat() { | ||
| @Override | ||
| public KnnVectorsFormat getKnnVectorsFormatForField(String field) { | ||
| return Elasticsearch813Codec.this.getKnnVectorsFormatForField(field); | ||
| } | ||
| }; | ||
|
|
||
| /** Public no-arg constructor, needed for SPI loading at read-time. */ | ||
| public Elasticsearch813Codec() { | ||
| this(Zstd813StoredFieldsFormat.Mode.BEST_SPEED); | ||
| } | ||
|
|
||
| /** | ||
| * Constructor. Takes a {@link Zstd813StoredFieldsFormat.Mode} that describes whether to optimize for retrieval speed at the expense of worse space-efficiency or vice-versa. | ||
| */ | ||
| public Elasticsearch813Codec(Zstd813StoredFieldsFormat.Mode mode) { | ||
| super("Elasticsearch813", new Lucene99Codec()); | ||
| this.storedFieldsFormat = new Zstd813StoredFieldsFormat(mode); | ||
| this.defaultPostingsFormat = new Lucene99PostingsFormat(); | ||
| this.defaultDVFormat = new Lucene90DocValuesFormat(); | ||
| this.defaultKnnVectorsFormat = new Lucene99HnswVectorsFormat(); | ||
| } | ||
|
|
||
| @Override | ||
| public StoredFieldsFormat storedFieldsFormat() { | ||
| return storedFieldsFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public final PostingsFormat postingsFormat() { | ||
| return postingsFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public final DocValuesFormat docValuesFormat() { | ||
| return docValuesFormat; | ||
| } | ||
|
|
||
| @Override | ||
| public final KnnVectorsFormat knnVectorsFormat() { | ||
| return knnVectorsFormat; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the postings format that should be used for writing new segments of <code>field</code>. | ||
| * | ||
| * <p>The default implementation always returns "Lucene99". | ||
| * | ||
| * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility: | ||
| * future version of Lucene are only guaranteed to be able to read the default implementation, | ||
| */ | ||
| public PostingsFormat getPostingsFormatForField(String field) { | ||
| return defaultPostingsFormat; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the docvalues format that should be used for writing new segments of <code>field</code> | ||
| * . | ||
| * | ||
| * <p>The default implementation always returns "Lucene99". | ||
| * | ||
| * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility: | ||
| * future version of Lucene are only guaranteed to be able to read the default implementation. | ||
| */ | ||
| public DocValuesFormat getDocValuesFormatForField(String field) { | ||
| return defaultDVFormat; | ||
| } | ||
|
|
||
| /** | ||
| * Returns the vectors format that should be used for writing new segments of <code>field</code> | ||
| * | ||
| * <p>The default implementation always returns "Lucene95". | ||
| * | ||
| * <p><b>WARNING:</b> if you subclass, you are responsible for index backwards compatibility: | ||
| * future version of Lucene are only guaranteed to be able to read the default implementation. | ||
| */ | ||
| public KnnVectorsFormat getKnnVectorsFormatForField(String field) { | ||
| return defaultKnnVectorsFormat; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
86 changes: 86 additions & 0 deletions
86
server/src/main/java/org/elasticsearch/index/codec/zstd/Zstd.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,86 @@ | ||
| /* | ||
| * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| * or more contributor license agreements. Licensed under the Elastic License | ||
| * 2.0 and the Server Side Public License, v 1; you may not use this file except | ||
| * in compliance with, at your election, the Elastic License 2.0 or the Server | ||
| * Side Public License, v 1. | ||
| */ | ||
|
|
||
| package org.elasticsearch.index.codec.zstd; | ||
|
|
||
| import com.sun.jna.Library; | ||
| import com.sun.jna.Native; | ||
|
|
||
| import java.nio.ByteBuffer; | ||
|
|
||
| /** JNA bindings for ZSTD. */ | ||
| final class Zstd { | ||
|
|
||
| // TODO: Move this under libs/ and make it public so that we can progressively replace all our usage of DEFLATE with ZSTD? | ||
|
|
||
| private static final ZstdLibrary LIBRARY; | ||
|
|
||
| static { | ||
| LIBRARY = Native.load("zstd", ZstdLibrary.class); | ||
| } | ||
|
|
||
| private interface ZstdLibrary extends Library { | ||
|
|
||
| long ZSTD_compressBound(int scrLen); | ||
|
|
||
| long ZSTD_compress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen, int compressionLevel); | ||
|
|
||
| boolean ZSTD_isError(long code); | ||
|
|
||
| String ZSTD_getErrorName(long code); | ||
|
|
||
| long ZSTD_decompress(ByteBuffer dst, int dstLen, ByteBuffer src, int srcLen); | ||
|
|
||
| } | ||
|
|
||
| /** | ||
| * Compress the content of {@code src} into {@code dst} at compression level {@code level}, and return the number of compressed bytes. | ||
| * {@link ByteBuffer#position()} and {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. | ||
| */ | ||
| public static int compress(ByteBuffer dst, ByteBuffer src, int level) { | ||
| long ret = LIBRARY.ZSTD_compress(dst, dst.remaining(), src, src.remaining(), level); | ||
| if (LIBRARY.ZSTD_isError(ret)) { | ||
| throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); | ||
| } else if (ret < 0 || ret > Integer.MAX_VALUE) { | ||
| throw new IllegalStateException("Integer overflow? ret=" + ret); | ||
| } | ||
| return (int) ret; | ||
| } | ||
|
|
||
| /** | ||
| * Compress the content of {@code src} into {@code dst}, and return the number of decompressed bytes. {@link ByteBuffer#position()} and | ||
| * {@link ByteBuffer#limit()} of both {@link ByteBuffer}s are left unmodified. | ||
| */ | ||
| public static int decompress(ByteBuffer dst, ByteBuffer src) { | ||
| long ret = LIBRARY.ZSTD_decompress(dst, dst.remaining(), src, src.remaining()); | ||
| if (LIBRARY.ZSTD_isError(ret)) { | ||
| throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); | ||
| } else if (ret < 0 || ret > Integer.MAX_VALUE) { | ||
| throw new IllegalStateException("Integer overflow? ret=" + ret); | ||
| } | ||
| return (int) ret; | ||
| } | ||
|
|
||
| /** | ||
| * Return the maximum number of compressed bytes given an input length. | ||
| */ | ||
| public static int getMaxCompressedLen(int srcLen) { | ||
| long ret = LIBRARY.ZSTD_compressBound(srcLen); | ||
| if (LIBRARY.ZSTD_isError(ret)) { | ||
| throw new IllegalArgumentException(LIBRARY.ZSTD_getErrorName(ret)); | ||
| } else if (ret < 0 || ret > Integer.MAX_VALUE) { | ||
| throw new IllegalArgumentException( | ||
| srcLen | ||
| + " bytes may require up to " | ||
| + Long.toUnsignedString(ret) | ||
| + " bytes, which overflows the maximum capacity of a ByteBuffer" | ||
| ); | ||
| } | ||
| return (int) ret; | ||
| } | ||
| } |
Oops, something went wrong.