Skip to content

Commit

Permalink
Upgrade lucene version (#5570)
Browse files Browse the repository at this point in the history
* Added bwc version 2.4.2

Signed-off-by: Daniel (dB.) Doubrovkine <[email protected]>

* Added 2.4.2.

Signed-off-by: Daniel (dB.) Doubrovkine <[email protected]>

* Update Lucene snapshot to 9.5.0-snapshot-d5cef1c

Signed-off-by: Suraj Singh <[email protected]>

* Update changelog entry

Signed-off-by: Suraj Singh <[email protected]>

* Add 2.4.2 bwc version

Signed-off-by: Suraj Singh <[email protected]>

* Internal changes post lucene upgrade

Signed-off-by: Suraj Singh <[email protected]>

Signed-off-by: Daniel (dB.) Doubrovkine <[email protected]>
Signed-off-by: Suraj Singh <[email protected]>
Co-authored-by: opensearch-ci-bot <[email protected]>
Co-authored-by: Daniel (dB.) Doubrovkine <[email protected]>
  • Loading branch information
3 people authored and ryanbogan committed Dec 14, 2022
1 parent 0f9b7a1 commit c7c908e
Show file tree
Hide file tree
Showing 59 changed files with 73 additions and 60 deletions.
1 change: 1 addition & 0 deletions .ci/bwcVersions
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@ BWC_VERSION:
- "2.3.1"
- "2.4.0"
- "2.4.1"
- "2.4.2"
- "2.5.0"
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bumps `protobuf-java` from 3.21.7 to 3.21.9 ([#5319](https://github.com/opensearch-project/OpenSearch/pull/5319))
- Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
- Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
- Update Apache Lucene to 9.5.0-snapshot-d5cef1c ([#5570](https://github.com/opensearch-project/OpenSearch/pull/5570))

### Changed
- [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
Expand Down
2 changes: 1 addition & 1 deletion buildSrc/version.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
opensearch = 3.0.0
lucene = 9.5.0-snapshot-a4ef70f
lucene = 9.5.0-snapshot-d5cef1c

bundled_jdk_vendor = adoptium
bundled_jdk = 19.0.1+10
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
afe877ebf5ec4be7d17636b695015c449a523a3b

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
670d8f48ea9cba542e263d3ec6c3e2a33accc561

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b1f42bad26470c8ef88096e0c8564a74223c52c9

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f3cf74fa91da5133667f8916f93071fed231f2ee

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e633c2d0cd677e4f1cef5aadc6bdc65e8e898d98

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
c12a2943e6f4977f15d489ac3e9802c5dfb3c4cc

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
5d257928a34e586a7de9fc7d4a013868f7a1db74

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
3d88f80ad07421b9470cb44a6f5b67dd47047b13

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
f2440fe126dad78e95f901c0f7a6eeb66da09938

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
51677b84f823e352ab366f6a6bf87de8816650c4

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
b11b5c54ab26152c0db003c7a514f4e6c6825fdd

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
58bd60d4c3ec753eef0d904601ab6d726633a8db

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
97f11221b89e37c575ed9538d88b1529872abd80

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
84980688c8eb9fbdb597a4291713ee630653392d

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
23d3d1eefec90950b34ddef988be21b8fdfeb415

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
e5f164991c11efeebf0697c76ffbe986af5341d5

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
8c382224253727794557200e97717b927ad8fa74

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
18cb84b504b8a57075efca72f4701aa8a720a057

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
76cc7f77a30864e853a6662a5b7f4937023bd5e6

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
70df807e8504f2fb1fe28ceaf33373e3de51aec8

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
75aaac030d36ddf7cdb09632fe1293c6ecd756ce

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
384e589a3d90773ff47ffbaa2797afe95609f183
11 changes: 3 additions & 8 deletions server/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,16 +83,11 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_2_1 = new Version(2020199, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_3_0 = new Version(2030099, org.apache.lucene.util.Version.LUCENE_9_3_0);
public static final Version V_2_4_0 = new Version(2040099, org.apache.lucene.util.Version.LUCENE_9_4_1);
public static final Version V_2_4_1 = new Version(2040199, org.apache.lucene.util.Version.LUCENE_9_4_2);

// UNRELEASED
public static final Version V_2_4_1 = new Version(
2040199,
org.apache.lucene.util.Version.fromBits(9, 4, 2) /** needs updated 9.5.0 snapshots */
);
public static final Version V_2_5_0 = new Version(
2050099,
org.apache.lucene.util.Version.fromBits(9, 4, 2) /** needs updated 9.5.0 snapshots */
);
public static final Version V_2_4_2 = new Version(2040299, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_2_5_0 = new Version(2050099, org.apache.lucene.util.Version.LUCENE_9_4_2);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_5_0);
public static final Version CURRENT = V_3_0_0;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@
* @opensearch.internal
*/
public class Lucene {
public static final String LATEST_CODEC = "Lucene94";
public static final String LATEST_CODEC = "Lucene95";

public static final String SOFT_DELETES_FIELD = "__soft_deletes";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@

import org.apache.logging.log4j.Logger;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec.Mode;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec.Mode;
import org.opensearch.common.Nullable;
import org.opensearch.common.collect.MapBuilder;
import org.opensearch.index.mapper.MapperService;
Expand All @@ -62,8 +62,8 @@ public class CodecService {
public CodecService(@Nullable MapperService mapperService, Logger logger) {
final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
if (mapperService == null) {
codecs.put(DEFAULT_CODEC, new Lucene94Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene94Codec(Mode.BEST_COMPRESSION));
codecs.put(DEFAULT_CODEC, new Lucene95Codec());
codecs.put(BEST_COMPRESSION_CODEC, new Lucene95Codec(Mode.BEST_COMPRESSION));
} else {
codecs.put(DEFAULT_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
codecs.put(BEST_COMPRESSION_CODEC, new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.DocValuesFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.codecs.lucene90.Lucene90DocValuesFormat;
import org.opensearch.common.lucene.Lucene;
import org.opensearch.index.mapper.CompletionFieldMapper;
Expand All @@ -53,7 +53,7 @@
*
* @opensearch.internal
*/
public class PerFieldMappingPostingFormatCodec extends Lucene94Codec {
public class PerFieldMappingPostingFormatCodec extends Lucene95Codec {
private final Logger logger;
private final MapperService mapperService;
private final DocValuesFormat dvFormat = new Lucene90DocValuesFormat();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,8 @@ public void close() throws IOException {
}

@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
in.visitDocument(docID, visitor);
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
in.document(docID, visitor);
}

@Override
Expand Down Expand Up @@ -268,11 +268,11 @@ private static class RecoverySourcePruningStoredFieldsReader extends FilterStore
}

@Override
public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
public void document(int docID, StoredFieldVisitor visitor) throws IOException {
if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
super.visitDocument(docID, visitor);
super.document(docID, visitor);
} else {
super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
super.document(docID, new FilterStoredFieldVisitor(visitor) {
@Override
public Status needsField(FieldInfo fieldInfo) throws IOException {
if (recoverySourceField.equals(fieldInfo.name)) {
Expand All @@ -293,7 +293,6 @@ public StoredFieldsReader getMergeInstance() {
public StoredFieldsReader clone() {
return new RecoverySourcePruningStoredFieldsReader(in.clone(), recoverySourceToKeep, recoverySourceField);
}

}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@
import org.apache.lucene.index.SortedNumericDocValues;
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.StoredFields;
import org.apache.lucene.index.TermVectors;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.VectorEncoding;
import org.apache.lucene.index.VectorSimilarityFunction;
Expand Down Expand Up @@ -201,6 +203,11 @@ public Fields getTermVectors(int docID) {
throw new UnsupportedOperationException();
}

@Override
public TermVectors termVectors() throws IOException {
throw new UnsupportedOperationException();
}

@Override
public int numDocs() {
return 1;
Expand Down Expand Up @@ -232,6 +239,11 @@ public void document(int docID, StoredFieldVisitor visitor) throws IOException {
}
}

@Override
public StoredFields storedFields() throws IOException {
throw new UnsupportedOperationException();
}

@Override
protected void doClose() {

Expand All @@ -251,4 +263,9 @@ public VectorValues getVectorValues(String field) throws IOException {
public TopDocs searchNearestVectors(String field, float[] target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
throw new UnsupportedOperationException();
}

@Override
public TopDocs searchNearestVectors(String field, BytesRef target, int k, Bits acceptDocs, int visitedLimit) throws IOException {
throw new UnsupportedOperationException();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -667,9 +667,9 @@ private void add(BooleanQuery.Builder q, String field, List<Term> current, Boole
} else {
// We don't apply prefix on synonyms
final TermAndBoost[] termAndBoosts = current.stream()
.map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
.map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
.toArray(TermAndBoost[]::new);
q.add(newSynonymQuery(termAndBoosts), operator);
q.add(newSynonymQuery(field, termAndBoosts), operator);
}
}

Expand Down Expand Up @@ -782,9 +782,9 @@ public Query next() {
} else {
// We don't apply prefix on synonyms
final TermAndBoost[] termAndBoosts = Arrays.stream(terms)
.map(t -> new TermAndBoost(t, BoostAttribute.DEFAULT_BOOST))
.map(t -> new TermAndBoost(t.bytes(), BoostAttribute.DEFAULT_BOOST))
.toArray(TermAndBoost[]::new);
queryPos = newSynonymQuery(termAndBoosts);
queryPos = newSynonymQuery(field, termAndBoosts);
}
}
if (queryPos != null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,10 +218,10 @@ private class BlendedQueryBuilder extends MatchQueryBuilder {
}

@Override
protected Query newSynonymQuery(TermAndBoost[] terms) {
protected Query newSynonymQuery(String field, TermAndBoost[] terms) {
BytesRef[] values = new BytesRef[terms.length];
for (int i = 0; i < terms.length; i++) {
values[i] = terms[i].term.bytes();
values[i] = terms[i].term;
}
return blendTerms(context, values, commonTermsCutoff, tieBreaker, lenient, blendedFields);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ public void execute(SearchContext context) {
// So we do a little hack here and pretend we're going to do merges in order to
// get better sequential access.
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) currentReaderContext.reader();
fieldReader = lf.getSequentialStoredFieldsReader()::visitDocument;
fieldReader = lf.getSequentialStoredFieldsReader()::document;
} else {
fieldReader = currentReaderContext.reader()::document;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ public void setSegmentAndDocument(LeafReaderContext context, int docId) {
// So we do a little hack here and pretend we're going to do merges in order to
// get better sequential access.
SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader();
fieldReader = lf.getSequentialStoredFieldsReader()::visitDocument;
fieldReader = lf.getSequentialStoredFieldsReader()::document;
} else {
fieldReader = context.reader()::document;
}
Expand Down
12 changes: 6 additions & 6 deletions server/src/test/java/org/opensearch/index/codec/CodecTests.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@

import org.apache.logging.log4j.LogManager;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.lucene94.Lucene94Codec;
import org.apache.lucene.codecs.lucene95.Lucene95Codec;
import org.apache.lucene.codecs.lucene90.Lucene90StoredFieldsFormat;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DirectoryReader;
Expand Down Expand Up @@ -65,21 +65,21 @@ public class CodecTests extends OpenSearchTestCase {
public void testResolveDefaultCodecs() throws Exception {
CodecService codecService = createCodecService();
assertThat(codecService.codec("default"), instanceOf(PerFieldMappingPostingFormatCodec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene94Codec.class));
assertThat(codecService.codec("default"), instanceOf(Lucene95Codec.class));
}

public void testDefault() throws Exception {
Codec codec = createCodecService().codec("default");
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_SPEED, codec);
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_SPEED, codec);
}

public void testBestCompression() throws Exception {
Codec codec = createCodecService().codec("best_compression");
assertStoredFieldsCompressionEquals(Lucene94Codec.Mode.BEST_COMPRESSION, codec);
assertStoredFieldsCompressionEquals(Lucene95Codec.Mode.BEST_COMPRESSION, codec);
}

// write some docs with it, inspect .si to see this was the used compression
private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Codec actual) throws Exception {
private void assertStoredFieldsCompressionEquals(Lucene95Codec.Mode expected, Codec actual) throws Exception {
Directory dir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(null);
iwc.setCodec(actual);
Expand All @@ -91,7 +91,7 @@ private void assertStoredFieldsCompressionEquals(Lucene94Codec.Mode expected, Co
SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader();
String v = sr.getSegmentInfo().info.getAttribute(Lucene90StoredFieldsFormat.MODE_KEY);
assertNotNull(v);
assertEquals(expected, Lucene94Codec.Mode.valueOf(v));
assertEquals(expected, Lucene95Codec.Mode.valueOf(v));
ir.close();
dir.close();
}
Expand Down
Loading

0 comments on commit c7c908e

Please sign in to comment.