diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index 414c944ad24dc..c1b8d51c255db 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -155,7 +155,6 @@ public final class IndexScopedSettings extends AbstractScopedSettings { MapperService.INDEX_MAPPING_DEPTH_LIMIT_SETTING, MapperService.INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING, MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING, - MapperService.INDEX_MAPPING_NON_TEXT_FIELDS_SHARED_INVERTED_INDEX, BitsetFilterCache.INDEX_LOAD_RANDOM_ACCESS_FILTERS_EAGERLY_SETTING, IndexModule.INDEX_STORE_TYPE_SETTING, IndexModule.INDEX_STORE_PRE_LOAD_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index ba8ef597d9814..83a6d9319c75a 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -44,7 +44,6 @@ import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_DOCS_LIMIT_SETTING; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NESTED_FIELDS_LIMIT_SETTING; -import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_NON_TEXT_FIELDS_SHARED_INVERTED_INDEX; import static org.elasticsearch.index.mapper.MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING; /** @@ -757,7 +756,6 @@ private void setRetentionLeaseMillis(final TimeValue retentionLease) { private volatile long mappingDepthLimit; private volatile long mappingFieldNameLengthLimit; private volatile long mappingDimensionFieldsLimit; - private final boolean mappingNonTextFieldsSharedInvertedIndex; /** * The maximum number of refresh listeners allows on this shard. @@ -902,7 +900,6 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti mappingDepthLimit = scopedSettings.get(INDEX_MAPPING_DEPTH_LIMIT_SETTING); mappingFieldNameLengthLimit = scopedSettings.get(INDEX_MAPPING_FIELD_NAME_LENGTH_LIMIT_SETTING); mappingDimensionFieldsLimit = scopedSettings.get(INDEX_MAPPING_DIMENSION_FIELDS_LIMIT_SETTING); - mappingNonTextFieldsSharedInvertedIndex = scopedSettings.get(INDEX_MAPPING_NON_TEXT_FIELDS_SHARED_INVERTED_INDEX); indexRouting = IndexRouting.fromIndexMetadata(indexMetadata); es87TSDBCodecEnabled = scopedSettings.get(TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING); @@ -1546,10 +1543,6 @@ private void setMappingDimensionFieldsLimit(long value) { this.mappingDimensionFieldsLimit = value; } - public boolean isMappingNonTextFieldsSharedInvertedIndex() { - return mappingNonTextFieldsSharedInvertedIndex; - } - /** * The bounds for {@code @timestamp} on this index or * {@code null} if there are no bounds. diff --git a/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java new file mode 100644 index 0000000000000..b6c18190e9325 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/mapper/AllFieldMapper.java @@ -0,0 +1,167 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.index.query.SearchExecutionContext; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; + +public class AllFieldMapper extends MetadataFieldMapper { + + static final byte FIELD_VALUE_SEPARATOR = 0; // nul code point + public static final String NAME = "_all"; + public static final AllFieldMapper ENABLED_INSTANCE = new AllFieldMapper(true); + private static final AllFieldMapper DISABLED_INSTANCE = new AllFieldMapper(false); + + private final boolean enabled; + + public static class Defaults { + public static final FieldType FIELD_TYPE; + + static { + FieldType ft = new FieldType(); + ft.setTokenized(false); + ft.setOmitNorms(true); + ft.setIndexOptions(IndexOptions.DOCS); + ft.setDocValuesType(DocValuesType.NONE); + ft.setStored(false); + FIELD_TYPE = freezeAndDeduplicateFieldType(ft); + } + + public static TextSearchInfo TEXT_SEARCH_INFO = new TextSearchInfo( + FIELD_TYPE, + null, + Lucene.KEYWORD_ANALYZER, + Lucene.KEYWORD_ANALYZER + ); + + } + + public static class Builder extends MetadataFieldMapper.Builder { + + private final Parameter enabled; + + public Builder() { + super(NAME); + this.enabled = Parameter.boolParam("enabled", true, m -> toType(m).enabled, false) + .setMergeValidator((previous, current, conflicts) -> previous == current); + } + + @Override + protected Parameter[] getParameters() { + return new Parameter[] { enabled }; + } + + @Override + public MetadataFieldMapper build() { + return enabled.getValue() ? ENABLED_INSTANCE : DISABLED_INSTANCE; + } + + private static AllFieldMapper toType(FieldMapper in) { + return (AllFieldMapper) in; + } + } + + public static final class AllFieldType extends MappedFieldType { + + static final AllFieldType INSTANCE = new AllFieldType(); + + private AllFieldType() { + super(NAME, true, false, false, Defaults.TEXT_SEARCH_INFO, Collections.emptyMap()); + } + + @Override + public String typeName() { + return NAME; + } + + @Override + public Query termQuery(Object value, SearchExecutionContext context) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support term queries"); + } + + @Override + public Query existsQuery(SearchExecutionContext context) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exists queries"); + } + + @Override + public ValueFetcher valueFetcher(SearchExecutionContext context, String format) { + throw new UnsupportedOperationException(); + } + } + + public static final TypeParser PARSER = new ConfigurableTypeParser(c -> DISABLED_INSTANCE, c -> new AllFieldMapper.Builder()); + + private AllFieldMapper(boolean enabled) { + super(AllFieldType.INSTANCE); + this.enabled = enabled; + } + + public boolean isEnabled() { + return enabled; + } + + @Override + public FieldMapper.Builder getMergeBuilder() { + return new Builder().init(this); + } + + @Override + public void postParse(DocumentParserContext context) throws IOException { + if (enabled == false) { + // not configured, so skip the validation + return; + } + + final List fields = context.rootDoc().getFields(); + for (int i = 0; i < fields.size(); i++) { + IndexableField indexableField = fields.get(i); + var mappedFieldType = context.mappingLookup().getFieldType(indexableField.name()); + if (mappedFieldType != null && "keyword".equals(mappedFieldType.typeName())) { + BytesRef value = toAllFieldTerm(indexableField.binaryValue(), new BytesRef(indexableField.name())); + if (value.length > MAX_TERM_LENGTH) { + // TODO + } + context.doc().add(new KeywordFieldMapper.KeywordField(NAME, value, Defaults.FIELD_TYPE)); + } + } + + } + + public static BytesRef toAllFieldTerm(BytesRef fieldValueBytes, BytesRef fieldNameBytes) { + BytesRefBuilder builder = new BytesRefBuilder(); + builder.append(fieldValueBytes); + builder.append(FIELD_VALUE_SEPARATOR); + builder.append(fieldNameBytes); + return builder.toBytesRef(); + } + + @Override + protected String contentType() { + return NAME; + } + + @Override + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { + return SourceLoader.SyntheticFieldLoader.NOTHING; + } +} diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java index 307550ca5ab53..d9cf1cbd9e6d7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java @@ -256,9 +256,8 @@ static Mapping createDynamicUpdate(DocumentParserContext context) { for (RuntimeField runtimeField : context.getDynamicRuntimeFields()) { rootBuilder.addRuntimeField(runtimeField); } - boolean nonTextFieldsSharedInvertedIndex = context.indexSettings().isMappingNonTextFieldsSharedInvertedIndex(); RootObjectMapper root = rootBuilder.build( - MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic(), false, nonTextFieldsSharedInvertedIndex) + MapperBuilderContext.root(context.mappingLookup().isSourceSynthetic(), false, context.mappingLookup().isALlFieldEnabled()) ); return context.mappingLookup().getMapping().mappingUpdate(root); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 7f4f68419f741..db1ddc79190dc 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -22,10 +22,14 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.Automata; import org.apache.lucene.util.automaton.Automaton; @@ -76,6 +80,7 @@ import static org.apache.lucene.index.IndexWriter.MAX_TERM_LENGTH; import static org.elasticsearch.core.Strings.format; +import static org.elasticsearch.index.mapper.AllFieldMapper.toAllFieldTerm; /** * A field mapper for keywords. This mapper accepts strings and indexes them as-is. @@ -307,7 +312,7 @@ private KeywordFieldType buildFieldType(MapperBuilderContext context, FieldType quoteAnalyzer, this, context.isSourceSynthetic(), - context.isSourceSynthetic() + context.isIndexIntoAllField() ); } @@ -318,6 +323,12 @@ public KeywordFieldMapper build(MapperBuilderContext context) { fieldtype.setIndexOptions(TextParams.toIndexOptions(this.indexed.getValue(), this.indexOptions.getValue())); fieldtype.setStored(this.stored.getValue()); fieldtype.setDocValuesType(this.hasDocValues.getValue() ? DocValuesType.SORTED_SET : DocValuesType.NONE); + + if (context.isIndexIntoAllField()) { + fieldtype.setIndexOptions(IndexOptions.NONE); + fieldtype.setOmitNorms(true); + } + if (fieldtype.equals(Defaults.FIELD_TYPE)) { // deduplicate in the common default case to save some memory fieldtype = Defaults.FIELD_TYPE; @@ -329,8 +340,7 @@ public KeywordFieldMapper build(MapperBuilderContext context) { multiFieldsBuilder.build(this, context), copyTo, context.isSourceSynthetic(), - this, - context.isMappingNonTextFieldsSharedInvertedIndex() + this ); } } @@ -351,7 +361,7 @@ public static final class KeywordFieldType extends StringFieldType { private final FieldValues scriptValues; private final boolean isDimension; private final boolean isSyntheticSource; - private final boolean mappingNonTextFieldsSharedInvertedIndex; + private final boolean indexIntoAllField; public KeywordFieldType( String name, @@ -361,13 +371,14 @@ public KeywordFieldType( NamedAnalyzer quoteAnalyzer, Builder builder, boolean isSyntheticSource, - boolean mappingNonTextFieldsSharedInvertedIndex + boolean indexIntoAllField ) { super( name, - fieldType.indexOptions() != IndexOptions.NONE && builder.indexCreatedVersion.isLegacyIndexVersion() == false, + isIndexed(fieldType, indexIntoAllField, builder), fieldType.stored(), builder.hasDocValues.getValue(), + // TODO: fix text info textSearchInfo(fieldType, builder.similarity.getValue(), searchAnalyzer, quoteAnalyzer), builder.meta.getValue() ); @@ -378,7 +389,12 @@ public KeywordFieldType( this.scriptValues = builder.scriptValues(); this.isDimension = builder.dimension.getValue(); this.isSyntheticSource = isSyntheticSource; - this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex; + this.indexIntoAllField = indexIntoAllField; + } + + static boolean isIndexed(FieldType fieldType, boolean indexIntoAllField, Builder builder) { + boolean isFieldIndexed = fieldType.indexOptions() != IndexOptions.NONE || indexIntoAllField; + return isFieldIndexed && builder.indexCreatedVersion.isLegacyIndexVersion() == false; } public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Map meta) { @@ -390,7 +406,7 @@ public KeywordFieldType(String name, boolean isIndexed, boolean hasDocValues, Ma this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.mappingNonTextFieldsSharedInvertedIndex = false; + this.indexIntoAllField = false; } public KeywordFieldType(String name) { @@ -413,7 +429,7 @@ public KeywordFieldType(String name, FieldType fieldType) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.mappingNonTextFieldsSharedInvertedIndex = false; + this.indexIntoAllField = false; } public KeywordFieldType(String name, NamedAnalyzer analyzer) { @@ -425,7 +441,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.scriptValues = null; this.isDimension = false; this.isSyntheticSource = false; - this.mappingNonTextFieldsSharedInvertedIndex = false; + this.indexIntoAllField = false; } @Override @@ -437,7 +453,11 @@ public boolean isSearchable() { public Query termQuery(Object value, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); if (isIndexed()) { - return super.termQuery(value, context); + if (indexIntoAllField) { + return new TermQuery(new Term(AllFieldMapper.NAME, toAllFieldTerm(indexedValueForSearch(value), new BytesRef(name())))); + } else { + return super.termQuery(value, context); + } } else { return SortedSetDocValuesField.newSlowExactQuery(name(), indexedValueForSearch(value)); } @@ -447,7 +467,14 @@ public Query termQuery(Object value, SearchExecutionContext context) { public Query termsQuery(Collection values, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); if (isIndexed()) { - return super.termsQuery(values, context); + if (indexIntoAllField) { + BytesRef[] bytesRefs = values.stream() + .map(value -> toAllFieldTerm(indexedValueForSearch(value), new BytesRef(name()))) + .toArray(BytesRef[]::new); + return new TermInSetQuery(AllFieldMapper.NAME, bytesRefs); + } else { + return super.termsQuery(values, context); + } } else { BytesRef[] bytesRefs = values.stream().map(this::indexedValueForSearch).toArray(BytesRef[]::new); return SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs); @@ -464,7 +491,17 @@ public Query rangeQuery( ) { failIfNotIndexedNorDocValuesFallback(context); if (isIndexed()) { - return super.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + if (indexIntoAllField) { + return new TermRangeQuery( + AllFieldMapper.NAME, + lowerTerm == null ? null : toAllFieldTerm(indexedValueForSearch(lowerTerm), new BytesRef(name())), + upperTerm == null ? null : toAllFieldTerm(indexedValueForSearch(upperTerm), new BytesRef(name())), + includeLower, + includeUpper + ); + } else { + return super.rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, context); + } } else { return SortedSetDocValuesField.newSlowRangeQuery( name(), @@ -859,7 +896,6 @@ public boolean hasNormalizer() { private final boolean storeIgnored; private final IndexAnalyzers indexAnalyzers; - private final boolean mappingNonTextFieldsSharedInvertedIndex; private KeywordFieldMapper( String simpleName, @@ -868,8 +904,7 @@ private KeywordFieldMapper( MultiFields multiFields, CopyTo copyTo, boolean storeIgnored, - Builder builder, - boolean mappingNonTextFieldsSharedInvertedIndex + Builder builder ) { super(simpleName, mappedFieldType, multiFields, copyTo, builder.script.get() != null, builder.onScriptError.getValue()); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; @@ -884,7 +919,6 @@ private KeywordFieldMapper( this.scriptCompiler = builder.scriptCompiler; this.indexCreatedVersion = builder.indexCreatedVersion; this.storeIgnored = storeIgnored; - this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex; } @Override diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java index 454939fd99f81..2aab7423de646 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperBuilderContext.java @@ -18,20 +18,20 @@ public class MapperBuilderContext { /** * The root context, to be used when building a tree of mappers */ - public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream, boolean nonTextFieldsSharedInvertedIndex) { - return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, nonTextFieldsSharedInvertedIndex); + public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDataStream, boolean indexIntoAllField) { + return new MapperBuilderContext(null, isSourceSynthetic, isDataStream, indexIntoAllField); } private final String path; private final boolean isSourceSynthetic; private final boolean isDataStream; - private final boolean mappingNonTextFieldsSharedInvertedIndex; + private final boolean indexIntoAllField; - MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean mappingNonTextFieldsSharedInvertedIndex) { + MapperBuilderContext(String path, boolean isSourceSynthetic, boolean isDataStream, boolean indexIntoAllField) { this.path = path; this.isSourceSynthetic = isSourceSynthetic; this.isDataStream = isDataStream; - this.mappingNonTextFieldsSharedInvertedIndex = mappingNonTextFieldsSharedInvertedIndex; + this.indexIntoAllField = indexIntoAllField; } /** @@ -40,7 +40,7 @@ public static MapperBuilderContext root(boolean isSourceSynthetic, boolean isDat * @return a new MapperBuilderContext with this context as its parent */ public MapperBuilderContext createChildContext(String name) { - return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, mappingNonTextFieldsSharedInvertedIndex); + return new MapperBuilderContext(buildFullName(name), isSourceSynthetic, isDataStream, indexIntoAllField); } /** @@ -67,7 +67,7 @@ public boolean isDataStream() { return isDataStream; } - public boolean isMappingNonTextFieldsSharedInvertedIndex() { - return mappingNonTextFieldsSharedInvertedIndex; + public boolean isIndexIntoAllField() { + return indexIntoAllField; } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java index 3092253ec4428..6811a8968e9e8 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperMergeContext.java @@ -20,8 +20,8 @@ public final class MapperMergeContext { /** * The root context, to be used when merging a tree of mappers */ - public static MapperMergeContext root(boolean isSourceSynthetic, boolean isDataStream) { - return new MapperMergeContext(MapperBuilderContext.root(isSourceSynthetic, isDataStream, false)); + public static MapperMergeContext root(boolean isSourceSynthetic, boolean isDataStream, boolean isAllFieldEnabled) { + return new MapperMergeContext(MapperBuilderContext.root(isSourceSynthetic, isDataStream, isAllFieldEnabled)); } /** diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java index d1d7a12de3050..b714eabbd2636 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MapperService.java @@ -121,11 +121,6 @@ public enum MergeReason { Property.Dynamic, Property.IndexScope ); - public static final Setting INDEX_MAPPING_NON_TEXT_FIELDS_SHARED_INVERTED_INDEX = Setting.boolSetting( - "index.mapping.non_text_fields.shared.inverted_index", - false, - Property.IndexScope - ); private final IndexAnalyzers indexAnalyzers; private final MappingParser mappingParser; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java index 4cbb6d610b2eb..74d94340f2fc7 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/Mapping.java @@ -124,6 +124,11 @@ private boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } + private boolean isAllFieldEnabled() { + AllFieldMapper sfm = (AllFieldMapper) metadataMappersByName.get(AllFieldMapper.NAME); + return sfm != null && sfm.isEnabled(); + } + public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { return root.syntheticFieldLoader(Arrays.stream(metadataMappers)); } @@ -136,7 +141,11 @@ public SourceLoader.SyntheticFieldLoader syntheticFieldLoader() { * @return the resulting merged mapping. */ Mapping merge(Mapping mergeWith, MergeReason reason) { - RootObjectMapper mergedRoot = root.merge(mergeWith.root, reason, MapperMergeContext.root(isSourceSynthetic(), false)); + RootObjectMapper mergedRoot = root.merge( + mergeWith.root, + reason, + MapperMergeContext.root(isSourceSynthetic(), false, isAllFieldEnabled()) + ); // When merging metadata fields as part of applying an index template, new field definitions // completely overwrite existing ones instead of being merged. This behavior matches how we @@ -148,7 +157,10 @@ Mapping merge(Mapping mergeWith, MergeReason reason) { if (mergeInto == null || reason == MergeReason.INDEX_TEMPLATE) { merged = metaMergeWith; } else { - merged = (MetadataFieldMapper) mergeInto.merge(metaMergeWith, MapperMergeContext.root(isSourceSynthetic(), false)); + merged = (MetadataFieldMapper) mergeInto.merge( + metaMergeWith, + MapperMergeContext.root(isSourceSynthetic(), false, isAllFieldEnabled()) + ); } mergedMetadataMappers.put(merged.getClass(), merged); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java index 4880ce5edc204..807bbcad5a01d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingLookup.java @@ -436,6 +436,11 @@ public boolean isSourceSynthetic() { return sfm != null && sfm.isSynthetic(); } + public boolean isALlFieldEnabled() { + AllFieldMapper allFieldMapper = mapping.getMetadataMapperByClass(AllFieldMapper.class); + return allFieldMapper != null && allFieldMapper.isEnabled(); + } + /** * Build something to load source {@code _source}. */ diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java index 96a4f3664036a..eb4c7de84595b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappingParser.java @@ -120,6 +120,7 @@ Mapping parse(@Nullable String type, Map mappingSource) throws M boolean isSourceSynthetic = mappingParserContext.getIndexSettings().getMode().isSyntheticSourceEnabled(); boolean isDataStream = false; + boolean nonTextFieldsSharedInvertedIndex = false; Iterator> iterator = mappingSource.entrySet().iterator(); while (iterator.hasNext()) { @@ -150,6 +151,9 @@ Mapping parse(@Nullable String type, Map mappingSource) throws M if (metadataFieldMapper instanceof DataStreamTimestampFieldMapper dsfm) { isDataStream = dsfm.isEnabled(); } + if (metadataFieldMapper instanceof AllFieldMapper allFieldMapper) { + nonTextFieldsSharedInvertedIndex = allFieldMapper.isEnabled(); + } } } @@ -177,7 +181,6 @@ Mapping parse(@Nullable String type, Map mappingSource) throws M checkNoRemainingFields(mappingSource, "Root mapping definition has unsupported parameters: "); } - boolean nonTextFieldsSharedInvertedIndex = mappingParserContext.getIndexSettings().isMappingNonTextFieldsSharedInvertedIndex(); return new Mapping( rootObjectMapper.build(MapperBuilderContext.root(isSourceSynthetic, isDataStream, nonTextFieldsSharedInvertedIndex)), metadataMappers.values().toArray(new MetadataFieldMapper[0]), diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java index 181852c2c3bc9..1fdf284d0eeb6 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -22,6 +22,7 @@ import org.elasticsearch.action.resync.TransportResyncReplicationAction; import org.elasticsearch.common.inject.AbstractModule; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.index.mapper.AllFieldMapper; import org.elasticsearch.index.mapper.BinaryFieldMapper; import org.elasticsearch.index.mapper.BooleanFieldMapper; import org.elasticsearch.index.mapper.BooleanScriptFieldType; @@ -252,6 +253,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); builtInMetadataMappers.put(DocCountFieldMapper.NAME, DocCountFieldMapper.PARSER); builtInMetadataMappers.put(DataStreamTimestampFieldMapper.NAME, DataStreamTimestampFieldMapper.PARSER); + builtInMetadataMappers.put(AllFieldMapper.NAME, AllFieldMapper.PARSER); // _field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, FieldNamesFieldMapper.PARSER); return Collections.unmodifiableMap(builtInMetadataMappers); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java new file mode 100644 index 0000000000000..dea366e3c7d18 --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/mapper/AllFieldMapperTests.java @@ -0,0 +1,92 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.mapper; + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.xcontent.XContentBuilder; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class AllFieldMapperTests extends MetadataMapperTestCase { + + public void testPostParseEnabled() throws IOException { + DocumentMapper docMapper = createDocumentMapper(allFieldMapping(true, b -> { + b.startObject("field1"); + b.field("type", "keyword"); + b.endObject(); + })); + + ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); + assertThat(doc.rootDoc().getFields(AllFieldMapper.NAME).size(), equalTo(1)); + assertThat(doc.rootDoc().getFields(AllFieldMapper.NAME).get(0).binaryValue(), equalTo(new BytesRef("value1\0field1"))); + + Query query = docMapper.mappers().getFieldType("field1").termQuery("value1", null); + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertThat(termQuery.getTerm().field(), equalTo(AllFieldMapper.NAME)); + assertThat(termQuery.getTerm().bytes(), equalTo(new BytesRef("value1\0field1"))); + } + + public void testPostParseDisabed() throws IOException { + DocumentMapper docMapper = createDocumentMapper(allFieldMapping(false, b -> { + b.startObject("field1"); + b.field("type", "keyword"); + b.endObject(); + })); + + ParsedDocument doc = docMapper.parse(source(b -> b.field("field1", "value1"))); + assertThat(doc.rootDoc().getFields(AllFieldMapper.NAME).size(), equalTo(0)); + + Query query = docMapper.mappers().getFieldType("field1").termQuery("value1", null); + assertThat(query, instanceOf(TermQuery.class)); + TermQuery termQuery = (TermQuery) query; + assertThat(termQuery.getTerm().field(), equalTo("field1")); + assertThat(termQuery.getTerm().bytes(), equalTo(new BytesRef("value1"))); + } + + @Override + protected String fieldName() { + return AllFieldMapper.NAME; + } + + @Override + protected boolean isConfigurable() { + return true; + } + + @Override + protected void registerParameters(ParameterChecker checker) throws IOException { + checker.registerConflictCheck( + "enabled", + allFieldMapping(true, b -> b.startObject("field1").field("type", "keyword").endObject()), + allFieldMapping(false, b -> b.startObject("field1").field("type", "keyword").endObject()) + ); + checker.registerConflictCheck( + "enabled", + allFieldMapping(false, b -> b.startObject("field1").field("type", "keyword").endObject()), + allFieldMapping(true, b -> b.startObject("field1").field("type", "keyword").endObject()) + ); + } + + private static XContentBuilder allFieldMapping(boolean enabled, CheckedConsumer propertiesBuilder) + throws IOException { + return topMapping(b -> { + b.startObject(AllFieldMapper.NAME).field("enabled", enabled).endObject(); + b.startObject("properties"); + propertiesBuilder.accept(b); + b.endObject(); + }); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java index 15390880c9290..7f94569f78783 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NestedObjectMapperTests.java @@ -1508,14 +1508,14 @@ public void testMergeNested() { MapperException e = expectThrows( MapperException.class, - () -> firstMapper.merge(secondMapper, MapperMergeContext.root(false, false)) + () -> firstMapper.merge(secondMapper, MapperMergeContext.root(false, false, false)) ); assertThat(e.getMessage(), containsString("[include_in_parent] parameter can't be updated on a nested object mapping")); NestedObjectMapper result = (NestedObjectMapper) firstMapper.merge( secondMapper, MapperService.MergeReason.INDEX_TEMPLATE, - MapperMergeContext.root(false, false) + MapperMergeContext.root(false, false, false) ); assertFalse(result.isIncludeInParent()); assertTrue(result.isIncludeInRoot()); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java index 6c233653c1fec..7fdca04dbe473 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ObjectMapperMergeTests.java @@ -41,7 +41,7 @@ public void testMerge() { ObjectMapper mergeWith = createMapping(false, true, true, true); // WHEN merging mappings - final ObjectMapper merged = rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false)); + final ObjectMapper merged = rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, false)); // THEN "baz" new field is added to merged mapping final ObjectMapper mergedFoo = (ObjectMapper) merged.getMapper("foo"); @@ -63,7 +63,7 @@ public void testMergeWhenDisablingField() { // THEN a MapperException is thrown with an excepted message MapperException e = expectThrows( MapperException.class, - () -> rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false)) + () -> rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, false)) ); assertEquals("the [enabled] parameter can't be updated for the object mapping [foo]", e.getMessage()); } @@ -75,7 +75,7 @@ public void testMergeDisabledField() { new ObjectMapper.Builder("disabled", Explicit.IMPLICIT_TRUE) ).build(MapperBuilderContext.root(false, false, false)); - RootObjectMapper merged = (RootObjectMapper) rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false)); + RootObjectMapper merged = (RootObjectMapper) rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, false)); assertFalse(((ObjectMapper) merged.getMapper("disabled")).isEnabled()); } @@ -84,14 +84,14 @@ public void testMergeEnabled() { MapperException e = expectThrows( MapperException.class, - () -> rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false)) + () -> rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, false)) ); assertEquals("the [enabled] parameter can't be updated for the object mapping [disabled]", e.getMessage()); ObjectMapper result = rootObjectMapper.merge( mergeWith, MapperService.MergeReason.INDEX_TEMPLATE, - MapperMergeContext.root(false, false) + MapperMergeContext.root(false, false, false) ); assertTrue(result.isEnabled()); } @@ -106,14 +106,14 @@ public void testMergeEnabledForRootMapper() { MapperException e = expectThrows( MapperException.class, - () -> firstMapper.merge(secondMapper, MapperMergeContext.root(false, false)) + () -> firstMapper.merge(secondMapper, MapperMergeContext.root(false, false, false)) ); assertEquals("the [enabled] parameter can't be updated for the object mapping [" + type + "]", e.getMessage()); ObjectMapper result = firstMapper.merge( secondMapper, MapperService.MergeReason.INDEX_TEMPLATE, - MapperMergeContext.root(false, false) + MapperMergeContext.root(false, false, false) ); assertFalse(result.isEnabled()); } @@ -128,7 +128,7 @@ public void testMergeDisabledRootMapper() { Collections.singletonMap("test", new TestRuntimeField("test", "long")) ).build(MapperBuilderContext.root(false, false, false)); - RootObjectMapper merged = (RootObjectMapper) rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false)); + RootObjectMapper merged = (RootObjectMapper) rootObjectMapper.merge(mergeWith, MapperMergeContext.root(false, false, false)); assertFalse(merged.isEnabled()); assertEquals(1, merged.runtimeFields().size()); assertEquals("test", merged.runtimeFields().iterator().next().name()); @@ -138,7 +138,7 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalseAtRoot() { RootObjectMapper mergeInto = createRootSubobjectFalseLeafWithDots(); RootObjectMapper mergeWith = createRootSubobjectFalseLeafWithDots(); - final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false)); + final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, false)); final KeywordFieldMapper keywordFieldMapper = (KeywordFieldMapper) merged.getMapper("host.name"); assertEquals("host.name", keywordFieldMapper.name()); @@ -153,7 +153,7 @@ public void testMergedFieldNamesFieldWithDotsSubobjectsFalse() { createObjectSubobjectsFalseLeafWithDots() ).build(MapperBuilderContext.root(false, false, false)); - final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false)); + final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, false)); ObjectMapper foo = (ObjectMapper) merged.getMapper("foo"); ObjectMapper metrics = (ObjectMapper) foo.getMapper("metrics"); @@ -168,7 +168,7 @@ public void testMergedFieldNamesMultiFields() { RootObjectMapper mergeWith = new RootObjectMapper.Builder("_doc", Explicit.IMPLICIT_TRUE).add(createTextKeywordMultiField("text")) .build(MapperBuilderContext.root(false, false, false)); - final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false)); + final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, false)); TextFieldMapper text = (TextFieldMapper) merged.getMapper("text"); assertEquals("text", text.name()); @@ -186,7 +186,7 @@ public void testMergedFieldNamesMultiFieldsWithinSubobjectsFalse() { createObjectSubobjectsFalseLeafWithMultiField() ).build(MapperBuilderContext.root(false, false, false)); - final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false)); + final ObjectMapper merged = mergeInto.merge(mergeWith, MapperMergeContext.root(false, false, false)); ObjectMapper foo = (ObjectMapper) merged.getMapper("foo"); ObjectMapper metrics = (ObjectMapper) foo.getMapper("metrics"); diff --git a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java index 28e047540eed5..61a59ecee9f43 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/ParametrizedMapperTests.java @@ -345,7 +345,7 @@ public void testMerging() { {"type":"test_mapper","fixed":true,"fixed2":true,"required":"value"}"""); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> mapper.merge(badMerge, MapperMergeContext.root(false, false)) + () -> mapper.merge(badMerge, MapperMergeContext.root(false, false, false)) ); String expectedError = """ Mapper for [field] conflicts with existing mapper: @@ -358,7 +358,7 @@ public void testMerging() { // TODO: should we have to include 'fixed' here? Or should updates take as 'defaults' the existing values? TestMapper goodMerge = fromMapping(""" {"type":"test_mapper","fixed":false,"variable":"updated","required":"value"}"""); - TestMapper merged = (TestMapper) mapper.merge(goodMerge, MapperMergeContext.root(false, false)); + TestMapper merged = (TestMapper) mapper.merge(goodMerge, MapperMergeContext.root(false, false, false)); assertEquals("{\"field\":" + mapping + "}", Strings.toString(mapper)); // original mapping is unaffected assertEquals(""" @@ -376,7 +376,7 @@ public void testMultifields() throws IOException { String addSubField = """ {"type":"test_mapper","variable":"foo","required":"value","fields":{"sub2":{"type":"keyword"}}}"""; TestMapper toMerge = fromMapping(addSubField); - TestMapper merged = (TestMapper) mapper.merge(toMerge, MapperMergeContext.root(false, false)); + TestMapper merged = (TestMapper) mapper.merge(toMerge, MapperMergeContext.root(false, false, false)); assertEquals(XContentHelper.stripWhitespace(""" { "field": { @@ -399,7 +399,7 @@ public void testMultifields() throws IOException { TestMapper badToMerge = fromMapping(badSubField); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> merged.merge(badToMerge, MapperMergeContext.root(false, false)) + () -> merged.merge(badToMerge, MapperMergeContext.root(false, false, false)) ); assertEquals("mapper [field.sub2] cannot be changed from type [keyword] to [binary]", e.getMessage()); } @@ -415,13 +415,13 @@ public void testCopyTo() { TestMapper toMerge = fromMapping(""" {"type":"test_mapper","variable":"updated","required":"value","copy_to":["foo","bar"]}"""); - TestMapper merged = (TestMapper) mapper.merge(toMerge, MapperMergeContext.root(false, false)); + TestMapper merged = (TestMapper) mapper.merge(toMerge, MapperMergeContext.root(false, false, false)); assertEquals(""" {"field":{"type":"test_mapper","variable":"updated","required":"value","copy_to":["foo","bar"]}}""", Strings.toString(merged)); TestMapper removeCopyTo = fromMapping(""" {"type":"test_mapper","variable":"updated","required":"value"}"""); - TestMapper noCopyTo = (TestMapper) merged.merge(removeCopyTo, MapperMergeContext.root(false, false)); + TestMapper noCopyTo = (TestMapper) merged.merge(removeCopyTo, MapperMergeContext.root(false, false, false)); assertEquals(""" {"field":{"type":"test_mapper","variable":"updated","required":"value"}}""", Strings.toString(noCopyTo)); } @@ -487,7 +487,7 @@ public void testCustomSerialization() { TestMapper toMerge = fromMapping(conflict); IllegalArgumentException e = expectThrows( IllegalArgumentException.class, - () -> mapper.merge(toMerge, MapperMergeContext.root(false, false)) + () -> mapper.merge(toMerge, MapperMergeContext.root(false, false, false)) ); assertEquals( "Mapper for [field] conflicts with existing mapper:\n" @@ -576,7 +576,7 @@ public void testAnalyzers() { TestMapper original = mapper; TestMapper toMerge = fromMapping(mapping); - e = expectThrows(IllegalArgumentException.class, () -> original.merge(toMerge, MapperMergeContext.root(false, false))); + e = expectThrows(IllegalArgumentException.class, () -> original.merge(toMerge, MapperMergeContext.root(false, false, false))); assertEquals( "Mapper for [field] conflicts with existing mapper:\n" + "\tCannot update parameter [analyzer] from [default] to [_standard]", e.getMessage()