Skip to content

Commit

Permalink
Supporting object fields in star-tree index
Browse files Browse the repository at this point in the history 
Signed-off-by: bharath-techie <[email protected]>
  • Loading branch information
@bharath-techie
bharath-techie committed Nov 27, 2024
1 parent 3da97f2 commit 1ae7561
Show file tree
Hide file tree
Showing 5 changed files with 387 additions and 6 deletions.
241 changes: 240 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/index/mapper/StarTreeMapperIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import org.opensearch.index.compositeindex.CompositeIndexSettings;
import org.opensearch.index.compositeindex.datacube.DataCubeDateTimeUnit;
import org.opensearch.index.compositeindex.datacube.DateDimension;
import org.opensearch.index.compositeindex.datacube.KeywordDimension;
import org.opensearch.index.compositeindex.datacube.MetricStat;
import org.opensearch.index.compositeindex.datacube.NumericDimension;
import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration;
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
import org.opensearch.index.compositeindex.datacube.startree.utils.date.DateTimeUnitAdapter;
Expand All @@ -41,6 +43,7 @@
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
Expand Down Expand Up @@ -113,6 +116,95 @@ private static XContentBuilder createMinimalTestMapping(boolean invalidDim, bool
}
}

private static XContentBuilder createNestedTestMapping() {
try {
return jsonBuilder().startObject()
.startObject("composite")
.startObject("startree-1")
.field("type", "star_tree")
.startObject("config")
.startObject("date_dimension")
.field("name", "timestamp")
.endObject()
.startArray("ordered_dimensions")
.startObject()
.field("name", "nested.nested1.status")
.endObject()
.startObject()
.field("name", "nested.nested1.keyword_dv")
.endObject()
.endArray()
.startArray("metrics")
.startObject()
.field("name", "nested3.numeric_dv")
.endObject()
.endArray()
.endObject()
.endObject()
.endObject()
.startObject("properties")
.startObject("timestamp")
.field("type", "date")
.endObject()
.startObject("nested3")
.startObject("properties")
.startObject("numeric_dv")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.startObject("numeric")
.field("type", "integer")
.field("doc_values", false)
.endObject()
.startObject("nested")
.startObject("properties")
.startObject("nested1")
.startObject("properties")
.startObject("status")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.startObject("keyword_dv")
.field("type", "keyword")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("nested-not-startree")
.startObject("properties")
.startObject("nested1")
.startObject("properties")
.startObject("status")
.field("type", "integer")
.field("doc_values", true)
.endObject()
.startObject("keyword_dv")
.field("type", "keyword")
.field("doc_values", true)
.endObject()
.endObject()
.endObject()
.endObject()
.endObject()
.startObject("keyword")
.field("type", "keyword")
.field("doc_values", false)
.endObject()
.startObject("ip")
.field("type", "ip")
.field("doc_values", false)
.endObject()
.endObject()
.endObject();
} catch (IOException e) {
throw new IllegalStateException(e);
}
}

private static XContentBuilder createDateTestMapping(boolean duplicate) {
try {
return jsonBuilder().startObject()
Expand Down Expand Up @@ -467,6 +559,46 @@ public void testValidCompositeIndexWithDates() {
}
}

public void testValidCompositeIndexWithNestedFields() {
prepareCreate(TEST_INDEX).setMapping(createNestedTestMapping()).setSettings(settings).get();
Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
for (IndicesService service : dataNodeInstances) {
final Index index = resolveIndex("test");
if (service.hasIndex(index)) {
IndexService indexService = service.indexService(index);
Set<CompositeMappedFieldType> fts = indexService.mapperService().getCompositeFieldTypes();

for (CompositeMappedFieldType ft : fts) {
assertTrue(ft instanceof StarTreeMapper.StarTreeFieldType);
StarTreeMapper.StarTreeFieldType starTreeFieldType = (StarTreeMapper.StarTreeFieldType) ft;
assertEquals("timestamp", starTreeFieldType.getDimensions().get(0).getField());
assertTrue(starTreeFieldType.getDimensions().get(0) instanceof DateDimension);
DateDimension dateDim = (DateDimension) starTreeFieldType.getDimensions().get(0);
List<DateTimeUnitRounding> expectedTimeUnits = Arrays.asList(
new DateTimeUnitAdapter(Rounding.DateTimeUnit.MINUTES_OF_HOUR),
DataCubeDateTimeUnit.HALF_HOUR_OF_DAY
);
for (int i = 0; i < dateDim.getIntervals().size(); i++) {
assertEquals(expectedTimeUnits.get(i).shortName(), dateDim.getSortedCalendarIntervals().get(i).shortName());
}
assertEquals("nested.nested1.status", starTreeFieldType.getDimensions().get(1).getField());
assertTrue(starTreeFieldType.getDimensions().get(1) instanceof NumericDimension);
assertEquals("nested.nested1.keyword_dv", starTreeFieldType.getDimensions().get(2).getField());
assertTrue(starTreeFieldType.getDimensions().get(2) instanceof KeywordDimension);
assertEquals("nested3.numeric_dv", starTreeFieldType.getMetrics().get(0).getField());
List<MetricStat> expectedMetrics = Arrays.asList(MetricStat.VALUE_COUNT, MetricStat.SUM, MetricStat.AVG);
assertEquals(expectedMetrics, starTreeFieldType.getMetrics().get(0).getMetrics());
assertEquals(10000, starTreeFieldType.getStarTreeConfig().maxLeafDocs());
assertEquals(
StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP,
starTreeFieldType.getStarTreeConfig().getBuildMode()
);
assertEquals(Collections.emptySet(), starTreeFieldType.getStarTreeConfig().getSkipStarNodeCreationInDims());
}
}
}
}

public void testValidCompositeIndexWithDuplicateDates() {
prepareCreate(TEST_INDEX).setMapping(createDateTestMapping(true)).setSettings(settings).get();
Iterable<IndicesService> dataNodeInstances = internalCluster().getDataNodeInstances(IndicesService.class);
Expand Down Expand Up @@ -555,11 +687,118 @@ public void testCompositeIndexWithArraysInCompositeField() throws IOException {
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
);
assertEquals(
"object mapping for [_doc] with array for [numeric_dv] cannot be accepted as field is also part of composite index mapping which does not accept arrays",
"object mapping for [_doc] with array for [numeric_dv] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
ex.getMessage()
);
}

public void testCompositeIndexWithArraysInNestedCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
XContentBuilder doc = jsonBuilder().startObject()
.field("timestamp", "2023-06-01T12:00:00Z")
.startArray("nested")
.startObject()
.startArray("nested1")
.startObject()
.field("status", 10)
.endObject()
.startObject()
.field("status", 10)
.endObject()
.startObject()
.field("status", 10)
.endObject()
.endArray()
.endObject()
.endArray()
.endObject();
// Index the document and refresh
MapperParsingException ex = expectThrows(
MapperParsingException.class,
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
);
assertEquals(
"object mapping for [_doc] with array for [nested] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
ex.getMessage()
);
}

public void testCompositeIndexWithArraysInChildNestedCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
XContentBuilder doc = jsonBuilder().startObject()
.field("timestamp", "2023-06-01T12:00:00Z")
.startObject("nested")
.startArray("nested1")
.startObject()
.field("status", 10)
.endObject()
.startObject()
.field("status", 10)
.endObject()
.startObject()
.field("status", 10)
.endObject()
.endArray()
.endObject()
.endObject();
// Index the document and refresh
MapperParsingException ex = expectThrows(
MapperParsingException.class,
() -> client().prepareIndex(TEST_INDEX).setSource(doc).get()
);
assertEquals(
"object mapping for [nested] with array for [nested1] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
ex.getMessage()
);
}

public void testCompositeIndexWithNestedArraysInNonCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createNestedTestMapping()).get();
// Attempt to index a document with an array field
XContentBuilder doc = jsonBuilder().startObject()
.field("timestamp", "2023-06-01T12:00:00Z")
.startObject("nested-not-startree")
.startArray("nested1")
.startObject()
.field("status", 10)
.endObject()
.startObject()
.field("status", 20)
.endObject()
.startObject()
.field("status", 30)
.endObject()
.endArray()
.endObject()
.endObject();

// Index the document and refresh
IndexResponse indexResponse = client().prepareIndex(TEST_INDEX).setSource(doc).get();

assertEquals(RestStatus.CREATED, indexResponse.status());

client().admin().indices().prepareRefresh(TEST_INDEX).get();
// Verify the document was indexed
SearchResponse searchResponse = client().prepareSearch(TEST_INDEX).setQuery(QueryBuilders.matchAllQuery()).get();

assertEquals(1, searchResponse.getHits().getTotalHits().value);

// Verify the values in the indexed document
SearchHit hit = searchResponse.getHits().getAt(0);
assertEquals("2023-06-01T12:00:00Z", hit.getSourceAsMap().get("timestamp"));

List<Object> values = (List<Object>) ((Map<String, Object>) (hit.getSourceAsMap().get("nested-not-startree"))).get("nested1");
assertEquals(3, values.size());
int i = 1;
for (Object val : values) {
Map<String, Object> valMap = (Map<String, Object>) val;
assertEquals(10 * i, valMap.get("status"));
i++;
}
}

public void testCompositeIndexWithArraysInNonCompositeField() throws IOException {
prepareCreate(TEST_INDEX).setSettings(settings).setMapping(createMinimalTestMapping(false, false, false)).get();
// Attempt to index a document with an array field
Expand Down
6 changes: 4 additions & 2 deletions server/src/main/java/org/opensearch/index/mapper/DocumentParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -662,11 +662,13 @@ private static void parseNonDynamicArray(ParseContext context, ObjectMapper mapp
XContentParser parser = context.parser();
XContentParser.Token token;
// block array values for composite index fields
if (context.indexSettings().isCompositeIndex() && context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)) {
if (context.indexSettings().isCompositeIndex()
&& (context.mapperService().isFieldPartOfCompositeIndex(arrayFieldName)
|| context.mapperService().isCompositeIndexFieldNestedField(context.path().pathAsText(arrayFieldName)))) {
throw new MapperParsingException(
String.format(
Locale.ROOT,
"object mapping for [%s] with array for [%s] cannot be accepted as field is also part of composite index mapping which does not accept arrays",
"object mapping for [%s] with array for [%s] cannot be accepted, as the field is also part of composite index mapping which does not accept arrays",
mapper.name(),
arrayFieldName
)
Expand Down
24 changes: 24 additions & 0 deletions server/src/main/java/org/opensearch/index/mapper/MapperService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,7 @@ public enum MergeReason {

private volatile Set<CompositeMappedFieldType> compositeMappedFieldTypes;
private volatile Set<String> fieldsPartOfCompositeMappings;
private volatile Set<String> nestedFieldsPartOfCompositeMappings;

public MapperService(
IndexSettings indexSettings,
Expand Down Expand Up @@ -554,10 +555,29 @@ private synchronized Map<String, DocumentMapper> internalMerge(DocumentMapper ma

private void buildCompositeFieldLookup() {
Set<String> fieldsPartOfCompositeMappings = new HashSet<>();
Set<String> nestedFieldsPartOfCompositeMappings = new HashSet<>();

for (CompositeMappedFieldType fieldType : compositeMappedFieldTypes) {
fieldsPartOfCompositeMappings.addAll(fieldType.fields());

for (String field : fieldType.fields()) {
String[] parts = field.split("\\.");
if (parts.length > 1) {
StringBuilder path = new StringBuilder();
for (int i = 0; i < parts.length; i++) {
if (i == 0) {
path.append(parts[i]);
} else {
path.append(".").append(parts[i]);
}
nestedFieldsPartOfCompositeMappings.add(path.toString());
}
}
}
}

this.fieldsPartOfCompositeMappings = fieldsPartOfCompositeMappings;
this.nestedFieldsPartOfCompositeMappings = nestedFieldsPartOfCompositeMappings;
}

private boolean assertSerialization(DocumentMapper mapper) {
Expand Down Expand Up @@ -690,6 +710,10 @@ public boolean isFieldPartOfCompositeIndex(String field) {
return fieldsPartOfCompositeMappings.contains(field);
}

public boolean isCompositeIndexFieldNestedField(String field) {
return nestedFieldsPartOfCompositeMappings.contains(field);
}

public ObjectMapper getObjectMapper(String name) {
return this.mapper == null ? null : this.mapper.objectMappers().get(name);
}
Expand Down
Loading

0 comments on commit 1ae7561

Please sign in to comment.