diff --git a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java index 81e19f6cdb1a3..1a088cf405932 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/CompositeIndexConstants.java @@ -19,13 +19,13 @@ public class CompositeIndexConstants { public static final long COMPOSITE_FIELD_MARKER = 0xC0950513F1E1DL; // Composite Field /** - * Represents the key to fetch number of non-star aggregated segment documents. + * Represents the key to fetch number of non-star aggregated records/entries */ - public static final String SEGMENT_DOCS_COUNT = "segmentDocsCount"; + public static final String SEGMENT_ENTRIES_COUNT = "segmentEntriesCount"; /** - * Represents the key to fetch number of total star tree documents in a segment. + * Represents the key to fetch number of total star tree records/entries in a segment. */ - public static final String STAR_TREE_DOCS_COUNT = "starTreeDocsCount"; + public static final String STAR_TREE_ENTRIES_COUNT = "starTreeEntriesCount"; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java index a1d638616f2aa..793f8275c04ae 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilder.java @@ -33,7 +33,8 @@ import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -177,13 +178,13 @@ public List generateMetricAggregatorInfos(MapperService ma * * @return list of MetricAggregatorInfo */ - public List getMetricReaders(SegmentWriteState state, Map fieldProducerMap) + public List getMetricReaders(SegmentWriteState state, Map fieldProducerMap) throws IOException { - List metricReaders = new ArrayList<>(); + List metricReaders = new ArrayList<>(); for (Metric metric : this.starTreeField.getMetrics()) { for (MetricStat metricStat : metric.getBaseMetrics()) { - SequentialDocValuesIterator metricReader; + SequentialValuesIterator metricReader; FieldInfo metricFieldInfo = state.fieldInfos.fieldInfo(metric.getField()); if (metricStat.equals(MetricStat.DOC_COUNT)) { // _doc_count is numeric field , so we convert to sortedNumericDocValues and get iterator @@ -192,8 +193,10 @@ public List getMetricReaders(SegmentWriteState stat if (metricFieldInfo == null) { metricFieldInfo = getFieldInfo(metric.getField(), DocValuesType.SORTED_NUMERIC); } - metricReader = new SequentialDocValuesIterator( - fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + metricReader = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator( + fieldProducerMap.get(metricFieldInfo.name).getSortedNumeric(metricFieldInfo) + ) ); } metricReaders.add(metricReader); @@ -218,17 +221,17 @@ public void build( long startTime = System.currentTimeMillis(); logger.debug("Star-tree build is a go with star tree field {}", starTreeField.getName()); - List metricReaders = getMetricReaders(writeState, fieldProducerMap); + List metricReaders = getMetricReaders(writeState, fieldProducerMap); List dimensionsSplitOrder = starTreeField.getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + SequentialValuesIterator[] dimensionReaders = new SequentialValuesIterator[dimensionsSplitOrder.size()]; for (int i = 0; i < numDimensions; i++) { String dimension = dimensionsSplitOrder.get(i).getField(); FieldInfo dimensionFieldInfo = writeState.fieldInfos.fieldInfo(dimension); if (dimensionFieldInfo == null) { dimensionFieldInfo = getFieldInfo(dimension, DocValuesType.SORTED_NUMERIC); } - dimensionReaders[i] = new SequentialDocValuesIterator( - fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo) + dimensionReaders[i] = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(fieldProducerMap.get(dimensionFieldInfo.name).getSortedNumeric(dimensionFieldInfo)) ); } Iterator starTreeDocumentIterator = sortAndAggregateSegmentDocuments(dimensionReaders, metricReaders); @@ -287,7 +290,7 @@ void appendDocumentsToStarTree(Iterator starTreeDocumentIterat } } - private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDocs) throws IOException { + private void serializeStarTree(int numSegmentStarTreeEntries, int numStarTreeEntries) throws IOException { // serialize the star tree data long dataFilePointer = dataOut.getFilePointer(); StarTreeWriter starTreeWriter = new StarTreeWriter(); @@ -299,8 +302,8 @@ private void serializeStarTree(int numSegmentStarTreeDocument, int numStarTreeDo starTreeField, metricAggregatorInfos, numStarTreeNodes, - numSegmentStarTreeDocument, - numStarTreeDocs, + numSegmentStarTreeEntries, + numStarTreeEntries, dataFilePointer, totalStarTreeDataLength ); @@ -395,27 +398,25 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) { */ protected StarTreeDocument getStarTreeDocument( int currentDocId, - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException { Long[] dims = new Long[numDimensions]; int i = 0; - for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); - Long val = dimensionDocValueIterator.value(currentDocId); + for (SequentialValuesIterator dimensionValueIterator : dimensionReaders) { + dimensionValueIterator.nextEntry(currentDocId); + Long val = dimensionValueIterator.value(currentDocId); dims[i] = val; i++; } i = 0; Object[] metrics = new Object[metricReaders.size()]; - for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + for (SequentialValuesIterator metricValuesIterator : metricReaders) { + metricValuesIterator.nextEntry(currentDocId); // As part of merge, we traverse the star tree doc values // The type of data stored in metric fields is different from the // actual indexing field they're based on - metrics[i] = metricAggregatorInfos.get(i) - .getValueAggregators() - .toAggregatedValueType(metricDocValuesIterator.value(currentDocId)); + metrics[i] = metricAggregatorInfos.get(i).getValueAggregators().toAggregatedValueType(metricValuesIterator.value(currentDocId)); i++; } return new StarTreeDocument(dims, metrics); @@ -463,8 +464,8 @@ protected StarTreeDocument getStarTreeDocument( * @return Iterator for the aggregated star-tree document */ public abstract Iterator sortAndAggregateSegmentDocuments( - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException; /** @@ -483,8 +484,8 @@ public abstract Iterator generateStarTreeDocumentsForStarNode( */ protected StarTreeDocument getSegmentStarTreeDocument( int currentDocId, - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException { Long[] dimensions = getStarTreeDimensionsFromSegment(currentDocId, dimensionReaders); Object[] metrics = getStarTreeMetricsFromSegment(currentDocId, metricReaders); @@ -497,12 +498,12 @@ protected StarTreeDocument getSegmentStarTreeDocument( * @return dimension values for each of the star-tree dimension * @throws IOException when we are unable to iterate to the next doc for the given dimension readers */ - Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIterator[] dimensionReaders) throws IOException { + Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialValuesIterator[] dimensionReaders) throws IOException { Long[] dimensions = new Long[numDimensions]; for (int i = 0; i < numDimensions; i++) { if (dimensionReaders[i] != null) { try { - dimensionReaders[i].nextDoc(currentDocId); + dimensionReaders[i].nextEntry(currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -524,13 +525,13 @@ Long[] getStarTreeDimensionsFromSegment(int currentDocId, SequentialDocValuesIte * @return metric values for each of the star-tree metric * @throws IOException when we are unable to iterate to the next doc for the given metric readers */ - private Object[] getStarTreeMetricsFromSegment(int currentDocId, List metricsReaders) throws IOException { + private Object[] getStarTreeMetricsFromSegment(int currentDocId, List metricsReaders) throws IOException { Object[] metrics = new Object[numMetrics]; for (int i = 0; i < numMetrics; i++) { - SequentialDocValuesIterator metricStatReader = metricsReaders.get(i); + SequentialValuesIterator metricStatReader = metricsReaders.get(i); if (metricStatReader != null) { try { - metricStatReader.nextDoc(currentDocId); + metricStatReader.nextEntry(currentDocId); } catch (IOException e) { logger.error("unable to iterate to next doc", e); throw new RuntimeException("unable to iterate to next doc", e); @@ -661,7 +662,7 @@ public StarTreeDocument reduceStarTreeDocuments(StarTreeDocument aggregatedDocum /** * Converts numericDocValues to sortedNumericDocValues and returns SequentialDocValuesIterator */ - private SequentialDocValuesIterator getIteratorForNumericField( + private SequentialValuesIterator getIteratorForNumericField( Map fieldProducerMap, FieldInfo fieldInfo, String name @@ -669,10 +670,10 @@ private SequentialDocValuesIterator getIteratorForNumericField( if (fieldInfo == null) { fieldInfo = getFieldInfo(name, DocValuesType.NUMERIC); } - SequentialDocValuesIterator sequentialDocValuesIterator; + SequentialValuesIterator sequentialDocValuesIterator; assert fieldProducerMap.containsKey(fieldInfo.name); - sequentialDocValuesIterator = new SequentialDocValuesIterator( - DocValues.singleton(fieldProducerMap.get(fieldInfo.name).getNumeric(fieldInfo)) + sequentialDocValuesIterator = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(DocValues.singleton(fieldProducerMap.get(fieldInfo.name).getNumeric(fieldInfo))) ); return sequentialDocValuesIterator; } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java index 1613b7c5a3ac0..66f305fc04133 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OffHeapStarTreeBuilder.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.DocValuesConsumer; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexOutput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.common.util.io.IOUtils; @@ -22,8 +21,9 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeDocumentsSorter; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -36,7 +36,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_ENTRIES_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; /** @@ -113,8 +113,8 @@ public void build( */ @Override public Iterator sortAndAggregateSegmentDocuments( - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException { // Write all dimensions for segment documents into the buffer, // and sort all documents using an int array @@ -143,19 +143,19 @@ public Iterator sortAndAggregateSegmentDocuments( * @return iterator of star tree documents */ Iterator mergeStarTrees(List starTreeValuesSubs) throws IOException { - int numDocs = 0; - int[] docIds; + int numEntries = 0; + int[] entryIds; try { for (StarTreeValues starTreeValues : starTreeValuesSubs) { List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[starTreeValues.getStarTreeField() + SequentialValuesIterator[] dimensionReaders = new SequentialValuesIterator[starTreeValues.getStarTreeField() .getDimensionsOrder() .size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } - List metricReaders = new ArrayList<>(); + List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { for (MetricStat metricStat : metric.getBaseMetrics()) { @@ -164,34 +164,35 @@ Iterator mergeStarTrees(List starTreeValuesSub metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } - int currentDocId = 0; - int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) + int currentEntryId = 0; + int numSegmentEntries = Integer.parseInt( + starTreeValues.getAttributes() + .getOrDefault(SEGMENT_ENTRIES_COUNT, String.valueOf(StarTreeValuesIterator.NO_MORE_ENTRIES)) ); - while (currentDocId < numSegmentDocs) { - StarTreeDocument starTreeDocument = getStarTreeDocument(currentDocId, dimensionReaders, metricReaders); + while (currentEntryId < numSegmentEntries) { + StarTreeDocument starTreeDocument = getStarTreeDocument(currentEntryId, dimensionReaders, metricReaders); segmentDocumentFileManager.writeStarTreeDocument(starTreeDocument, true); - numDocs++; - currentDocId++; + numEntries++; + currentEntryId++; } } - docIds = new int[numDocs]; - for (int i = 0; i < numDocs; i++) { - docIds[i] = i; + entryIds = new int[numEntries]; + for (int i = 0; i < numEntries; i++) { + entryIds[i] = i; } } catch (IOException ex) { segmentDocumentFileManager.close(); throw ex; } - if (numDocs == 0) { + if (numEntries == 0) { return Collections.emptyIterator(); } - return sortAndReduceDocuments(docIds, numDocs, true); + return sortAndReduceDocuments(entryIds, numEntries, true); } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java index 1a5c906ad413b..4ea62aba5756c 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/OnHeapStarTreeBuilder.java @@ -18,7 +18,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeDocument; import org.opensearch.index.compositeindex.datacube.startree.StarTreeField; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; import org.opensearch.index.mapper.MapperService; import java.io.IOException; @@ -29,7 +29,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicInteger; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_ENTRIES_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; /** @@ -91,8 +91,8 @@ public Long getDimensionValue(int docId, int dimensionId) { */ @Override public Iterator sortAndAggregateSegmentDocuments( - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException { StarTreeDocument[] starTreeDocuments = new StarTreeDocument[totalSegmentDocs]; for (int currentDocId = 0; currentDocId < totalSegmentDocs; currentDocId++) { @@ -134,14 +134,14 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + SequentialValuesIterator[] dimensionReaders = new SequentialValuesIterator[dimensionsSplitOrder.size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } - List metricReaders = new ArrayList<>(); + List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { for (MetricStat metricStat : metric.getBaseMetrics()) { @@ -150,14 +150,14 @@ StarTreeDocument[] getSegmentsStarTreeDocuments(List starTreeVal metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } int currentDocId = 0; int numSegmentDocs = Integer.parseInt( - starTreeValues.getAttributes().getOrDefault(SEGMENT_DOCS_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) + starTreeValues.getAttributes().getOrDefault(SEGMENT_ENTRIES_COUNT, String.valueOf(DocIdSetIterator.NO_MORE_DOCS)) ); while (currentDocId < numSegmentDocs) { starTreeDocuments.add(getStarTreeDocument(currentDocId, dimensionReaders, metricReaders)); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java index 779ed77b0540a..15ed153249243 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/builder/StarTreeDocsFileManager.java @@ -21,7 +21,6 @@ import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -55,11 +54,9 @@ public class StarTreeDocsFileManager extends AbstractDocumentsFileManager implem private RandomAccessInput starTreeDocsFileRandomInput; private IndexOutput starTreeDocsFileOutput; private final Map fileToEndDocIdMap; - private final List starTreeDocumentOffsets = new ArrayList<>(); private int currentFileStartDocId; private int numReadableStarTreeDocuments; private int starTreeFileCount = -1; - private int currBytes = 0; private final int fileCountMergeThreshold; private int numStarTreeDocs = 0; @@ -98,7 +95,11 @@ IndexOutput createStarTreeDocumentsFileOutput() throws IOException { public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; int numBytes = writeStarTreeDocument(starTreeDocument, starTreeDocsFileOutput, true); - addStarTreeDocumentOffset(numBytes); + if (docSizeInBytes == -1) { + docSizeInBytes = numBytes; + } else { + assert docSizeInBytes == numBytes; + } numStarTreeDocs++; } @@ -106,7 +107,14 @@ public void writeStarTreeDocument(StarTreeDocument starTreeDocument, boolean isA public StarTreeDocument readStarTreeDocument(int docId, boolean isAggregatedDoc) throws IOException { assert isAggregatedDoc == true; ensureDocumentReadable(docId); - return readStarTreeDocument(starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId), true); + return readStarTreeDocument(starTreeDocsFileRandomInput, getOffset(docId), true); + } + + /** + * Returns offset for the docId based on the current file start id + */ + private long getOffset(int docId) { + return (long) (docId - currentFileStartDocId) * docSizeInBytes; } @Override @@ -119,19 +127,10 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { public Long[] readDimensions(int docId) throws IOException { ensureDocumentReadable(docId); Long[] dims = new Long[starTreeField.getDimensionsOrder().size()]; - readDimensions(dims, starTreeDocsFileRandomInput, starTreeDocumentOffsets.get(docId)); + readDimensions(dims, starTreeDocsFileRandomInput, getOffset(docId)); return dims; } - private void addStarTreeDocumentOffset(int bytes) { - starTreeDocumentOffsets.add(currBytes); - currBytes += bytes; - if (docSizeInBytes == -1) { - docSizeInBytes = bytes; - } - assert docSizeInBytes == bytes; - } - /** * Load the correct StarTreeDocuments file based on the docId */ @@ -199,7 +198,6 @@ private void loadStarTreeDocumentFile(int docId) throws IOException { * If the operation is only for reading existing documents, a new file is not created. */ private void closeAndMaybeCreateNewFile(boolean shouldCreateFileForAppend, int numStarTreeDocs) throws IOException { - currBytes = 0; if (starTreeDocsFileOutput != null) { fileToEndDocIdMap.put(starTreeDocsFileOutput.getName(), numStarTreeDocs); IOUtils.close(starTreeDocsFileOutput); @@ -232,7 +230,6 @@ private void mergeFiles(int numStarTreeDocs) throws IOException { deleteOldFiles(); fileToEndDocIdMap.clear(); fileToEndDocIdMap.put(mergedOutput.getName(), numStarTreeDocs); - resetStarTreeDocumentOffsets(); } } @@ -259,17 +256,6 @@ private void deleteOldFiles() throws IOException { } } - /** - * Reset the star tree document offsets based on the merged file - */ - private void resetStarTreeDocumentOffsets() { - int curr = 0; - for (int i = 0; i < starTreeDocumentOffsets.size(); i++) { - starTreeDocumentOffsets.set(i, curr); - curr += docSizeInBytes; - } - } - @Override public void close() { try { @@ -288,7 +274,6 @@ public void close() { tmpDirectory.deleteFile(file); } catch (IOException ignored) {} // similar to IOUtils.deleteFilesIgnoringExceptions } - starTreeDocumentOffsets.clear(); fileToEndDocIdMap.clear(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java index e5890be3ccb5b..afd77a453c9f2 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/StarTreeWriter.java @@ -53,8 +53,8 @@ public long writeStarTree(IndexOutput dataOut, InMemoryTreeNode rootNode, int nu * @param starTreeField star tree field * @param metricAggregatorInfos metric aggregator infos * @param numNodes number of nodes in the star tree - * @param segmentAggregatedCount segment aggregated count - * @param numStarTreeDocs the total number of star tree documents for the segment + * @param segmentEntriesCount non star record/entries count + * @param totalStarTreeEntriesCount the total number of star tree records/entries for the segment * @param dataFilePointer data file pointer * @param dataFileLength data file length * @throws IOException when star-tree data serialization fails @@ -64,8 +64,8 @@ public void writeStarTreeMetadata( StarTreeField starTreeField, List metricAggregatorInfos, Integer numNodes, - Integer segmentAggregatedCount, - Integer numStarTreeDocs, + Integer segmentEntriesCount, + Integer totalStarTreeEntriesCount, long dataFilePointer, long dataFileLength ) throws IOException { @@ -74,8 +74,8 @@ public void writeStarTreeMetadata( starTreeField, metricAggregatorInfos, numNodes, - segmentAggregatedCount, - numStarTreeDocs, + segmentEntriesCount, + totalStarTreeEntriesCount, dataFilePointer, dataFileLength ); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java index 7352c215ee390..839b451d7be36 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadata.java @@ -72,14 +72,14 @@ public class StarTreeMetadata extends CompositeIndexMetadata { private final List metrics; /** - * The total number of documents aggregated in this star-tree segment. + * The total number of entries/records aggregated in this star-tree segment. */ - private final int segmentAggregatedDocCount; + private final int segmentAggregatedEntriesCount; /** - * The maximum number of documents allowed in a leaf node. + * The maximum number of entries/records allowed in a leaf node. */ - private final int maxLeafDocs; + private final int maxLeafEntries; /** * Set of dimensions for which star node creation should be skipped. @@ -102,9 +102,9 @@ public class StarTreeMetadata extends CompositeIndexMetadata { private final long dataLength; /** - * The number of star tree documents in the star tree. + * The number of star tree entries/records in the star tree. */ - private final int starTreeDocCount; + private final int starTreeEntriesCount; /** * A star tree metadata constructor to initialize star tree metadata from the segment file (.cim) using index input. @@ -130,9 +130,9 @@ public StarTreeMetadata( this.numberOfNodes = readNumberOfNodes(); this.dimensionFields = readStarTreeDimensions(); this.metrics = readMetricEntries(); - this.segmentAggregatedDocCount = readSegmentAggregatedDocCount(); - this.starTreeDocCount = readStarTreeDocCount(); - this.maxLeafDocs = readMaxLeafDocs(); + this.segmentAggregatedEntriesCount = readSegmentEntriesCount(); + this.starTreeEntriesCount = readStarTreeEntriesCount(); + this.maxLeafEntries = readMaxLeafDocs(); this.skipStarNodeCreationInDims = readSkipStarNodeCreationInDims(); this.starTreeBuildMode = readBuildMode(); this.dataStartFilePointer = readDataStartFilePointer(); @@ -153,9 +153,9 @@ public StarTreeMetadata( * @param version The version of the star tree stored in the segments. * @param dimensionFields list of dimension fields * @param metrics list of metric entries - * @param segmentAggregatedDocCount segment aggregated doc count - * @param starTreeDocCount the total number of star tree documents for the segment - * @param maxLeafDocs max leaf docs + * @param segmentAggregatedEntriesCount segment aggregated records/entries count + * @param starTreeEntriesCount the total number of star tree records/entries for the segment + * @param maxLeafEntries max leaf records/entries * @param skipStarNodeCreationInDims set of dimensions to skip star node creation * @param starTreeBuildMode star tree build mode * @param dataStartFilePointer star file pointer to the associated star tree data in (.cid) file @@ -169,9 +169,9 @@ public StarTreeMetadata( Integer numberOfNodes, List dimensionFields, List metrics, - Integer segmentAggregatedDocCount, - Integer starTreeDocCount, - Integer maxLeafDocs, + Integer segmentAggregatedEntriesCount, + Integer starTreeEntriesCount, + Integer maxLeafEntries, Set skipStarNodeCreationInDims, StarTreeFieldConfiguration.StarTreeBuildMode starTreeBuildMode, long dataStartFilePointer, @@ -185,9 +185,9 @@ public StarTreeMetadata( this.numberOfNodes = numberOfNodes; this.dimensionFields = dimensionFields; this.metrics = metrics; - this.segmentAggregatedDocCount = segmentAggregatedDocCount; - this.starTreeDocCount = starTreeDocCount; - this.maxLeafDocs = maxLeafDocs; + this.segmentAggregatedEntriesCount = segmentAggregatedEntriesCount; + this.starTreeEntriesCount = starTreeEntriesCount; + this.maxLeafEntries = maxLeafEntries; this.skipStarNodeCreationInDims = skipStarNodeCreationInDims; this.starTreeBuildMode = starTreeBuildMode; this.dataStartFilePointer = dataStartFilePointer; @@ -253,11 +253,11 @@ private void addEligibleDerivedMetrics(List metricStatsList) { } } - private int readSegmentAggregatedDocCount() throws IOException { + private int readSegmentEntriesCount() throws IOException { return meta.readVInt(); } - private Integer readStarTreeDocCount() throws IOException { + private Integer readStarTreeEntriesCount() throws IOException { return meta.readVInt(); } @@ -328,21 +328,21 @@ public List getMetrics() { } /** - * Returns the aggregated document count for the star-tree. + * Returns the aggregated non-star entries/records count for the star-tree. * - * @return the aggregated document count for the star-tree. + * @return the aggregated non-star entries/records count for the star-tree. */ - public int getSegmentAggregatedDocCount() { - return segmentAggregatedDocCount; + public int getSegmentEntriesCount() { + return segmentAggregatedEntriesCount; } /** - * Returns the total number of star tree documents in the segment + * Returns the total number of star tree entries/records in the segment * * @return the number of star tree documents in the segment */ - public int getStarTreeDocCount() { - return starTreeDocCount; + public int getStarTreeEntriesCount() { + return starTreeEntriesCount; } /** @@ -350,8 +350,8 @@ public int getStarTreeDocCount() { * * @return the max leaf docs. */ - public int getMaxLeafDocs() { - return maxLeafDocs; + public int getMaxLeafEntries() { + return maxLeafEntries; } /** diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java index 1c04350e25047..893e04d0b1794 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataWriter.java @@ -39,8 +39,9 @@ public class StarTreeMetadataWriter { * @param metaOut the IndexOutput to write the metadata * @param starTreeField the star-tree field * @param metricAggregatorInfos the list of metric aggregator information - * @param segmentAggregatedCount the aggregated document count for the segment * @param numNodes number of nodes in the star tree + * @param numSegmentEntries the aggregated non-star entries count for the segment + * @param numStarTreeEntries the aggregated total star tree entries count for the segment * @param dataFilePointer the file pointer to the start of the star tree data * @param dataFileLength the length of the star tree data file * @throws IOException if an I/O error occurs while serializing the metadata @@ -50,8 +51,8 @@ public static void writeStarTreeMetadata( StarTreeField starTreeField, List metricAggregatorInfos, Integer numNodes, - Integer segmentAggregatedCount, - Integer numStarTreeDocs, + Integer numSegmentEntries, + Integer numStarTreeEntries, long dataFilePointer, long dataFileLength ) throws IOException { @@ -66,8 +67,8 @@ public static void writeStarTreeMetadata( metricAggregatorInfos, starTreeField, numNodes, - segmentAggregatedCount, - numStarTreeDocs, + numSegmentEntries, + numStarTreeEntries, dataFilePointer, dataFileLength ); @@ -101,8 +102,8 @@ private static void writeMetaHeader(IndexOutput metaOut) throws IOException { * @param metricAggregatorInfos the list of metric aggregator information * @param starTreeField the star tree field * @param numNodes number of nodes in the star tree - * @param segmentAggregatedDocCount the aggregated document count for the segment - * @param numStarTreeDocs the total number of star tree documents for the segment + * @param segmentEntriesCount the aggregated document count for the segment + * @param numStarTreeEntries the total number of star tree documents for the segment * @param dataFilePointer the file pointer to the start of the star-tree data * @param dataFileLength the length of the star-tree data file * @throws IOException if an I/O error occurs while writing the metadata @@ -112,8 +113,8 @@ private static void writeMeta( List metricAggregatorInfos, StarTreeField starTreeField, int numNodes, - Integer segmentAggregatedDocCount, - Integer numStarTreeDocs, + Integer segmentEntriesCount, + Integer numStarTreeEntries, long dataFilePointer, long dataFileLength ) throws IOException { @@ -147,11 +148,11 @@ private static void writeMeta( metaOut.writeVInt(metricStatOrdinal); } - // segment aggregated document count - metaOut.writeVInt(segmentAggregatedDocCount); + // segment aggregated entries/records count + metaOut.writeVInt(segmentEntriesCount); - // segment star tree document count - metaOut.writeVInt(numStarTreeDocs); + // segment star tree entries/records count + metaOut.writeVInt(numStarTreeEntries); // max leaf docs metaOut.writeVInt(starTreeField.getStarTreeConfig().maxLeafDocs()); diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java index df2ce9096bfc1..56f81dd24ca50 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNode.java @@ -60,10 +60,10 @@ public class FixedLengthStarTreeNode implements StarTreeNode { // Byte offsets for each field in the serialized data static final int DIMENSION_ID_OFFSET = 0; static final int DIMENSION_VALUE_OFFSET = DIMENSION_ID_OFFSET + Integer.BYTES; - static final int START_DOC_ID_OFFSET = DIMENSION_VALUE_OFFSET + Long.BYTES; - static final int END_DOC_ID_OFFSET = START_DOC_ID_OFFSET + Integer.BYTES; - static final int AGGREGATE_DOC_ID_OFFSET = END_DOC_ID_OFFSET + Integer.BYTES; - static final int STAR_NODE_TYPE_OFFSET = AGGREGATE_DOC_ID_OFFSET + Integer.BYTES; + static final int START_ENTRY_ID_OFFSET = DIMENSION_VALUE_OFFSET + Long.BYTES; + static final int END_ENTRY_ID_OFFSET = START_ENTRY_ID_OFFSET + Integer.BYTES; + static final int AGGREGATE_ENTRY_ID_OFFSET = END_ENTRY_ID_OFFSET + Integer.BYTES; + static final int STAR_NODE_TYPE_OFFSET = AGGREGATE_ENTRY_ID_OFFSET + Integer.BYTES; static final int FIRST_CHILD_ID_OFFSET = STAR_NODE_TYPE_OFFSET + Byte.BYTES; static final int LAST_CHILD_ID_OFFSET = FIRST_CHILD_ID_OFFSET + Integer.BYTES; @@ -153,18 +153,18 @@ public int getChildDimensionId() throws IOException { } @Override - public int getStartDocId() throws IOException { - return getInt(START_DOC_ID_OFFSET); + public int getStartEntryId() throws IOException { + return getInt(START_ENTRY_ID_OFFSET); } @Override - public int getEndDocId() throws IOException { - return getInt(END_DOC_ID_OFFSET); + public int getEndEntryId() throws IOException { + return getInt(END_ENTRY_ID_OFFSET); } @Override - public int getAggregatedDocId() throws IOException { - return getInt(AGGREGATE_DOC_ID_OFFSET); + public int getAggregatedEntryId() throws IOException { + return getInt(AGGREGATE_ENTRY_ID_OFFSET); } @Override diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java index a34bbbe9ee738..962808ebb1b08 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/index/StarTreeValues.java @@ -12,7 +12,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.IndexInput; import org.opensearch.common.annotation.ExperimentalApi; import org.opensearch.index.compositeindex.CompositeIndexMetadata; @@ -25,6 +24,8 @@ import org.opensearch.index.compositeindex.datacube.startree.fileformats.meta.StarTreeMetadata; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import java.io.IOException; import java.util.ArrayList; @@ -35,8 +36,8 @@ import java.util.function.Supplier; import static org.opensearch.index.codec.composite.composite99.Composite99DocValuesReader.getSortedNumericDocValues; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_DOCS_COUNT; -import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_DOCS_COUNT; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.SEGMENT_ENTRIES_COUNT; +import static org.opensearch.index.compositeindex.CompositeIndexConstants.STAR_TREE_ENTRIES_COUNT; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeDimensionsDocValues; import static org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils.fullyQualifiedFieldNameForStarTreeMetricsDocValues; @@ -59,14 +60,14 @@ public class StarTreeValues implements CompositeIndexValues { private final StarTreeNode root; /** - * A map containing suppliers for DocIdSetIterators for dimensions. + * A map containing suppliers for StarTreeValues iterators for dimensions. */ - private final Map> dimensionDocValuesIteratorMap; + private final Map> dimensionValuesIteratorMap; /** - * A map containing suppliers for DocIdSetIterators for metrics. + * A map containing suppliers for StarTreeValues iterators for metrics. */ - private final Map> metricDocValuesIteratorMap; + private final Map> metricValuesIteratorMap; /** * A map containing attributes associated with the star tree values. @@ -84,22 +85,22 @@ public class StarTreeValues implements CompositeIndexValues { * * @param starTreeField The StarTreeField object representing the star tree field configuration. * @param root The root node of the star tree. - * @param dimensionDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for dimensions. - * @param metricDocValuesIteratorMap A map containing suppliers for DocIdSetIterators for metrics. + * @param dimensionValuesIteratorMap A map containing suppliers for StarTreeValues iterators for dimensions. + * @param metricValuesIteratorMap A map containing suppliers for StarTreeValues iterators for metrics. * @param attributes A map containing attributes associated with the star tree values. */ public StarTreeValues( StarTreeField starTreeField, StarTreeNode root, - Map> dimensionDocValuesIteratorMap, - Map> metricDocValuesIteratorMap, + Map> dimensionValuesIteratorMap, + Map> metricValuesIteratorMap, Map attributes, StarTreeMetadata compositeIndexMetadata ) { this.starTreeField = starTreeField; this.root = root; - this.dimensionDocValuesIteratorMap = dimensionDocValuesIteratorMap; - this.metricDocValuesIteratorMap = metricDocValuesIteratorMap; + this.dimensionValuesIteratorMap = dimensionValuesIteratorMap; + this.metricValuesIteratorMap = metricValuesIteratorMap; this.attributes = attributes; this.starTreeMetadata = compositeIndexMetadata; } @@ -137,7 +138,7 @@ public StarTreeValues( readDimensions, starTreeMetadata.getMetrics(), new StarTreeFieldConfiguration( - starTreeMetadata.getMaxLeafDocs(), + starTreeMetadata.getMaxLeafEntries(), skipStarNodeCreationInDims, starTreeMetadata.getStarTreeBuildMode() ) @@ -146,12 +147,12 @@ public StarTreeValues( this.root = StarTreeFactory.createStarTree(compositeIndexDataIn, starTreeMetadata); // get doc id set iterators for metrics and dimensions - dimensionDocValuesIteratorMap = new LinkedHashMap<>(); - metricDocValuesIteratorMap = new LinkedHashMap<>(); + dimensionValuesIteratorMap = new LinkedHashMap<>(); + metricValuesIteratorMap = new LinkedHashMap<>(); // get doc id set iterators for dimensions for (String dimension : starTreeMetadata.getDimensionFields()) { - dimensionDocValuesIteratorMap.put(dimension, () -> { + dimensionValuesIteratorMap.put(dimension, () -> { try { SortedNumericDocValues dimensionSortedNumericDocValues = null; if (readState != null) { @@ -162,9 +163,9 @@ public StarTreeValues( dimensionSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(dimensionfieldInfo); } } - return getSortedNumericDocValues(dimensionSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(dimensionSortedNumericDocValues)); } catch (IOException e) { - throw new RuntimeException("Error loading dimension DocIdSetIterator", e); + throw new RuntimeException("Error loading dimension StarTreeValuesIterator", e); } }); } @@ -177,7 +178,7 @@ public StarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocValuesIteratorMap.put(metricFullName, () -> { + metricValuesIteratorMap.put(metricFullName, () -> { try { SortedNumericDocValues metricSortedNumericDocValues = null; if (readState != null) { @@ -186,7 +187,7 @@ public StarTreeValues( metricSortedNumericDocValues = compositeDocValuesProducer.getSortedNumeric(metricFieldInfo); } } - return getSortedNumericDocValues(metricSortedNumericDocValues); + return new SortedNumericStarTreeValuesIterator(getSortedNumericDocValues(metricSortedNumericDocValues)); } catch (IOException e) { throw new RuntimeException("Error loading metric DocIdSetIterator", e); } @@ -198,10 +199,10 @@ public StarTreeValues( // Create an unmodifiable view of the map attributes = Map.of( - SEGMENT_DOCS_COUNT, - String.valueOf(starTreeMetadata.getSegmentAggregatedDocCount()), - STAR_TREE_DOCS_COUNT, - String.valueOf(starTreeMetadata.getStarTreeDocCount()) + SEGMENT_ENTRIES_COUNT, + String.valueOf(starTreeMetadata.getSegmentEntriesCount()), + STAR_TREE_ENTRIES_COUNT, + String.valueOf(starTreeMetadata.getStarTreeEntriesCount()) ); } @@ -239,36 +240,36 @@ public Map getAttributes() { } /** - * Returns the DocIdSetIterator for the specified dimension. + * Returns the StarTreeValues iterator for the specified dimension. * * @param dimension The name of the dimension. - * @return The DocIdSetIterator for the specified dimension. + * @return The StarTreeValuesIterator for the specified dimension. */ - public DocIdSetIterator getDimensionDocIdSetIterator(String dimension) { + public StarTreeValuesIterator getDimensionValuesIterator(String dimension) { - if (dimensionDocValuesIteratorMap.containsKey(dimension)) { - return dimensionDocValuesIteratorMap.get(dimension).get(); + if (dimensionValuesIteratorMap.containsKey(dimension)) { + return dimensionValuesIteratorMap.get(dimension).get(); } throw new IllegalArgumentException("dimension [" + dimension + "] does not exist in the segment."); } /** - * Returns the DocIdSetIterator for the specified fully qualified metric name. + * Returns the StarTreeValues iterator for the specified fully qualified metric name. * * @param fullyQualifiedMetricName The fully qualified name of the metric. - * @return The DocIdSetIterator for the specified fully qualified metric name. + * @return The StarTreeValuesIterator for the specified fully qualified metric name. */ - public DocIdSetIterator getMetricDocIdSetIterator(String fullyQualifiedMetricName) { + public StarTreeValuesIterator getMetricValuesIterator(String fullyQualifiedMetricName) { - if (metricDocValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { - return metricDocValuesIteratorMap.get(fullyQualifiedMetricName).get(); + if (metricValuesIteratorMap.containsKey(fullyQualifiedMetricName)) { + return metricValuesIteratorMap.get(fullyQualifiedMetricName).get(); } throw new IllegalArgumentException("metric [" + fullyQualifiedMetricName + "] does not exist in the segment."); } public int getStarTreeDocumentCount() { - return starTreeMetadata.getStarTreeDocCount(); + return starTreeMetadata.getStarTreeEntriesCount(); } } diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java index fce3e30e9ebf6..b687f8f1543f3 100644 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/node/StarTreeNode.java @@ -46,28 +46,28 @@ public interface StarTreeNode { int getChildDimensionId() throws IOException; /** - * Returns the start document ID of the current star-tree node. + * Returns the start record/entry ID of the current star-tree node. * * @return the start document ID - * @throws IOException if an I/O error occurs while reading the start document ID + * @throws IOException if an I/O error occurs while reading the start record/entry ID */ - int getStartDocId() throws IOException; + int getStartEntryId() throws IOException; /** - * Returns the end document ID of the current star-tree node. + * Returns the end record/entry ID of the current star-tree node. * - * @return the end document ID - * @throws IOException if an I/O error occurs while reading the end document ID + * @return the end record/entry ID + * @throws IOException if an I/O error occurs while reading the end record/entry ID */ - int getEndDocId() throws IOException; + int getEndEntryId() throws IOException; /** - * Returns the aggregated document ID of the current star-tree node. + * Returns the aggregated record/entry ID of the current star-tree node. * - * @return the aggregated document ID - * @throws IOException if an I/O error occurs while reading the aggregated document ID + * @return the aggregated record/entry ID + * @throws IOException if an I/O error occurs while reading the aggregated record/entry ID */ - int getAggregatedDocId() throws IOException; + int getAggregatedEntryId() throws IOException; /** * Returns the number of children of the current star-tree node. diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java deleted file mode 100644 index 061841d3e140a..0000000000000 --- a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIterator.java +++ /dev/null @@ -1,104 +0,0 @@ - -/* - * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. - */ - -package org.opensearch.index.compositeindex.datacube.startree.utils; - -import org.apache.lucene.index.SortedNumericDocValues; -import org.apache.lucene.search.DocIdSetIterator; -import org.opensearch.common.annotation.ExperimentalApi; - -import java.io.IOException; - -/** - * Coordinates the reading of documents across multiple DocIdSetIterators. - * It encapsulates a single DocIdSetIterator and maintains the latest document ID and its associated value. - * @opensearch.experimental - */ -@ExperimentalApi -public class SequentialDocValuesIterator { - - /** - * The doc id set iterator associated for each field. - */ - private final DocIdSetIterator docIdSetIterator; - - /** - * The value associated with the latest document. - */ - private Long docValue; - - /** - * The id of the latest document. - */ - private int docId = -1; - - /** - * Constructs a new SequentialDocValuesIterator instance with the given DocIdSetIterator. - * - * @param docIdSetIterator the DocIdSetIterator to be associated with this instance - */ - public SequentialDocValuesIterator(DocIdSetIterator docIdSetIterator) { - this.docIdSetIterator = docIdSetIterator; - } - - /** - * Returns the id of the latest document. - * - * @return the id of the latest document - */ - public int getDocId() { - return docId; - } - - /** - * Sets the id of the latest document. - * - * @param docId the ID of the latest document - */ - private void setDocId(int docId) { - this.docId = docId; - } - - /** - * Returns the DocIdSetIterator associated with this instance. - * - * @return the DocIdSetIterator associated with this instance - */ - public DocIdSetIterator getDocIdSetIterator() { - return docIdSetIterator; - } - - public int nextDoc(int currentDocId) throws IOException { - // if doc id stored is less than or equal to the requested doc id , return the stored doc id - if (docId >= currentDocId) { - return docId; - } - setDocId(this.docIdSetIterator.nextDoc()); - return docId; - } - - public Long value(int currentDocId) throws IOException { - if (this.getDocIdSetIterator() instanceof SortedNumericDocValues) { - SortedNumericDocValues sortedNumericDocValues = (SortedNumericDocValues) this.getDocIdSetIterator(); - if (currentDocId < 0) { - throw new IllegalStateException("invalid doc id to fetch the next value"); - } - if (currentDocId == DocIdSetIterator.NO_MORE_DOCS) { - throw new IllegalStateException("DocValuesIterator is already exhausted"); - } - if (docId == DocIdSetIterator.NO_MORE_DOCS || docId != currentDocId) { - return null; - } - return sortedNumericDocValues.nextValue(); - - } else { - throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); - } - } -} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIterator.java new file mode 100644 index 0000000000000..c699c42f5235b --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIterator.java @@ -0,0 +1,88 @@ + +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils; + +import org.opensearch.common.annotation.ExperimentalApi; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; + +import java.io.IOException; + +/** + * Coordinates the reading of documents across multiple StarTreeValuesIterator. + * It encapsulates a single StarTreeValuesIterator and maintains the latest document ID and its associated value. + * + * In case of merge , this will be reading the entries of star tree values and in case of flush this will go through + * the actual segment documents. + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SequentialValuesIterator { + + /** + * The doc id set iterator associated for each field. + */ + private final StarTreeValuesIterator starTreeValuesIterator; + + /** + * The id of the latest record/entry. + */ + private int entryId = -1; + + public SequentialValuesIterator(StarTreeValuesIterator starTreeValuesIterator) { + this.starTreeValuesIterator = starTreeValuesIterator; + } + + /** + * Returns the ID of the star tree record/entry or the segment document id + * + * @return the ID of the star tree record/entry or the segment document id + */ + public int getEntryId() { + return entryId; + } + + /** + * Sets the id of the latest entry. + * + * @param entryId the ID of the star tree record/entry or the segment document id + */ + private void setEntryId(int entryId) { + this.entryId = entryId; + } + + public int nextEntry(int currentEntryId) throws IOException { + // if doc id stored is less than or equal to the requested doc id , return the stored doc id + if (entryId >= currentEntryId) { + return entryId; + } + setEntryId(this.starTreeValuesIterator.nextEntry()); + return entryId; + } + + public Long value(int currentEntryId) throws IOException { + if (starTreeValuesIterator instanceof SortedNumericStarTreeValuesIterator) { + if (currentEntryId < 0) { + throw new IllegalStateException("invalid entry id to fetch the next value"); + } + if (currentEntryId == StarTreeValuesIterator.NO_MORE_ENTRIES) { + throw new IllegalStateException("StarTreeValuesIterator is already exhausted"); + } + if (entryId == StarTreeValuesIterator.NO_MORE_ENTRIES || entryId != currentEntryId) { + return null; + } + return ((SortedNumericStarTreeValuesIterator) starTreeValuesIterator).nextValue(); + + } else { + throw new IllegalStateException("Unsupported Iterator requested for SequentialDocValuesIterator"); + } + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java new file mode 100644 index 0000000000000..27afdf1479b4e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/SortedNumericStarTreeValuesIterator.java @@ -0,0 +1,32 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.index.SortedNumericDocValues; +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index to traverse through SortedNumericDocValues + * + * @opensearch.experimental + */ +@ExperimentalApi +public class SortedNumericStarTreeValuesIterator extends StarTreeValuesIterator { + + public SortedNumericStarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + super(docIdSetIterator); + } + + public long nextValue() throws IOException { + return ((SortedNumericDocValues) docIdSetIterator).nextValue(); + } +} diff --git a/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java new file mode 100644 index 0000000000000..84d6649a0631e --- /dev/null +++ b/server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/iterator/StarTreeValuesIterator.java @@ -0,0 +1,48 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.index.compositeindex.datacube.startree.utils.iterator; + +import org.apache.lucene.search.DocIdSetIterator; +import org.opensearch.common.annotation.ExperimentalApi; + +import java.io.IOException; + +/** + * Wrapper iterator class for StarTree index in place of DocIdSetIterator to read / traverse the docValues formats. + * This is needed since star tree values are different from segment documents and number of star tree values + * can even exceed segment docs in the worst cases. + * + * @opensearch.experimental + */ +@ExperimentalApi +public abstract class StarTreeValuesIterator { + + protected final DocIdSetIterator docIdSetIterator; + public static final int NO_MORE_ENTRIES = Integer.MAX_VALUE; + + public StarTreeValuesIterator(DocIdSetIterator docIdSetIterator) { + this.docIdSetIterator = docIdSetIterator; + } + + public int entryId() { + return docIdSetIterator.docID(); + } + + public int nextEntry() throws IOException { + return docIdSetIterator.nextDoc(); + } + + public int advance(int target) throws IOException { + return docIdSetIterator.advance(target); + } + + public long cost() { + return docIdSetIterator.cost(); + } +} diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java index b7395b993f67b..1bd56b0c79acc 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/StarTreeTestUtils.java @@ -18,7 +18,7 @@ import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeFactory; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; import org.opensearch.index.mapper.CompositeMappedFieldType; import org.opensearch.index.mapper.FieldValueConverter; @@ -50,14 +50,14 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( List starTreeDocuments = new ArrayList<>(); for (StarTreeValues starTreeValues : starTreeValuesSubs) { List dimensionsSplitOrder = starTreeValues.getStarTreeField().getDimensionsOrder(); - SequentialDocValuesIterator[] dimensionReaders = new SequentialDocValuesIterator[dimensionsSplitOrder.size()]; + SequentialValuesIterator[] dimensionReaders = new SequentialValuesIterator[dimensionsSplitOrder.size()]; for (int i = 0; i < dimensionsSplitOrder.size(); i++) { String dimension = dimensionsSplitOrder.get(i).getField(); - dimensionReaders[i] = new SequentialDocValuesIterator(starTreeValues.getDimensionDocIdSetIterator(dimension)); + dimensionReaders[i] = new SequentialValuesIterator(starTreeValues.getDimensionValuesIterator(dimension)); } - List metricReaders = new ArrayList<>(); + List metricReaders = new ArrayList<>(); // get doc id set iterators for metrics for (Metric metric : starTreeValues.getStarTreeField().getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { @@ -69,7 +69,7 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( metric.getField(), metricStat.getTypeName() ); - metricReaders.add(new SequentialDocValuesIterator(starTreeValues.getMetricDocIdSetIterator(metricFullName))); + metricReaders.add(new SequentialValuesIterator(starTreeValues.getMetricValuesIterator(metricFullName))); } } @@ -85,22 +85,22 @@ public static StarTreeDocument[] getSegmentsStarTreeDocuments( public static StarTreeDocument getStarTreeDocument( int currentDocId, - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders, + SequentialValuesIterator[] dimensionReaders, + List metricReaders, List fieldValueConverters ) throws IOException { Long[] dims = new Long[dimensionReaders.length]; int i = 0; - for (SequentialDocValuesIterator dimensionDocValueIterator : dimensionReaders) { - dimensionDocValueIterator.nextDoc(currentDocId); + for (SequentialValuesIterator dimensionDocValueIterator : dimensionReaders) { + dimensionDocValueIterator.nextEntry(currentDocId); Long val = dimensionDocValueIterator.value(currentDocId); dims[i] = val; i++; } i = 0; Object[] metrics = new Object[metricReaders.size()]; - for (SequentialDocValuesIterator metricDocValuesIterator : metricReaders) { - metricDocValuesIterator.nextDoc(currentDocId); + for (SequentialValuesIterator metricDocValuesIterator : metricReaders) { + metricDocValuesIterator.nextEntry(currentDocId); metrics[i] = toAggregatorValueType(metricDocValuesIterator.value(currentDocId), fieldValueConverters.get(i)); i++; } @@ -236,10 +236,10 @@ public static void validateFileFormats( public static void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { assertEquals(starTreeNode.getDimensionId(), treeNode.getDimensionId()); assertEquals(starTreeNode.getDimensionValue(), treeNode.getDimensionValue()); - assertEquals(starTreeNode.getStartDocId(), treeNode.getStartDocId()); - assertEquals(starTreeNode.getEndDocId(), treeNode.getEndDocId()); + assertEquals(starTreeNode.getStartEntryId(), treeNode.getStartDocId()); + assertEquals(starTreeNode.getEndEntryId(), treeNode.getEndDocId()); assertEquals(starTreeNode.getChildDimensionId(), treeNode.getChildDimensionId()); - assertEquals(starTreeNode.getAggregatedDocId(), treeNode.getAggregatedDocId()); + assertEquals(starTreeNode.getAggregatedEntryId(), treeNode.getAggregatedDocId()); if (starTreeNode.getChildDimensionId() != -1) { assertFalse(starTreeNode.isLeaf()); @@ -302,9 +302,9 @@ public static void assertStarTreeMetadata(StarTreeMetadata expectedStarTreeMetad } - assertEquals(expectedStarTreeMetadata.getSegmentAggregatedDocCount(), resultStarTreeMetadata.getSegmentAggregatedDocCount()); - assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), resultStarTreeMetadata.getStarTreeDocCount()); - assertEquals(expectedStarTreeMetadata.getMaxLeafDocs(), resultStarTreeMetadata.getMaxLeafDocs()); + assertEquals(expectedStarTreeMetadata.getSegmentEntriesCount(), resultStarTreeMetadata.getSegmentEntriesCount()); + assertEquals(expectedStarTreeMetadata.getStarTreeEntriesCount(), resultStarTreeMetadata.getStarTreeEntriesCount()); + assertEquals(expectedStarTreeMetadata.getMaxLeafEntries(), resultStarTreeMetadata.getMaxLeafEntries()); assertEquals( expectedStarTreeMetadata.getSkipStarNodeCreationInDims().size(), resultStarTreeMetadata.getSkipStarNodeCreationInDims().size() diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java index 65adc43ea8bea..d253537fb4440 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/AbstractStarTreeBuilderTests.java @@ -51,8 +51,10 @@ import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; import org.opensearch.index.compositeindex.datacube.startree.node.InMemoryTreeNode; import org.opensearch.index.compositeindex.datacube.startree.node.StarTreeNodeType; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; import org.opensearch.index.compositeindex.datacube.startree.utils.StarTreeUtils; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.StarTreeValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.FieldValueConverter; @@ -352,8 +354,8 @@ public void test_sortAndAggregateStarTreeDocuments() throws IOException { ); Iterator expectedStarTreeDocumentIterator = inorderStarTreeDocuments.iterator(); - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -382,9 +384,8 @@ public void test_sortAndAggregateStarTreeDocuments() throws IOException { assertEquals(inorderStarTreeDocuments.size(), numOfAggregatedDocuments); } - SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeDocuments) { - SequentialDocValuesIterator[] sequentialDocValuesIterators = - new SequentialDocValuesIterator[starTreeDocuments[0].dimensions.length]; + SequentialValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeDocuments) { + SequentialValuesIterator[] sequentialDocValuesIterators = new SequentialValuesIterator[starTreeDocuments[0].dimensions.length]; for (int j = 0; j < starTreeDocuments[0].dimensions.length; j++) { List dimList = new ArrayList<>(); List docsWithField = new ArrayList<>(); @@ -395,13 +396,15 @@ SequentialDocValuesIterator[] getDimensionIterators(StarTreeDocument[] starTreeD docsWithField.add(i); } } - sequentialDocValuesIterators[j] = new SequentialDocValuesIterator(getSortedNumericMock(dimList, docsWithField)); + sequentialDocValuesIterators[j] = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)) + ); } return sequentialDocValuesIterators; } - List getMetricIterators(StarTreeDocument[] starTreeDocuments) { - List sequentialDocValuesIterators = new ArrayList<>(); + List getMetricIterators(StarTreeDocument[] starTreeDocuments) { + List sequentialDocValuesIterators = new ArrayList<>(); for (int j = 0; j < starTreeDocuments[0].metrics.length; j++) { List metricslist = new ArrayList<>(); List docsWithField = new ArrayList<>(); @@ -412,7 +415,9 @@ List getMetricIterators(StarTreeDocument[] starTree docsWithField.add(i); } } - sequentialDocValuesIterators.add(new SequentialDocValuesIterator(getSortedNumericMock(metricslist, docsWithField))); + sequentialDocValuesIterators.add( + new SequentialValuesIterator(new SortedNumericStarTreeValuesIterator(getSortedNumericMock(metricslist, docsWithField))) + ); } return sequentialDocValuesIterators; } @@ -448,8 +453,8 @@ public void test_sortAndAggregateStarTreeDocuments_nullMetric() throws IOExcepti new Object[] { metric1, metric2, metric3, metric4, metric5, null } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -519,8 +524,8 @@ public void test_sortAndAggregateStarTreeDocuments_nullMetricField() throws IOEx new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -590,8 +595,8 @@ public void test_sortAndAggregateStarTreeDocuments_nullAndMinusOneInDimensionFie new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -671,8 +676,8 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndNullMetrics( new Object[] { metric1, metric2, metric3, metric4, metric5, null } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -748,8 +753,8 @@ public void test_sortAndAggregateStarTreeDocuments_nullDimensionsAndFewNullMetri new Object[] { metric1, metric2, metric3, metric4, metric5, null } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -820,8 +825,8 @@ public void test_sortAndAggregateStarTreeDocuments_emptyDimensions() throws IOEx new Object[] { metric1, metric2, metric3, metric4, metric5, metric6 } ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -889,8 +894,8 @@ public void test_sortAndAggregateStarTreeDocument_longMaxAndLongMinDimensions() ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -965,8 +970,8 @@ public void test_sortAndAggregateStarTreeDocument_DoubleMaxAndDoubleMinMetrics() ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -1097,8 +1102,8 @@ public void test_build_halfFloatMetrics() throws IOException { ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, @@ -1211,8 +1216,8 @@ public void test_build_floatMetrics() throws IOException { ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -1311,8 +1316,8 @@ public void test_build_longMetrics() throws IOException { ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( writeState, Composite99DocValuesFormat.DATA_DOC_VALUES_CODEC, @@ -1398,8 +1403,8 @@ public void test_build_multipleStarTrees() throws IOException { ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); metrics = List.of( new Metric("field2", List.of(MetricStat.SUM)), @@ -1514,8 +1519,8 @@ public void test_build_multipleStarTrees() throws IOException { segmentStarTreeDocuments2[i] = new StarTreeDocument(starTreeDocuments2[i].dimensions, new Long[] { metric1 }); } - SequentialDocValuesIterator[] dimsIterators2 = getDimensionIterators(segmentStarTreeDocuments2); - List metricsIterators2 = getMetricIterators(segmentStarTreeDocuments2); + SequentialValuesIterator[] dimsIterators2 = getDimensionIterators(segmentStarTreeDocuments2); + List metricsIterators2 = getMetricIterators(segmentStarTreeDocuments2); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator2 = builder.sortAndAggregateSegmentDocuments( dimsIterators2, @@ -1628,8 +1633,8 @@ public void test_build() throws IOException { ); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -1789,8 +1794,8 @@ public void test_build_starTreeDataset() throws IOException { segmentStarTreeDocuments[i] = new StarTreeDocument(starTreeDocuments[i].dimensions, new Long[] { metric1, null }); } - SequentialDocValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); - List metricsIterators = getMetricIterators(segmentStarTreeDocuments); + SequentialValuesIterator[] dimsIterators = getDimensionIterators(segmentStarTreeDocuments); + List metricsIterators = getMetricIterators(segmentStarTreeDocuments); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); Iterator segmentStarTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimsIterators, @@ -1892,7 +1897,7 @@ private void validateStarTreeFileFormats( IndexInput metaIn = readState.directory.openInput(metaFileName, IOContext.DEFAULT); StarTreeValues starTreeValues = new StarTreeValues(expectedStarTreeMetadata, dataIn, compositeDocValuesProducer, readState); - assertEquals(expectedStarTreeMetadata.getStarTreeDocCount(), starTreeValues.getStarTreeDocumentCount()); + assertEquals(expectedStarTreeMetadata.getStarTreeEntriesCount(), starTreeValues.getStarTreeDocumentCount()); List fieldValueConverters = new ArrayList<>(); builder.metricAggregatorInfos.forEach( metricAggregatorInfo -> fieldValueConverters.add(metricAggregatorInfo.getValueAggregators().getAggregatedValueType()) @@ -1985,17 +1990,23 @@ public void testFlushFlow() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + SequentialValuesIterator[] dimDvs = { new SequentialValuesIterator(d1sndv), new SequentialValuesIterator(d2sndv) }; Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimDvs, - List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + List.of(new SequentialValuesIterator(m1sndv), new SequentialValuesIterator(m2sndv)) ); /** * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] @@ -2081,10 +2092,16 @@ public void testFlushFlowDimsReverse() throws IOException { List metricsWithField = List.of(0, 1, 2, 3, 4, 5); compositeField = getStarTreeFieldWithMultipleMetrics(); - SortedNumericDocValues d1sndv = getSortedNumericMock(dimList, docsWithField); - SortedNumericDocValues d2sndv = getSortedNumericMock(dimList2, docsWithField2); - SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); - SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList, metricsWithField); + SortedNumericStarTreeValuesIterator d1sndv = new SortedNumericStarTreeValuesIterator(getSortedNumericMock(dimList, docsWithField)); + SortedNumericStarTreeValuesIterator d2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(dimList2, docsWithField2) + ); + SortedNumericStarTreeValuesIterator m1sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); + SortedNumericStarTreeValuesIterator m2sndv = new SortedNumericStarTreeValuesIterator( + getSortedNumericMock(metricsList, metricsWithField) + ); writeState = getWriteState(6, writeState.segmentInfo.getId()); this.docValuesConsumer = LuceneDocValuesConsumerFactory.getDocValuesConsumerForCompositeCodec( @@ -2095,10 +2112,10 @@ public void testFlushFlowDimsReverse() throws IOException { Composite99DocValuesFormat.META_DOC_VALUES_EXTENSION ); builder = getStarTreeBuilder(metaOut, dataOut, compositeField, writeState, mapperService); - SequentialDocValuesIterator[] dimDvs = { new SequentialDocValuesIterator(d1sndv), new SequentialDocValuesIterator(d2sndv) }; + SequentialValuesIterator[] dimDvs = { new SequentialValuesIterator(d1sndv), new SequentialValuesIterator(d2sndv) }; Iterator starTreeDocumentIterator = builder.sortAndAggregateSegmentDocuments( dimDvs, - List.of(new SequentialDocValuesIterator(m1sndv), new SequentialDocValuesIterator(m2sndv)) + List.of(new SequentialValuesIterator(m1sndv), new SequentialValuesIterator(m2sndv)) ); /** * Asserting following dim / metrics [ dim1, dim2 / Sum [metric], count [metric] ] @@ -2508,9 +2525,14 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d1sndv = dimList; SortedNumericDocValues d2sndv = dimList2; SortedNumericDocValues m1sndv = metricsList; - Map> dimDocIdSetIterators = Map.of("field1", () -> d1sndv, "field3", () -> d2sndv); + Map> dimDocIdSetIterators = Map.of( + "field1", + () -> new SortedNumericStarTreeValuesIterator(d1sndv), + "field3", + () -> new SortedNumericStarTreeValuesIterator(d2sndv) + ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); for (Metric metric : sf.getMetrics()) { for (MetricStat metricStat : metric.getMetrics()) { String metricFullName = fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -2518,7 +2540,7 @@ private StarTreeValues getStarTreeValues( metric.getField(), metricStat.getTypeName() ); - metricDocIdSetIterators.put(metricFullName, () -> m1sndv); + metricDocIdSetIterators.put(metricFullName, () -> new SortedNumericStarTreeValuesIterator(m1sndv)); } } @@ -2527,7 +2549,7 @@ private StarTreeValues getStarTreeValues( null, dimDocIdSetIterators, metricDocIdSetIterators, - Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, number), + Map.of(CompositeIndexConstants.SEGMENT_ENTRIES_COUNT, number), null ); return starTreeValues; @@ -3648,18 +3670,18 @@ private StarTreeValues getStarTreeValues( SortedNumericDocValues d4sndv = getSortedNumericMock(dimList4, docsWithField4); SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues m2sndv = getSortedNumericMock(metricsList1, metricsWithField1); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = new LinkedHashMap<>(); + Map> metricDocIdSetIterators = new LinkedHashMap<>(); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3667,7 +3689,7 @@ private StarTreeValues getStarTreeValues( "field2", sf.getMetrics().get(0).getMetrics().get(0).getTypeName() ), - () -> m1sndv + () -> new SortedNumericStarTreeValuesIterator(m1sndv) ); metricDocIdSetIterators.put( fullyQualifiedFieldNameForStarTreeMetricsDocValues( @@ -3675,7 +3697,7 @@ private StarTreeValues getStarTreeValues( "_doc_count", sf.getMetrics().get(1).getMetrics().get(0).getTypeName() ), - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); // metricDocIdSetIterators.put("field2", () -> m1sndv); // metricDocIdSetIterators.put("_doc_count", () -> m2sndv); @@ -4093,24 +4115,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues m2sndv = DocValues.emptySortedNumeric(); - Map> dimDocIdSetIterators = Map.of( + Map> dimDocIdSetIterators = Map.of( "field1", - () -> d1sndv, + () -> new SortedNumericStarTreeValuesIterator(d1sndv), "field3", - () -> d2sndv, + () -> new SortedNumericStarTreeValuesIterator(d2sndv), "field5", - () -> d3sndv, + () -> new SortedNumericStarTreeValuesIterator(d3sndv), "field8", - () -> d4sndv + () -> new SortedNumericStarTreeValuesIterator(d4sndv) ); - Map> metricDocIdSetIterators = Map.of( + Map> metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> m1sndv, + () -> new SortedNumericStarTreeValuesIterator(m1sndv), "sf_field2_value_count_metric", - () -> valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(valucountsndv), "sf__doc_count_doc_count_metric", - () -> m2sndv + () -> new SortedNumericStarTreeValuesIterator(m2sndv) ); StarTreeValues starTreeValues = new StarTreeValues( @@ -4129,24 +4151,24 @@ public void testMergeFlow() throws IOException { SortedNumericDocValues f2m1sndv = getSortedNumericMock(metricsList, metricsWithField); SortedNumericDocValues f2valucountsndv = getSortedNumericMock(metricsListValueCount, metricsWithFieldValueCount); SortedNumericDocValues f2m2sndv = DocValues.emptySortedNumeric(); - Map> f2dimDocIdSetIterators = Map.of( + Map> f2dimDocIdSetIterators = Map.of( "field1", - () -> f2d1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d1sndv), "field3", - () -> f2d2sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d2sndv), "field5", - () -> f2d3sndv, + () -> new SortedNumericStarTreeValuesIterator(f2d3sndv), "field8", - () -> f2d4sndv + () -> new SortedNumericStarTreeValuesIterator(f2d4sndv) ); - Map> f2metricDocIdSetIterators = Map.of( + Map> f2metricDocIdSetIterators = Map.of( "sf_field2_sum_metric", - () -> f2m1sndv, + () -> new SortedNumericStarTreeValuesIterator(f2m1sndv), "sf_field2_value_count_metric", - () -> f2valucountsndv, + () -> new SortedNumericStarTreeValuesIterator(f2valucountsndv), "sf__doc_count_doc_count_metric", - () -> f2m2sndv + () -> new SortedNumericStarTreeValuesIterator(f2m2sndv) ); StarTreeValues starTreeValues2 = new StarTreeValues( compositeField, @@ -4391,7 +4413,7 @@ private int compareDocuments(StarTreeDocument doc1, StarTreeDocument doc2, int s } Map getAttributes(int numSegmentDocs) { - return Map.of(CompositeIndexConstants.SEGMENT_DOCS_COUNT, String.valueOf(numSegmentDocs)); + return Map.of(CompositeIndexConstants.SEGMENT_ENTRIES_COUNT, String.valueOf(numSegmentDocs)); } private StarTreeField getStarTreeField(MetricStat count) { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java index 8f750b09c75ee..61648dbe165db 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/builder/BaseStarTreeBuilderTests.java @@ -35,7 +35,7 @@ import org.opensearch.index.compositeindex.datacube.startree.StarTreeFieldConfiguration; import org.opensearch.index.compositeindex.datacube.startree.aggregators.MetricAggregatorInfo; import org.opensearch.index.compositeindex.datacube.startree.index.StarTreeValues; -import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialDocValuesIterator; +import org.opensearch.index.compositeindex.datacube.startree.utils.SequentialValuesIterator; import org.opensearch.index.mapper.ContentPath; import org.opensearch.index.mapper.DocumentMapper; import org.opensearch.index.mapper.Mapper; @@ -206,8 +206,8 @@ public Long getDimensionValue(int docId, int dimensionId) throws IOException { @Override public Iterator sortAndAggregateSegmentDocuments( - SequentialDocValuesIterator[] dimensionReaders, - List metricReaders + SequentialValuesIterator[] dimensionReaders, + List metricReaders ) throws IOException { return null; } diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java index 4d2aa5eaf78cf..532720833a1a2 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/data/StarTreeFileFormatsTests.java @@ -136,10 +136,10 @@ public void test_starTreeSearch() throws IOException { private void assertStarTreeNode(StarTreeNode starTreeNode, InMemoryTreeNode treeNode) throws IOException { assertEquals(starTreeNode.getDimensionId(), treeNode.getDimensionId()); assertEquals(starTreeNode.getDimensionValue(), treeNode.getDimensionValue()); - assertEquals(starTreeNode.getStartDocId(), treeNode.getStartDocId()); - assertEquals(starTreeNode.getEndDocId(), treeNode.getEndDocId()); + assertEquals(starTreeNode.getStartEntryId(), treeNode.getStartDocId()); + assertEquals(starTreeNode.getEndEntryId(), treeNode.getEndDocId()); assertEquals(starTreeNode.getChildDimensionId(), treeNode.getChildDimensionId()); - assertEquals(starTreeNode.getAggregatedDocId(), treeNode.getAggregatedDocId()); + assertEquals(starTreeNode.getAggregatedEntryId(), treeNode.getAggregatedDocId()); assertEquals(starTreeNode.getStarTreeNodeType(), treeNode.getNodeType()); if (starTreeNode.getChildDimensionId() != -1) { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java index 2127eeb9636d3..50b990db5900d 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/meta/StarTreeMetadataTests.java @@ -199,8 +199,8 @@ public void test_starTreeMetadata() throws IOException { assertEquals(expectedMetric.getMetrics().get(j), resultMetric.getMetrics().get(j)); } } - assertEquals(segmentDocumentCount, starTreeMetadata.getSegmentAggregatedDocCount(), 0); - assertEquals(maxLeafDocs, starTreeMetadata.getMaxLeafDocs(), 0); + assertEquals(segmentDocumentCount, starTreeMetadata.getSegmentEntriesCount(), 0); + assertEquals(maxLeafDocs, starTreeMetadata.getMaxLeafEntries(), 0); assertEquals( starTreeFieldConfiguration.getSkipStarNodeCreationInDims().size(), starTreeMetadata.getSkipStarNodeCreationInDims().size() diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java index 08815d5ef55f5..8becb4b11674f 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/fileformats/node/FixedLengthStarTreeNodeTests.java @@ -95,9 +95,9 @@ public void setup() throws IOException { public void testOffsets() { assertEquals(0, FixedLengthStarTreeNode.DIMENSION_ID_OFFSET); assertEquals(4, FixedLengthStarTreeNode.DIMENSION_VALUE_OFFSET); - assertEquals(12, FixedLengthStarTreeNode.START_DOC_ID_OFFSET); - assertEquals(16, FixedLengthStarTreeNode.END_DOC_ID_OFFSET); - assertEquals(20, FixedLengthStarTreeNode.AGGREGATE_DOC_ID_OFFSET); + assertEquals(12, FixedLengthStarTreeNode.START_ENTRY_ID_OFFSET); + assertEquals(16, FixedLengthStarTreeNode.END_ENTRY_ID_OFFSET); + assertEquals(20, FixedLengthStarTreeNode.AGGREGATE_ENTRY_ID_OFFSET); assertEquals(24, FixedLengthStarTreeNode.STAR_NODE_TYPE_OFFSET); assertEquals(25, FixedLengthStarTreeNode.FIRST_CHILD_ID_OFFSET); assertEquals(29, FixedLengthStarTreeNode.LAST_CHILD_ID_OFFSET); @@ -115,16 +115,16 @@ public void testGetDimensionValue() throws IOException { assertEquals(node.getDimensionValue(), starTreeNode.getDimensionValue()); } - public void testGetStartDocId() throws IOException { - assertEquals(node.getStartDocId(), starTreeNode.getStartDocId()); + public void testGetStartEntryId() throws IOException { + assertEquals(node.getStartDocId(), starTreeNode.getStartEntryId()); } - public void testGetEndDocId() throws IOException { - assertEquals(node.getEndDocId(), starTreeNode.getEndDocId()); + public void testGetEndEntryId() throws IOException { + assertEquals(node.getEndDocId(), starTreeNode.getEndEntryId()); } - public void testGetAggregatedDocId() throws IOException { - assertEquals(node.getAggregatedDocId(), starTreeNode.getAggregatedDocId()); + public void testGetAggregatedEntryId() throws IOException { + assertEquals(node.getAggregatedDocId(), starTreeNode.getAggregatedEntryId()); } public void testGetNumChildren() throws IOException { diff --git a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIteratorTests.java similarity index 57% rename from server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java rename to server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIteratorTests.java index f56f7d9906ae1..af332aa9c14ff 100644 --- a/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialDocValuesIteratorTests.java +++ b/server/src/test/java/org/opensearch/index/compositeindex/datacube/startree/utils/SequentialValuesIteratorTests.java @@ -9,7 +9,6 @@ package org.opensearch.index.compositeindex.datacube.startree.utils; import org.apache.lucene.codecs.DocValuesProducer; -import org.apache.lucene.index.BinaryDocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexOptions; @@ -17,7 +16,7 @@ import org.apache.lucene.index.VectorEncoding; import org.apache.lucene.index.VectorSimilarityFunction; import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.util.BytesRef; +import org.opensearch.index.compositeindex.datacube.startree.utils.iterator.SortedNumericStarTreeValuesIterator; import org.opensearch.test.OpenSearchTestCase; import org.junit.BeforeClass; @@ -28,7 +27,7 @@ import static org.mockito.Mockito.when; -public class SequentialDocValuesIteratorTests extends OpenSearchTestCase { +public class SequentialValuesIteratorTests extends OpenSearchTestCase { private static FieldInfo mockFieldInfo; @@ -59,50 +58,42 @@ public void testCreateIterator_SortedNumeric() throws IOException { DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(producer.getSortedNumeric(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getSortedNumeric(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - } - - public void testCreateIterator_UnsupportedType() throws IOException { - DocValuesProducer producer = Mockito.mock(DocValuesProducer.class); - BinaryDocValues iterator = Mockito.mock(BinaryDocValues.class); - when(producer.getBinary(mockFieldInfo)).thenReturn(iterator); - SequentialDocValuesIterator result = new SequentialDocValuesIterator(producer.getBinary(mockFieldInfo)); - assertEquals(iterator.getClass(), result.getDocIdSetIterator().getClass()); - when(iterator.nextDoc()).thenReturn(0); - when(iterator.binaryValue()).thenReturn(new BytesRef("123")); + SequentialValuesIterator result = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(producer.getSortedNumeric(mockFieldInfo)) + ); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { - result.nextDoc(0); - result.value(0); - }); - assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); } public void testGetNextValue_SortedNumeric() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); when(iterator.nextDoc()).thenReturn(0); when(iterator.nextValue()).thenReturn(123L); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); - sequentialDocValuesIterator.nextDoc(0); + SequentialValuesIterator sequentialDocValuesIterator = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); + sequentialDocValuesIterator.nextEntry(0); long result = sequentialDocValuesIterator.value(0); assertEquals(123L, result); } public void testGetNextValue_UnsupportedIterator() { DocIdSetIterator iterator = Mockito.mock(DocIdSetIterator.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + IllegalStateException exception = expectThrows( + IllegalStateException.class, + () -> new SequentialValuesIterator(new SortedNumericStarTreeValuesIterator(iterator)) + ); - IllegalStateException exception = expectThrows(IllegalStateException.class, () -> { sequentialDocValuesIterator.value(0); }); assertEquals("Unsupported Iterator requested for SequentialDocValuesIterator", exception.getMessage()); } - public void testNextDoc() throws IOException { + public void testNextEntry() throws IOException { SortedNumericDocValues iterator = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator = new SequentialDocValuesIterator(iterator); + SequentialValuesIterator sequentialDocValuesIterator = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator) + ); when(iterator.nextDoc()).thenReturn(5); - int result = sequentialDocValuesIterator.nextDoc(5); + int result = sequentialDocValuesIterator.nextEntry(5); assertEquals(5, result); } @@ -110,8 +101,12 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { SortedNumericDocValues iterator1 = Mockito.mock(SortedNumericDocValues.class); SortedNumericDocValues iterator2 = Mockito.mock(SortedNumericDocValues.class); - SequentialDocValuesIterator sequentialDocValuesIterator1 = new SequentialDocValuesIterator(iterator1); - SequentialDocValuesIterator sequentialDocValuesIterator2 = new SequentialDocValuesIterator(iterator2); + SequentialValuesIterator sequentialDocValuesIterator1 = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator1) + ); + SequentialValuesIterator sequentialDocValuesIterator2 = new SequentialValuesIterator( + new SortedNumericStarTreeValuesIterator(iterator2) + ); when(iterator1.nextDoc()).thenReturn(0); when(iterator2.nextDoc()).thenReturn(1); @@ -119,13 +114,13 @@ public void test_multipleCoordinatedDocumentReader() throws IOException { when(iterator1.nextValue()).thenReturn(9L); when(iterator2.nextValue()).thenReturn(9L); - sequentialDocValuesIterator1.nextDoc(0); - sequentialDocValuesIterator2.nextDoc(0); - assertEquals(0, sequentialDocValuesIterator1.getDocId()); + sequentialDocValuesIterator1.nextEntry(0); + sequentialDocValuesIterator2.nextEntry(0); + assertEquals(0, sequentialDocValuesIterator1.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator1.value(0)); assertNull(sequentialDocValuesIterator2.value(0)); - assertNotEquals(0, sequentialDocValuesIterator2.getDocId()); - assertEquals(1, sequentialDocValuesIterator2.getDocId()); + assertNotEquals(0, sequentialDocValuesIterator2.getEntryId()); + assertEquals(1, sequentialDocValuesIterator2.getEntryId()); assertEquals(9L, (long) sequentialDocValuesIterator2.value(1)); } }