-
Notifications
You must be signed in to change notification settings - Fork 65
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Changed format for hybrid query results to a single list of scores wi…
…th delimiter (#259) * Changed approach for storing hybrid query results from compound top docs to signle list of scores with delimiter Signed-off-by: Martin Gaievski <[email protected]>
- Loading branch information
1 parent
d12f480
commit 75b59cd
Showing
26 changed files
with
902 additions
and
385 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
120 changes: 120 additions & 0 deletions
120
src/main/java/org/opensearch/neuralsearch/processor/CompoundTopDocs.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,120 @@ | ||
| /* | ||
| * Copyright OpenSearch Contributors | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
|
|
||
| package org.opensearch.neuralsearch.processor; | ||
|
|
||
| import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryDelimiterElement; | ||
| import static org.opensearch.neuralsearch.search.util.HybridSearchResultFormatUtil.isHybridQueryStartStopElement; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.Objects; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import lombok.AllArgsConstructor; | ||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import lombok.ToString; | ||
| import lombok.extern.log4j.Log4j2; | ||
|
|
||
| import org.apache.lucene.search.ScoreDoc; | ||
| import org.apache.lucene.search.TopDocs; | ||
| import org.apache.lucene.search.TotalHits; | ||
|
|
||
| /** | ||
| * Class stores collection of TopDocs for each sub query from hybrid query. Collection of results is at shard level. We do store | ||
| * list of TopDocs and list of ScoreDoc as well as total hits for the shard. | ||
| */ | ||
| @AllArgsConstructor | ||
| @Getter | ||
| @ToString(includeFieldNames = true) | ||
| @Log4j2 | ||
| public class CompoundTopDocs { | ||
|
|
||
| @Setter | ||
| private TotalHits totalHits; | ||
| private List<TopDocs> topDocs; | ||
| @Setter | ||
| private List<ScoreDoc> scoreDocs; | ||
|
|
||
| public CompoundTopDocs(final TotalHits totalHits, final List<TopDocs> topDocs) { | ||
| initialize(totalHits, topDocs); | ||
| } | ||
|
|
||
| private void initialize(TotalHits totalHits, List<TopDocs> topDocs) { | ||
| this.totalHits = totalHits; | ||
| this.topDocs = topDocs; | ||
| scoreDocs = cloneLargestScoreDocs(topDocs); | ||
| } | ||
|
|
||
| /** | ||
| * Create new instance from TopDocs by parsing scores of sub-queries. Final format looks like: | ||
| * doc_id | magic_number_1 | ||
| * doc_id | magic_number_2 | ||
| * ... | ||
| * doc_id | magic_number_2 | ||
| * ... | ||
| * doc_id | magic_number_2 | ||
| * ... | ||
| * doc_id | magic_number_1 | ||
| * | ||
| * where doc_id is one of valid ids from result. For example, this is list with results for there sub-queries | ||
| * | ||
| * 0, 9549511920.4881596047 | ||
| * 0, 4422440593.9791198149 | ||
| * 0, 0.8 | ||
| * 2, 0.5 | ||
| * 0, 4422440593.9791198149 | ||
| * 0, 4422440593.9791198149 | ||
| * 2, 0.7 | ||
| * 5, 0.65 | ||
| * 6, 0.15 | ||
| * 0, 9549511920.4881596047 | ||
| */ | ||
| public CompoundTopDocs(final TopDocs topDocs) { | ||
| ScoreDoc[] scoreDocs = topDocs.scoreDocs; | ||
| if (Objects.isNull(scoreDocs) || scoreDocs.length < 2) { | ||
| initialize(topDocs.totalHits, new ArrayList<>()); | ||
| return; | ||
| } | ||
| // skipping first two elements, it's a start-stop element and delimiter for first series | ||
| List<TopDocs> topDocsList = new ArrayList<>(); | ||
| List<ScoreDoc> scoreDocList = new ArrayList<>(); | ||
| for (int index = 2; index < scoreDocs.length; index++) { | ||
| // getting first element of score's series | ||
| ScoreDoc scoreDoc = scoreDocs[index]; | ||
| if (isHybridQueryDelimiterElement(scoreDoc) || isHybridQueryStartStopElement(scoreDoc)) { | ||
| ScoreDoc[] subQueryScores = scoreDocList.toArray(new ScoreDoc[0]); | ||
| TotalHits totalHits = new TotalHits(subQueryScores.length, TotalHits.Relation.EQUAL_TO); | ||
| TopDocs subQueryTopDocs = new TopDocs(totalHits, subQueryScores); | ||
| topDocsList.add(subQueryTopDocs); | ||
| scoreDocList.clear(); | ||
| } else { | ||
| scoreDocList.add(scoreDoc); | ||
| } | ||
| } | ||
| initialize(topDocs.totalHits, topDocsList); | ||
| } | ||
|
|
||
| private List<ScoreDoc> cloneLargestScoreDocs(final List<TopDocs> docs) { | ||
| if (docs == null) { | ||
| return null; | ||
| } | ||
| ScoreDoc[] maxScoreDocs = new ScoreDoc[0]; | ||
| int maxLength = -1; | ||
| for (TopDocs topDoc : docs) { | ||
| if (topDoc == null || topDoc.scoreDocs == null) { | ||
| continue; | ||
| } | ||
| if (topDoc.scoreDocs.length > maxLength) { | ||
| maxLength = topDoc.scoreDocs.length; | ||
| maxScoreDocs = topDoc.scoreDocs; | ||
| } | ||
| } | ||
| // do deep copy | ||
| return Arrays.stream(maxScoreDocs).map(doc -> new ScoreDoc(doc.doc, doc.score, doc.shardIndex)).collect(Collectors.toList()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.