Skip to content

Commit

Permalink
Add ability to configure date types for no expansion
Browse files Browse the repository at this point in the history 
Add the ability to configure date types that will not result in date
filter and SHARD_AND_DAYS hint addition if the query's date type is one
that should not be expanded when the query's end date is the current
date.

Fixes #2636
  • Loading branch information
@lbschanno
lbschanno committed Dec 6, 2024
1 parent 3a7faf6 commit e726c22
Show file tree
Hide file tree
Showing 5 changed files with 341 additions and 62 deletions.
47 changes: 42 additions & 5 deletions warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -515,6 +515,18 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
*/
private double fieldIndexHoleMinThreshold = 1.0d;

/**
* The set of date types that, if the query's end date is the current date, will NOT result in any date range adjustments or the addition of a
* SHARDS_AND_DAYS hint.
*/
private Set<String> noExpansionIfCurrent = Collections.emptySet();

/**
* Whether the SHARDS_AND_DAYS hint should be allowed for the query. This will be set to false iff the query specified a date type, and the date type is
* present in {@link #noExpansionIfCurrent}, and the query's end date is the current date.
*/
private boolean shardsAndDaysHintAllowed = true;

/**
* Default constructor
*/
Expand Down Expand Up @@ -753,6 +765,8 @@ public void copyFrom(ShardQueryConfiguration other) {
this.setUseQueryTreeScanHintRules(other.isUseQueryTreeScanHintRules());
this.setQueryTreeScanHintRules(other.getQueryTreeScanHintRules());
this.setFieldIndexHoleMinThreshold(other.getFieldIndexHoleMinThreshold());
this.setNoExpansionIfCurrent(other.getNoExpansionIfCurrent() == null ? null : Sets.newHashSet(other.getNoExpansionIfCurrent()));
this.setShardsAndDaysHintAllowed(other.isShardsAndDaysHintAllowed());
}

/**
Expand Down Expand Up @@ -2791,12 +2805,15 @@ public void setSortQueryPostIndexWithTermCounts(boolean sortQueryPostIndexWithTe

@Override
public boolean equals(Object o) {
if (this == o)
if (this == o) {
return true;
if (o == null || getClass() != o.getClass())
}
if (o == null || getClass() != o.getClass()) {
return false;
if (!super.equals(o))
}
if (!super.equals(o)) {
return false;
}
// @formatter:off
ShardQueryConfiguration that = (ShardQueryConfiguration) o;
return isTldQuery() == that.isTldQuery() &&
Expand Down Expand Up @@ -2995,7 +3012,9 @@ public boolean equals(Object o) {
isSortQueryPreIndexWithImpliedCounts() == isSortQueryPreIndexWithImpliedCounts() &&
isSortQueryPreIndexWithFieldCounts() == isSortQueryPreIndexWithFieldCounts() &&
isSortQueryPostIndexWithTermCounts() == isSortQueryPostIndexWithTermCounts() &&
isSortQueryPostIndexWithFieldCounts() == isSortQueryPostIndexWithFieldCounts();
isSortQueryPostIndexWithFieldCounts() == isSortQueryPostIndexWithFieldCounts() &&
Objects.equals(getNoExpansionIfCurrent(), that.getNoExpansionIfCurrent()) &&
isShardsAndDaysHintAllowed() == that.isShardsAndDaysHintAllowed();
// @formatter:on
}

Expand Down Expand Up @@ -3199,7 +3218,9 @@ public int hashCode() {
isSortQueryPreIndexWithImpliedCounts(),
isSortQueryPreIndexWithFieldCounts(),
isSortQueryPostIndexWithTermCounts(),
isSortQueryPostIndexWithFieldCounts()
isSortQueryPostIndexWithFieldCounts(),
getNoExpansionIfCurrent(),
isShardsAndDaysHintAllowed()
);
// @formatter:on
}
Expand Down Expand Up @@ -3234,4 +3255,20 @@ public long getMaxAnyFieldScanTimeMillis() {
public void setMaxAnyFieldScanTimeMillis(long maxAnyFieldScanTimeMillis) {
this.maxAnyFieldScanTimeMillis = maxAnyFieldScanTimeMillis;
}

public Set<String> getNoExpansionIfCurrent() {
return noExpansionIfCurrent;
}

public void setNoExpansionIfCurrent(Set<String> noExpansionIfCurrent) {
this.noExpansionIfCurrent = noExpansionIfCurrent;
}

public boolean isShardsAndDaysHintAllowed() {
return shardsAndDaysHintAllowed;
}

public void setShardsAndDaysHintAllowed(boolean shardsAndDaysHintAllowed) {
this.shardsAndDaysHintAllowed = shardsAndDaysHintAllowed;
}
}
135 changes: 78 additions & 57 deletions warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -834,15 +834,18 @@ protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataH
throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
}

if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
// Do not add a SHARDS_AND_DAYS hint if it is specifically not allowed. This was checked and updated when timedIncludeDateFilters was called.
if (config.isShardsAndDaysHintAllowed()) {
if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
}
}
}

Expand Down Expand Up @@ -2113,57 +2116,73 @@ public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactor
}
}

// if we are using something other than the default of EVENT date
// time, then we need to modify the query
if (!dateType.equals(defaultDateType)) {
// Get the set of date types that should not be expanded if the end date is the current date.
// @formatter:off
Set<String> noExpansionIfCurrent = config.getNoExpansionIfCurrent() == null ? Collections.emptySet() :
config.getNoExpansionIfCurrent().stream()
.map(String::trim)
.map(String::toUpperCase)
.collect(Collectors.toSet());
// @formatter:on

log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
// If the date type is one marked for no expansion if current, and the query's end date is the current date, do not add any date filters, and do not
// allow a SHARDS_AND_DAYS hint to be added later.
if (config.getNoExpansionIfCurrent().contains(dateType) && DateUtils.isSameDay(new Date(), config.getEndDate())) {
log.info("Query end date equals current date and date type " + dateType
+ " is marked for no expansion if current. SHARDS_AND_DAYS hint will be forbidden.");
config.setShardsAndDaysHintAllowed(false);
} else {
// If we are using something other than the default of EVENT date time, then we need to modify the query.
if (!dateType.equals(defaultDateType)) {
log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired
// effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
andChildren.addAll(orChildren);
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
andChildren.addAll(orChildren);
log.info("Date index not needed for this query");
}
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
log.info("Date index not needed for this query");
}

return queryTree;
Expand Down Expand Up @@ -2447,16 +2466,18 @@ public List<IvaratorCacheDirConfig> getShuffledIvaratoCacheDirConfigs(ShardQuery
*/
protected IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, String queryString, Boolean isFullTable,
boolean isPreload) throws DatawaveQueryException {
if (null == settingFuture)
if (null == settingFuture) {
settingFuture = loadQueryIterator(metadataHelper, config, isFullTable, isPreload);
if (settingFuture.isDone())
}
if (settingFuture.isDone()) {
try {
return settingFuture.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e.getCause());
}
else
} else {
return null;
}
}

public void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings)
Expand Down
8 changes: 8 additions & 0 deletions warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2990,4 +2990,12 @@ public void setFieldIndexHoleMinThreshold(double fieldIndexHoleMinThreshold) {
public double getFieldIndexHoleMinThreshold(int fieldIndexHoleMinThreshold) {
return getConfig().getFieldIndexHoleMinThreshold();
}

public Set<String> getNoExpansionIfCurrent() {
return getConfig().getNoExpansionIfCurrent();
}

public void setNoExpansionIfCurrent(Set<String> noExpansionIfCurrent) {
getConfig().setNoExpansionIfCurrent(noExpansionIfCurrent);
}
}
6 changes: 6 additions & 0 deletions warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -596,6 +596,12 @@ public void setUp() throws Exception {
updatedValues.put("useQueryTreeScanHintRules", true);
defaultValues.put("queryTreeScanHintRules", Collections.emptyList());
updatedValues.put("queryTreeScanHintRules", Collections.singletonList(new IvaratorScanHint()));

defaultValues.put("noExpansionIfCurrent", Collections.emptySet());
updatedValues.put("noExpansionIfCurrent", Collections.singleton("EVENT"));

defaultValues.put("shardsAndDaysHintAllowed", true);
updatedValues.put("shardsAndDaysHintAllowed", false);
}

private Query createQuery(String query) {
Expand Down
Loading

0 comments on commit e726c22

Please sign in to comment.