Skip to content

Commit

Permalink
Add ability to configure date types for no expansion (#2662)
Browse files Browse the repository at this point in the history 
* Add ability to configure date types for no expansion

Add the ability to configure date types that will not result in date
filter and SHARD_AND_DAYS hint addition if the query's date type is one
that should not be expanded when the query's end date is the current
date.

Fixes #2636
  • Loading branch information
@lbschanno
lbschanno authored Jan 14, 2025
1 parent a4750b1 commit dc0cf07
Show file tree
Hide file tree
Showing 5 changed files with 339 additions and 62 deletions.
49 changes: 44 additions & 5 deletions warehouse/query-core/src/main/java/datawave/query/config/ShardQueryConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -527,6 +527,18 @@ public class ShardQueryConfiguration extends GenericQueryConfiguration implement
*/
private double fieldIndexHoleMinThreshold = 1.0d;

/**
* The set of date types that, if the query's end date is the current date, will NOT result in any date range adjustments or the addition of a
* SHARDS_AND_DAYS hint.
*/
private Set<String> noExpansionIfCurrentDateTypes = Collections.emptySet();

/**
* Whether the SHARDS_AND_DAYS hint should be allowed for the query. This will be set to false iff the query specified a date type, and the date type is
* present in {@link #noExpansionIfCurrentDateTypes}, and the query's end date is the current date.
*/
private boolean shardsAndDaysHintAllowed = true;

/**
* Default constructor
*/
Expand Down Expand Up @@ -768,6 +780,9 @@ public void copyFrom(ShardQueryConfiguration other) {
this.setUseQueryTreeScanHintRules(other.isUseQueryTreeScanHintRules());
this.setQueryTreeScanHintRules(other.getQueryTreeScanHintRules());
this.setFieldIndexHoleMinThreshold(other.getFieldIndexHoleMinThreshold());
this.setNoExpansionIfCurrentDateTypes(
other.getNoExpansionIfCurrentDateTypes() == null ? null : Sets.newHashSet(other.getNoExpansionIfCurrentDateTypes()));
this.setShardsAndDaysHintAllowed(other.isShardsAndDaysHintAllowed());
}

/**
Expand Down Expand Up @@ -2832,12 +2847,15 @@ public void setCardinalityThreshold(int cardinalityThreshold) {

@Override
public boolean equals(Object o) {
if (this == o)
if (this == o) {
return true;
if (o == null || getClass() != o.getClass())
}
if (o == null || getClass() != o.getClass()) {
return false;
if (!super.equals(o))
}
if (!super.equals(o)) {
return false;
}
// @formatter:off
ShardQueryConfiguration that = (ShardQueryConfiguration) o;
return isTldQuery() == that.isTldQuery() &&
Expand Down Expand Up @@ -3038,7 +3056,10 @@ public boolean equals(Object o) {
isSortQueryPreIndexWithFieldCounts() == that.isSortQueryPreIndexWithFieldCounts() &&
isSortQueryPostIndexWithTermCounts() == that.isSortQueryPostIndexWithTermCounts() &&
isSortQueryPostIndexWithFieldCounts() == that.isSortQueryPostIndexWithFieldCounts() &&
getCardinalityThreshold() == that.getCardinalityThreshold();
getCardinalityThreshold() == that.getCardinalityThreshold() &&
Objects.equals(getNoExpansionIfCurrentDateTypes(), that.getNoExpansionIfCurrentDateTypes()) &&
isShardsAndDaysHintAllowed() == that.isShardsAndDaysHintAllowed();

// @formatter:on
}

Expand Down Expand Up @@ -3244,7 +3265,9 @@ public int hashCode() {
isSortQueryPreIndexWithFieldCounts(),
isSortQueryPostIndexWithTermCounts(),
isSortQueryPostIndexWithFieldCounts(),
getCardinalityThreshold()
getCardinalityThreshold(),
getNoExpansionIfCurrentDateTypes(),
isShardsAndDaysHintAllowed()
);
// @formatter:on
}
Expand Down Expand Up @@ -3279,4 +3302,20 @@ public long getMaxAnyFieldScanTimeMillis() {
public void setMaxAnyFieldScanTimeMillis(long maxAnyFieldScanTimeMillis) {
this.maxAnyFieldScanTimeMillis = maxAnyFieldScanTimeMillis;
}

public Set<String> getNoExpansionIfCurrentDateTypes() {
return noExpansionIfCurrentDateTypes;
}

public void setNoExpansionIfCurrentDateTypes(Set<String> noExpansionIfCurrentDateTypes) {
this.noExpansionIfCurrentDateTypes = noExpansionIfCurrentDateTypes;
}

public boolean isShardsAndDaysHintAllowed() {
return shardsAndDaysHintAllowed;
}

public void setShardsAndDaysHintAllowed(boolean shardsAndDaysHintAllowed) {
this.shardsAndDaysHintAllowed = shardsAndDaysHintAllowed;
}
}
135 changes: 78 additions & 57 deletions warehouse/query-core/src/main/java/datawave/query/planner/DefaultQueryPlanner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -843,15 +843,18 @@ protected ASTJexlScript updateQueryTree(ScannerFactory scannerFactory, MetadataH
throw new DatawaveFatalQueryException("Found incorrectly marked bounded ranges");
}

if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
// Do not add a SHARDS_AND_DAYS hint if it is specifically not allowed. This was checked and updated when timedIncludeDateFilters was called.
if (config.isShardsAndDaysHintAllowed()) {
if (optionsMap.containsKey(QueryParameters.SHARDS_AND_DAYS)) {
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
} else {
// look for the shards and days hint in the query settings
// the shards and days hint cannot always be specified in the query string when using certain query parsers
Parameter parameter = settings.findParameter(QueryParameters.SHARDS_AND_DAYS);
if (StringUtils.isNotBlank(parameter.getParameterValue())) {
optionsMap.put(QueryParameters.SHARDS_AND_DAYS, parameter.getParameterValue());
config.setQueryTree(timedAddShardsAndDaysFromOptions(timers, config.getQueryTree(), optionsMap));
}
}
}

Expand Down Expand Up @@ -2122,57 +2125,73 @@ public ASTJexlScript addDateFilters(final ASTJexlScript queryTree, ScannerFactor
}
}

// if we are using something other than the default of EVENT date
// time, then we need to modify the query
if (!dateType.equals(defaultDateType)) {
// Get the set of date types that should not be expanded if the end date is the current date.
// @formatter:off
Set<String> noExpansionIfCurrentDateTypes = config.getNoExpansionIfCurrentDateTypes() == null ? Collections.emptySet() :
config.getNoExpansionIfCurrentDateTypes().stream()
.map(String::trim)
.map(String::toUpperCase)
.collect(Collectors.toSet());
// @formatter:on

log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
// If the date type is one marked for no expansion if current, and the query's end date is the current date, do not add any date filters, and do not
// allow a SHARDS_AND_DAYS hint to be added later.
if (config.getNoExpansionIfCurrentDateTypes().contains(dateType) && DateUtils.isSameDay(new Date(), config.getEndDate())) {
log.info("Query end date equals current date and date type " + dateType
+ " is marked for no expansion if current. SHARDS_AND_DAYS hint will be forbidden.");
config.setShardsAndDaysHintAllowed(false);
} else {
// If we are using something other than the default of EVENT date time, then we need to modify the query.
if (!dateType.equals(defaultDateType)) {
log.info("Using the date index for " + dateType);
// if no date index helper configured, then we are in error
if (dateIndexHelper == null) {
throw new DatawaveQueryException("Requested date range of type " + dateType + " but no date index is configured");
}
// get all of the fields used for this date type
DateIndexHelper.DateTypeDescription dateIndexData = dateIndexHelper.getTypeDescription(dateType, config.getBeginDate(), config.getEndDate(),
config.getDatatypeFilter());
if (dateIndexData.getFields().isEmpty()) {
log.warn("The specified date type: " + dateType + " is unknown for the specified data types");
// If this is the case, then essentially we have no dates to search. Adding the filter function with _NO_FIELD_ will have the desired
// effect.
// Also it will be understandable from the plan as to why no results were returned.
dateIndexData.getFields().add(Constants.NO_FIELD);
}
log.info("Adding date filters for the following fields: " + dateIndexData.getFields());
// now for each field, add an expression to filter that date
List<JexlNode> andChildren = new ArrayList<>();
for (int i = 0; i < queryTree.jjtGetNumChildren(); i++) {
if (queryTree.jjtGetChild(i) instanceof ASTAndNode) {
andChildren.add(queryTree.jjtGetChild(i));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
} else {
andChildren.add(JexlNodeFactory.createExpression(queryTree.jjtGetChild(i)));
andChildren.addAll(orChildren);
}
}
List<JexlNode> orChildren = new ArrayList<>();
for (String field : dateIndexData.getFields()) {
orChildren.add(createDateFilter(dateType, field, config.getBeginDate(), config.getEndDate()));
}
if (orChildren.size() > 1) {
andChildren.add(JexlNodeFactory.createOrNode(orChildren));
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
andChildren.addAll(orChildren);
log.info("Date index not needed for this query");
}
JexlNode andNode = JexlNodeFactory.createAndNode(andChildren);
JexlNodeFactory.setChildren(queryTree, Collections.singleton(andNode));

// now lets update the query parameters with the correct start and
// end dates
log.info("Remapped " + dateType + " dates [" + config.getBeginDate() + "," + config.getEndDate() + "] to EVENT dates "
+ dateIndexData.getBeginDate() + "," + dateIndexData.getEndDate());

// reset the dates in the configuration, no need to reset then in
// the Query settings object
config.setBeginDate(dateIndexData.getBeginDate());
config.setEndDate(dateIndexData.getEndDate());
} else {
log.info("Date index not needed for this query");
}

return queryTree;
Expand Down Expand Up @@ -2456,16 +2475,18 @@ public List<IvaratorCacheDirConfig> getShuffledIvaratoCacheDirConfigs(ShardQuery
*/
protected IteratorSetting getQueryIterator(MetadataHelper metadataHelper, ShardQueryConfiguration config, String queryString, Boolean isFullTable,
boolean isPreload) throws DatawaveQueryException {
if (null == settingFuture)
if (null == settingFuture) {
settingFuture = loadQueryIterator(metadataHelper, config, isFullTable, isPreload);
if (settingFuture.isDone())
}
if (settingFuture.isDone()) {
try {
return settingFuture.get();
} catch (InterruptedException | ExecutionException e) {
throw new RuntimeException(e.getCause());
}
else
} else {
return null;
}
}

public void configureTypeMappings(ShardQueryConfiguration config, IteratorSetting cfg, MetadataHelper metadataHelper, boolean compressMappings)
Expand Down
8 changes: 8 additions & 0 deletions warehouse/query-core/src/main/java/datawave/query/tables/ShardQueryLogic.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3027,4 +3027,12 @@ public void setFieldIndexHoleMinThreshold(double fieldIndexHoleMinThreshold) {
public double getFieldIndexHoleMinThreshold(int fieldIndexHoleMinThreshold) {
return getConfig().getFieldIndexHoleMinThreshold();
}

public Set<String> getNoExpansionIfCurrentDateTypes() {
return getConfig().getNoExpansionIfCurrentDateTypes();
}

public void setNoExpansionIfCurrentDateTypes(Set<String> noExpansionIfCurrentDateTypes) {
getConfig().setNoExpansionIfCurrentDateTypes(noExpansionIfCurrentDateTypes);
}
}
6 changes: 6 additions & 0 deletions warehouse/query-core/src/test/java/datawave/query/config/ShardQueryConfigurationTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -604,6 +604,12 @@ public void setUp() throws Exception {
updatedValues.put("useQueryTreeScanHintRules", true);
defaultValues.put("queryTreeScanHintRules", Collections.emptyList());
updatedValues.put("queryTreeScanHintRules", Collections.singletonList(new IvaratorScanHint()));

defaultValues.put("noExpansionIfCurrentDateTypes", Collections.emptySet());
updatedValues.put("noExpansionIfCurrentDateTypes", Collections.singleton("EVENT"));

defaultValues.put("shardsAndDaysHintAllowed", true);
updatedValues.put("shardsAndDaysHintAllowed", false);
}

private Query createQuery(String query) {
Expand Down
Loading

0 comments on commit dc0cf07

Please sign in to comment.