From 2845b6a424acc9b5cea93a95325c1c94c94aeb76 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 8 Aug 2023 03:29:18 -0700 Subject: [PATCH] add new filters to unnest filter pushdown (#14777) --- .../druid/segment/UnnestStorageAdapter.java | 12 +++- .../segment/UnnestStorageAdapterTest.java | 58 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java index 00a119388a64..048cd10f8efa 100644 --- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java @@ -19,6 +19,7 @@ package org.apache.druid.segment; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import org.apache.druid.java.util.common.Pair; @@ -28,8 +29,11 @@ import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.filter.BooleanFilter; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.Filter; import org.apache.druid.query.filter.InDimFilter; +import org.apache.druid.query.filter.NullFilter; +import org.apache.druid.query.filter.RangeFilter; import org.apache.druid.segment.column.ColumnCapabilities; import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.Indexed; @@ -452,7 +456,8 @@ private Filter rewriteFilterOnUnnestColumnIfPossible( * over multi-value strings. (Rather than treat them as arrays.) There isn't a method on the Filter interface that * tells us this, so resort to instanceof. */ - private static boolean filterMapsOverMultiValueStrings(final Filter filter) + @VisibleForTesting + static boolean filterMapsOverMultiValueStrings(final Filter filter) { if (filter instanceof BooleanFilter) { for (Filter child : ((BooleanFilter) filter).getFilters()) { @@ -468,7 +473,10 @@ private static boolean filterMapsOverMultiValueStrings(final Filter filter) return filter instanceof SelectorFilter || filter instanceof InDimFilter || filter instanceof LikeFilter - || filter instanceof BoundFilter; + || filter instanceof BoundFilter + || filter instanceof NullFilter + || filter instanceof EqualityFilter + || filter instanceof RangeFilter; } } diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java index 757a60d59e63..480c86f51fd4 100644 --- a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java @@ -20,6 +20,7 @@ package org.apache.druid.segment; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; @@ -28,11 +29,26 @@ import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.dimension.DefaultDimensionSpec; +import org.apache.druid.query.filter.BoundDimFilter; +import org.apache.druid.query.filter.EqualityFilter; import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.InDimFilter; +import org.apache.druid.query.filter.LikeDimFilter; +import org.apache.druid.query.filter.NullFilter; +import org.apache.druid.query.filter.RangeFilter; import org.apache.druid.query.filter.SelectorDimFilter; import org.apache.druid.segment.column.ColumnCapabilities; +import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.ValueType; +import org.apache.druid.segment.filter.AndFilter; +import org.apache.druid.segment.filter.BoundFilter; +import org.apache.druid.segment.filter.ColumnComparisonFilter; +import org.apache.druid.segment.filter.FalseFilter; +import org.apache.druid.segment.filter.LikeFilter; +import org.apache.druid.segment.filter.NotFilter; import org.apache.druid.segment.filter.OrFilter; +import org.apache.druid.segment.filter.SelectorFilter; +import org.apache.druid.segment.filter.TrueFilter; import org.apache.druid.segment.generator.GeneratorBasicSchemas; import org.apache.druid.segment.generator.GeneratorSchemaInfo; import org.apache.druid.segment.generator.SegmentGenerator; @@ -392,6 +408,48 @@ public void test_pushdown_filters_unnested_dimension_outside() return null; }); } + + @Test + public void testAllowedFiltersForPushdown() + { + Filter[] allowed = new Filter[] { + new SelectorFilter("column", "value"), + new InDimFilter("column", ImmutableSet.of("a", "b")), + new LikeFilter("column", null, LikeDimFilter.LikeMatcher.from("hello%", null), null), + new BoundFilter( + new BoundDimFilter("column", "a", "b", true, true, null, null, null) + ), + NullFilter.forColumn("column"), + new EqualityFilter("column", ColumnType.LONG, 1234L, null), + new RangeFilter("column", ColumnType.LONG, 0L, 1234L, true, false, null) + }; + // not exhaustive + Filter[] notAllowed = new Filter[] { + TrueFilter.instance(), + FalseFilter.instance(), + new ColumnComparisonFilter(ImmutableList.of(DefaultDimensionSpec.of("col1"), DefaultDimensionSpec.of("col2"))) + }; + + for (Filter f : allowed) { + Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f)); + } + for (Filter f : notAllowed) { + Assert.assertFalse(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f)); + } + + Filter notAnd = new NotFilter( + new AndFilter( + Arrays.asList(allowed) + ) + ); + + Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(notAnd)); + Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new OrFilter(Arrays.asList(allowed)))); + Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(notAnd))); + Assert.assertFalse( + UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(new OrFilter(Arrays.asList(notAllowed)))) + ); + } } /**