add new filters to unnest filter pushdown (apache#14777)

zachjsh · Aug 8, 2023 · 2845b6a · 2845b6a
1 parent d0403f0
commit 2845b6a
Show file tree

Hide file tree

Showing 2 changed files with 68 additions and 2 deletions.
diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
@@ -19,6 +19,7 @@
 
 package org.apache.druid.segment;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
 import org.apache.druid.java.util.common.Pair;
@@ -28,8 +29,11 @@
 import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.filter.BooleanFilter;
 import org.apache.druid.query.filter.DimFilter;
+import org.apache.druid.query.filter.EqualityFilter;
 import org.apache.druid.query.filter.Filter;
 import org.apache.druid.query.filter.InDimFilter;
+import org.apache.druid.query.filter.NullFilter;
+import org.apache.druid.query.filter.RangeFilter;
 import org.apache.druid.segment.column.ColumnCapabilities;
 import org.apache.druid.segment.column.ValueType;
 import org.apache.druid.segment.data.Indexed;
@@ -452,7 +456,8 @@ private Filter rewriteFilterOnUnnestColumnIfPossible(
    * over multi-value strings. (Rather than treat them as arrays.) There isn't a method on the Filter interface that
    * tells us this, so resort to instanceof.
    */
-  private static boolean filterMapsOverMultiValueStrings(final Filter filter)
+  @VisibleForTesting
+  static boolean filterMapsOverMultiValueStrings(final Filter filter)
   {
     if (filter instanceof BooleanFilter) {
       for (Filter child : ((BooleanFilter) filter).getFilters()) {
@@ -468,7 +473,10 @@ private static boolean filterMapsOverMultiValueStrings(final Filter filter)
       return filter instanceof SelectorFilter
              || filter instanceof InDimFilter
              || filter instanceof LikeFilter
-             || filter instanceof BoundFilter;
+             || filter instanceof BoundFilter
+             || filter instanceof NullFilter
+             || filter instanceof EqualityFilter
+             || filter instanceof RangeFilter;
     }
   }
 

diff --git a/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java b/processing/src/test/java/org/apache/druid/segment/UnnestStorageAdapterTest.java
@@ -20,6 +20,7 @@
 package org.apache.druid.segment;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
 import org.apache.druid.java.util.common.DateTimes;
 import org.apache.druid.java.util.common.granularity.Granularities;
 import org.apache.druid.java.util.common.granularity.Granularity;
@@ -28,11 +29,26 @@
 import org.apache.druid.math.expr.ExprMacroTable;
 import org.apache.druid.query.QueryMetrics;
 import org.apache.druid.query.dimension.DefaultDimensionSpec;
+import org.apache.druid.query.filter.BoundDimFilter;
+import org.apache.druid.query.filter.EqualityFilter;
 import org.apache.druid.query.filter.Filter;
+import org.apache.druid.query.filter.InDimFilter;
+import org.apache.druid.query.filter.LikeDimFilter;
+import org.apache.druid.query.filter.NullFilter;
+import org.apache.druid.query.filter.RangeFilter;
 import org.apache.druid.query.filter.SelectorDimFilter;
 import org.apache.druid.segment.column.ColumnCapabilities;
+import org.apache.druid.segment.column.ColumnType;
 import org.apache.druid.segment.column.ValueType;
+import org.apache.druid.segment.filter.AndFilter;
+import org.apache.druid.segment.filter.BoundFilter;
+import org.apache.druid.segment.filter.ColumnComparisonFilter;
+import org.apache.druid.segment.filter.FalseFilter;
+import org.apache.druid.segment.filter.LikeFilter;
+import org.apache.druid.segment.filter.NotFilter;
 import org.apache.druid.segment.filter.OrFilter;
+import org.apache.druid.segment.filter.SelectorFilter;
+import org.apache.druid.segment.filter.TrueFilter;
 import org.apache.druid.segment.generator.GeneratorBasicSchemas;
 import org.apache.druid.segment.generator.GeneratorSchemaInfo;
 import org.apache.druid.segment.generator.SegmentGenerator;
@@ -392,6 +408,48 @@ public void test_pushdown_filters_unnested_dimension_outside()
       return null;
     });
   }
+
+  @Test
+  public void testAllowedFiltersForPushdown()
+  {
+    Filter[] allowed = new Filter[] {
+        new SelectorFilter("column", "value"),
+        new InDimFilter("column", ImmutableSet.of("a", "b")),
+        new LikeFilter("column", null, LikeDimFilter.LikeMatcher.from("hello%", null), null),
+        new BoundFilter(
+            new BoundDimFilter("column", "a", "b", true, true, null, null, null)
+        ),
+        NullFilter.forColumn("column"),
+        new EqualityFilter("column", ColumnType.LONG, 1234L, null),
+        new RangeFilter("column", ColumnType.LONG, 0L, 1234L, true, false, null)
+    };
+    // not exhaustive
+    Filter[] notAllowed = new Filter[] {
+        TrueFilter.instance(),
+        FalseFilter.instance(),
+        new ColumnComparisonFilter(ImmutableList.of(DefaultDimensionSpec.of("col1"), DefaultDimensionSpec.of("col2")))
+    };
+
+    for (Filter f : allowed) {
+      Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
+    }
+    for (Filter f : notAllowed) {
+      Assert.assertFalse(UnnestStorageAdapter.filterMapsOverMultiValueStrings(f));
+    }
+
+    Filter notAnd = new NotFilter(
+        new AndFilter(
+            Arrays.asList(allowed)
+        )
+    );
+
+    Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(notAnd));
+    Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new OrFilter(Arrays.asList(allowed))));
+    Assert.assertTrue(UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(notAnd)));
+    Assert.assertFalse(
+        UnnestStorageAdapter.filterMapsOverMultiValueStrings(new NotFilter(new OrFilter(Arrays.asList(notAllowed))))
+    );
+  }
 }
 
 /**