Use quidem to run tests (apache#16249)

* test scoped jdbc driver for druidtest:/// backed DruidAvaticaTestDriver
** DecoupledTestConfig is used inside the URI - this will make it possible to attach to existing things more easily
* DruidQuidemTestBase can be used to create module level set of quidem tests
* added quidem commands: !convertedPlan, !logicalPlan, !druidPlan, !nativePlan
** for these I've used some values of the Hook which was there in calcite
* there are some shortcuts with proxies(they are only used during testing) - we can probably remove those later

.gitignore

@@ -47,3 +47,4 @@ website/i18n/*
 nbproject
 nbactions.xml
 nb-configuration.xml
+*.iq.out

benchmarks/pom.xml

@@ -180,6 +180,31 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-api</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-migrationsupport</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.jupiter</groupId>
+      <artifactId>junit-jupiter-params</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.junit.vintage</groupId>
+      <artifactId>junit-vintage-engine</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.druid.extensions</groupId>
       <artifactId>druid-protobuf-extensions</artifactId>

benchmarks/src/test/resources/META-INF/services/java.sql.Driver

@@ -0,0 +1,2 @@
+org.apache.druid.quidem.DruidAvaticaTestDriver
+org.apache.calcite.avatica.remote.Driver

pom.xml

@@ -140,6 +140,7 @@
 
         <!-- Allow the handful of flaky tests with transient failures to pass. -->
         <surefire.rerunFailingTestsCount>3</surefire.rerunFailingTestsCount>
+        <surefire.trimStackTrace>false</surefire.trimStackTrace>
 
         <!-- Using -DskipTests or -P skip-tests will skip both the unit tests and
              the "new" integration tests. To skip just the unit tests (but run the
@@ -1764,7 +1765,7 @@
                         <!-- Skip the tests which Surefire runs. Surefire runs the unit tests,
                              while its sister plugin, Failsafe, runs the "new" ITs. -->
                         <skipTests>${skipUTs}</skipTests>
-                        <trimStackTrace>false</trimStackTrace>
+                        <trimStackTrace>${surefire.trimStackTrace}</trimStackTrace>
                         <!-- our tests are very verbose, let's keep the volume down -->
                         <redirectTestOutputToFile>true</redirectTestOutputToFile>
                         <forkNode implementation="org.apache.maven.plugin.surefire.extensions.SurefireForkNodeFactory"/>
@@ -2167,6 +2168,7 @@
                                 <exclude>**/.settings/**/*</exclude> <!-- Eclipse -->
                                 <exclude>**/.classpath</exclude> <!-- Eclipse -->
                                 <exclude>**/.project</exclude> <!-- Eclipse -->
+                                <exclude>**/*.iq</exclude> <!-- quidem testfiles -->
                             </excludes>
                         </configuration>
                     </plugin>

quidem

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#--------------------------------------------------------------------
+
+# Utility script for running the new integration tests, since the Maven
+# commands are unwieldy. Allows straightforward usage of ITs on the desktop
+# and in various build scripts. Handles configuration of various kinds.
+
+
+set -e
+
+OPTS+=" -Pskip-static-checks"
+OPTS+=" -Dsurefire.rerunFailingTestsCount=0"
+OPTS+=" -Dorg.slf4j.simpleLogger.log.org.apache.maven.plugin.surefire.SurefirePlugin=INFO"
+[[ $@ =~ "-q" ]] && OPTS+=" -Dsurefire.trimStackTrace=true"
+
+OPTS+=" -pl sql -Dtest=SqlQuidemTest"
+OPTS+=" org.apache.maven.plugins:maven-surefire-plugin:test"
+
+case "$1" in
+ -h|--help)
+	cat << EOF
+Run SqlQuidemTest tests.
+  -q				quiet (recommened)
+  -Dquidem.overwrite		enables overwrite mode
+  -Dquidem.filter=*join*	runs only tests matching path expression
+EOF
+exit 1
+	;;
+esac
+
+exec mvn "$@" $OPTS

sql/pom.xml

@@ -192,6 +192,22 @@
       <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>net.hydromatic</groupId>
+      <artifactId>quidem</artifactId>
+      <version>0.11</version>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpclient</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.httpcomponents</groupId>
+      <artifactId>httpcore</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.junit.jupiter</groupId>
       <artifactId>junit-jupiter-api</artifactId>

sql/src/main/java/org/apache/druid/sql/avatica/DruidMeta.java

@@ -752,7 +752,7 @@ public MetaResultSet getTableTypes(final ConnectionHandle ch)
   }
 
   @VisibleForTesting
-  void closeAllConnections()
+  public void closeAllConnections()
   {
     for (String connectionId : ImmutableSet.copyOf(connections.keySet())) {
       closeConnection(new ConnectionHandle(connectionId));

sql/src/main/java/org/apache/druid/sql/calcite/planner/CalcitePlanner.java

@@ -43,6 +43,7 @@
 import org.apache.calcite.rel.type.RelDataTypeSystem;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexExecutor;
+import org.apache.calcite.runtime.Hook;
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.sql.SqlNode;
 import org.apache.calcite.sql.SqlOperatorTable;
@@ -234,6 +235,7 @@ public SqlNode parse(final Reader reader) throws SqlParseException
   @Override
   public SqlNode validate(SqlNode sqlNode) throws ValidationException
   {
+    Hook.PARSE_TREE.run(new Object[] {null, sqlNode});
     ensure(CalcitePlanner.State.STATE_3_PARSED);
     this.validator = createSqlValidator(createCatalogReader());
     try {

sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java

@@ -39,6 +39,7 @@
 import org.apache.calcite.rel.rules.DateRangeRules;
 import org.apache.calcite.rel.rules.JoinPushThroughJoinRule;
 import org.apache.calcite.rel.rules.PruneEmptyRules;
+import org.apache.calcite.runtime.Hook;
 import org.apache.calcite.sql.SqlExplainFormat;
 import org.apache.calcite.sql.SqlExplainLevel;
 import org.apache.calcite.sql2rel.RelDecorrelator;
@@ -239,17 +240,20 @@ public List<Program> programs(final PlannerContext plannerContext)
     return ImmutableList.of(
         Programs.sequence(
             druidPreProgram,
+            SaveLogicalPlanProgram.INSTANCE,
             Programs.ofRules(druidConventionRuleSet(plannerContext)),
             new LoggingProgram("After Druid volcano planner program", isDebug)
         ),
         Programs.sequence(
             bindablePreProgram,
+            SaveLogicalPlanProgram.INSTANCE,
             Programs.ofRules(bindableConventionRuleSet(plannerContext)),
             new LoggingProgram("After bindable volcano planner program", isDebug)
         ),
         Programs.sequence(
             druidPreProgram,
             buildDecoupledLogicalOptimizationProgram(plannerContext),
+            SaveLogicalPlanProgram.INSTANCE,
             new LoggingProgram("After DecoupledLogicalOptimizationProgram program", isDebug),
             Programs.ofRules(logicalConventionRuleSet(plannerContext)),
             new LoggingProgram("After logical volcano planner program", isDebug)
@@ -368,6 +372,19 @@ private Program buildReductionProgram(final PlannerContext plannerContext, final
     return Programs.of(builder.build(), true, DefaultRelMetadataProvider.INSTANCE);
   }
 
+  private static class SaveLogicalPlanProgram implements Program
+  {
+    public static SaveLogicalPlanProgram INSTANCE = new SaveLogicalPlanProgram();
+
+    @Override
+    public RelNode run(RelOptPlanner planner, RelNode rel, RelTraitSet requiredOutputTraits,
+        List<RelOptMaterialization> materializations, List<RelOptLattice> lattices)
+    {
+      Hook.TRIMMED.run(rel);
+      return rel;
+    }
+  }
+
   private static class LoggingProgram implements Program
   {
     private final String stage;

sql/src/main/java/org/apache/druid/sql/calcite/planner/QueryHandler.java

@@ -47,6 +47,7 @@
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.rex.RexBuilder;
 import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.runtime.Hook;
 import org.apache.calcite.schema.ScannableTable;
 import org.apache.calcite.sql.SqlExplain;
 import org.apache.calcite.sql.SqlNode;
@@ -154,6 +155,7 @@ public void prepare()
     isPrepared = true;
     SqlNode validatedQueryNode = validatedQueryNode();
     rootQueryRel = handlerContext.planner().rel(validatedQueryNode);
+    Hook.CONVERTED.run(rootQueryRel.rel);
     handlerContext.hook().captureQueryRel(rootQueryRel);
     final RelDataTypeFactory typeFactory = rootQueryRel.rel.getCluster().getTypeFactory();
     final SqlValidator validator = handlerContext.planner().getValidator();
@@ -561,6 +563,7 @@ protected PlannerResult planWithDruidConvention() throws ValidationException
                  .plus(DruidLogicalConvention.instance()),
           newRoot
       );
+      Hook.JAVA_PLAN.run(newRoot);
 
       DruidQueryGenerator generator = new DruidQueryGenerator(plannerContext, (DruidLogicalNode) newRoot, rexBuilder);
       DruidQuery baseQuery = generator.buildQuery();

sql/src/main/java/org/apache/druid/sql/calcite/schema/BrokerSegmentMetadataCache.java

@@ -159,7 +159,7 @@ public ServerView.CallbackAction segmentSchemasAnnounced(SegmentSchemas segmentS
   public void start() throws InterruptedException
   {
     log.info("Initializing cache.");
-    cacheExec.submit(this::cacheExecLoop);
+    cacheExec.submit((Runnable) this::cacheExecLoop);
     if (config.isAwaitInitializationOnStart()) {
       awaitInitialization();
     }

sql/src/test/java/org/apache/druid/quidem/DruidAvaticaDriverTest.java

@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.quidem;
+
+import org.junit.jupiter.api.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+public class DruidAvaticaDriverTest
+{
+  // create a new driver instance; this will load the class and register it
+  DruidAvaticaTestDriver driver = new DruidAvaticaTestDriver();
+
+  @Test
+  public void testSelect() throws SQLException
+  {
+    try (Connection con = DriverManager.getConnection("druidtest:///");
+        Statement stmt = con.createStatement();
+        ResultSet rs = stmt.executeQuery("select 42");) {
+      assertTrue(rs.next());
+      assertEquals(42, rs.getInt(1));
+      assertFalse(rs.next());
+    }
+  }
+
+  @Test
+  public void testURIParse() throws SQLException
+  {
+    DruidAvaticaTestDriver.buildConfigfromURIParams("druidtest:///");
+  }
+}