diff --git a/quidem-ut/README.md b/quidem-ut/README.md
index 7ff5feacee62..18d5f16b34bc 100644
--- a/quidem-ut/README.md
+++ b/quidem-ut/README.md
@@ -22,6 +22,9 @@
Enables to write sql level tests easily.
Can be used to write tests against existing test backends (ComponentSupplier) - by doing so the testcases can be moved closer to the exercised codes.
+These tests might come from real usages of Druid by some external tool - by utilizing the capture mode of this module iq tests could be captured and validated later that they retain their results.
+By adding tests for those here could act as an early warning that something might have changed.
+
## Usage
### Install java&maven (if needed)
@@ -44,8 +47,6 @@ git clone https://github.com/apache/druid
```
-
-
### Running these tests
* CI execution happens by a standard JUnit test `QTest` in this module
@@ -72,3 +73,5 @@ git clone https://github.com/apache/druid
* running `dev/quidem -Dquidem.overwrite` updates the resultsets of all `iq` files around there
* rename the testfile to have a more descriptive name
+
+
diff --git a/quidem-ut/pom.xml b/quidem-ut/pom.xml
index 9999c69db13b..dd88d7fc84fa 100644
--- a/quidem-ut/pom.xml
+++ b/quidem-ut/pom.xml
@@ -38,6 +38,10 @@
confluent
https://packages.confluent.io/maven/
+
+ datasets
+ https://raw.githubusercontent.com/kgyrtkirk/datasets/repo/
+
@@ -488,6 +492,11 @@
0.9.3
compile
+
+ com.github.kgyrtkirk.datasets
+ kttm-nested
+ 0.1
+
com.amazonaws
aws-java-sdk-sts
diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/KttmNestedComponentSupplier.java b/quidem-ut/src/main/java/org/apache/druid/quidem/KttmNestedComponentSupplier.java
new file mode 100644
index 000000000000..5939f6b5f6a2
--- /dev/null
+++ b/quidem-ut/src/main/java/org/apache/druid/quidem/KttmNestedComponentSupplier.java
@@ -0,0 +1,153 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.quidem;
+
+import com.google.inject.Injector;
+import org.apache.druid.data.input.InputSource;
+import org.apache.druid.data.input.ResourceInputSource;
+import org.apache.druid.data.input.impl.DimensionSchema;
+import org.apache.druid.data.input.impl.DimensionsSpec;
+import org.apache.druid.data.input.impl.LongDimensionSchema;
+import org.apache.druid.data.input.impl.StringDimensionSchema;
+import org.apache.druid.data.input.impl.TimestampSpec;
+import org.apache.druid.java.util.common.FileUtils;
+import org.apache.druid.java.util.common.Intervals;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.query.NestedDataTestUtils;
+import org.apache.druid.query.QueryRunnerFactoryConglomerate;
+import org.apache.druid.segment.AutoTypeColumnSchema;
+import org.apache.druid.segment.IndexBuilder;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.TestIndex;
+import org.apache.druid.segment.incremental.IncrementalIndex;
+import org.apache.druid.segment.incremental.IncrementalIndexSchema;
+import org.apache.druid.segment.join.JoinableFactoryWrapper;
+import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory;
+import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker;
+import org.apache.druid.sql.calcite.TempDirProducer;
+import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier;
+import org.apache.druid.timeline.DataSegment;
+import org.apache.druid.timeline.partition.NumberedShardSpec;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
+
+public class KttmNestedComponentSupplier extends StandardComponentSupplier
+{
+ public KttmNestedComponentSupplier(TempDirProducer tempDirProducer)
+ {
+ super(tempDirProducer);
+ }
+
+ @Override
+ public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate,
+ JoinableFactoryWrapper joinableFactory, Injector injector)
+ {
+ SpecificSegmentsQuerySegmentWalker walker = super.createQuerySegmentWalker(conglomerate, joinableFactory, injector);
+ QueryableIndex idx = makeKttmIndex(tempDirProducer.newTempFolder());
+
+ walker.add(
+ DataSegment.builder()
+ .dataSource("kttm_nested")
+ .interval(Intervals.of("2019-08-25/2019-08-26"))
+ .version("1")
+ .shardSpec(new NumberedShardSpec(0, 0))
+ .size(0)
+ .build(),
+ idx
+ );
+ return walker;
+ }
+
+ public QueryableIndex makeKttmIndex(File tmpDir)
+ {
+ try {
+ final File directory = new File(tmpDir, StringUtils.format("kttm-index-%s", UUID.randomUUID()));
+ final IncrementalIndex index = makeKttmNestedIndex();
+ TestIndex.INDEX_MERGER.persist(index, directory, IndexSpec.DEFAULT, null);
+ return TestIndex.INDEX_IO.loadIndex(directory);
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public IncrementalIndex makeKttmNestedIndex()
+ {
+ final List dimensions = Arrays.asList(
+ new StringDimensionSchema("session"),
+ new StringDimensionSchema("number"),
+ new AutoTypeColumnSchema("event", null),
+ new AutoTypeColumnSchema("agent", null),
+ new StringDimensionSchema("client_ip"),
+ new StringDimensionSchema("geo_ip"),
+ new StringDimensionSchema("language"),
+ new StringDimensionSchema("adblock_list"),
+ new StringDimensionSchema("app_version"),
+ new StringDimensionSchema("path"),
+ new StringDimensionSchema("loaded_image"),
+ new StringDimensionSchema("referrer"),
+ new StringDimensionSchema("referrer_host"),
+ new StringDimensionSchema("server_ip"),
+ new StringDimensionSchema("screen"),
+ new StringDimensionSchema("window"),
+ new LongDimensionSchema("session_length"),
+ new StringDimensionSchema("timezone"),
+ new LongDimensionSchema("timezone_offset")
+ );
+
+ final File tmpDir;
+ try {
+ tmpDir = FileUtils.createTempDir("test-index-input-source");
+ try {
+ InputSource inputSource = ResourceInputSource.of(
+ TestIndex.class.getClassLoader(),
+ "kttm-nested-v2-2019-08-25.json"
+ );
+ return IndexBuilder
+ .create()
+ .segmentWriteOutMediumFactory(OffHeapMemorySegmentWriteOutMediumFactory.instance())
+ .schema(
+ new IncrementalIndexSchema.Builder()
+ .withRollup(false)
+ .withTimestampSpec(new TimestampSpec("timestamp", null, null))
+ .withDimensionsSpec(new DimensionsSpec(dimensions))
+ .build()
+ )
+ .inputSource(
+ inputSource
+ )
+ .inputFormat(NestedDataTestUtils.DEFAULT_JSON_INPUT_FORMAT)
+ .inputTmpDir(new File(tmpDir, "tmpKttm"))
+ .buildIncrementalIndex();
+ }
+ finally {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+ }
+ catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/quidem-ut/src/test/java/org/apache/druid/quidem/KttmNestedComponentSupplierTest.java b/quidem-ut/src/test/java/org/apache/druid/quidem/KttmNestedComponentSupplierTest.java
new file mode 100644
index 000000000000..a874d3f60b53
--- /dev/null
+++ b/quidem-ut/src/test/java/org/apache/druid/quidem/KttmNestedComponentSupplierTest.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.quidem;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.druid.common.config.NullHandling;
+import org.apache.druid.sql.calcite.BaseCalciteQueryTest;
+import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.ComponentSupplier;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledIf;
+
+@EnabledIf(value = "isReplaceWithDefault", disabledReason = "Needed to provide coverage in defaults mode")
+@ComponentSupplier(KttmNestedComponentSupplier.class)
+public class KttmNestedComponentSupplierTest extends BaseCalciteQueryTest
+{
+ static {
+ NullHandling.initializeForTests();
+ }
+
+ public static boolean isReplaceWithDefault()
+ {
+ return NullHandling.replaceWithDefault();
+ }
+
+ @Test
+ public void testDataset()
+ {
+ msqIncompatible();
+ testBuilder()
+ .sql("SELECT count(1),sum(session_length) from kttm_nested")
+ .expectedResults(ImmutableList.of(new Object[] {465346L, 153573448620L}))
+ .run();
+ }
+}
diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/README.generated b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/README.generated
new file mode 100644
index 000000000000..d18364a641de
--- /dev/null
+++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/README.generated
@@ -0,0 +1,22 @@
+
+
+Tests mentioning this file contain queries which were recorded while using a BI tool that was issuing druid queries behind the scenes.
+
+As these tests are auto-generated there shouldn't be a need to ever change the tests manually - unless there is some backward incompatible change.
diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_Heatmap.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_Heatmap.iq
new file mode 100644
index 000000000000..ec2a3c2a5f62
--- /dev/null
+++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_Heatmap.iq
@@ -0,0 +1,570 @@
+#The queries in this file were generated; see README.generated
+#started Wed Jul 10 19:21:24 IST 2024
+!use druidtest:///?componentSupplier=KttmNestedComponentSupplier
+!set outputformat mysql
+# Wed Jul 10 19:23:06 IST 2024
+SELECT
+MAX(CAST(t."__time" AS TIMESTAMP)) AS "__VALUE__"
+FROM "kttm_nested" AS t
+GROUP BY ();
++-------------------------+
+| __VALUE__ |
++-------------------------+
+| 2019-08-25 23:59:59.686 |
++-------------------------+
+(1 row)
+
+!ok
+# Wed Jul 10 19:26:00 IST 2024
+SELECT
+MAX(CAST(t."__time" AS TIMESTAMP)) AS "__VALUE__"
+FROM "kttm_nested" AS t
+GROUP BY ();
++-------------------------+
+| __VALUE__ |
++-------------------------+
+| 2019-08-25 23:59:59.686 |
++-------------------------+
+(1 row)
+
+!ok
+# Wed Jul 10 19:26:00 IST 2024
+SELECT
+MAX(CAST(t."__time" AS TIMESTAMP)) AS "__VALUE__"
+FROM "kttm_nested" AS t
+GROUP BY ();
++-------------------------+
+| __VALUE__ |
++-------------------------+
+| 2019-08-25 23:59:59.686 |
++-------------------------+
+(1 row)
+
+!ok
+# Wed Jul 10 19:26:01 IST 2024
+SELECT
+(COUNT(*)) AS "__VALUE__"
+FROM "kttm_nested" AS t
+WHERE (TIMESTAMP '2019-08-25 00:00:00'<=CAST(t."__time" AS TIMESTAMP) AND CAST(t."__time" AS TIMESTAMP)