From b1a62a3e1539017f8d7ce4cb56a8e06a5d7ee432 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 08:33:54 +0530 Subject: [PATCH 01/12] add count,sum,avg,min,max to group request --- .../foxtrot/common/group/GroupRequest.java | 45 ++-- .../core/querystore/actions/GroupAction.java | 68 +++++- .../querystore/actions/GroupActionTest.java | 214 +++++++++++++++++- .../flipkart/foxtrot/sql/FqlQueryType.java | 4 + .../flipkart/foxtrot/sql/QueryTranslator.java | 66 +++++- .../sql/{ => constants}/Constants.java | 2 +- .../sql/constants/FqlFunctionType.java | 19 ++ .../com/flipkart/foxtrot/sql/ParseTest.java | 7 + 8 files changed, 369 insertions(+), 56 deletions(-) rename foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/{ => constants}/Constants.java (84%) create mode 100644 foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/FqlFunctionType.java diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java index cd0545d23..7c00a1bfe 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java @@ -15,11 +15,16 @@ */ package com.flipkart.foxtrot.common.group; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionRequestVisitor; import com.flipkart.foxtrot.common.Opcodes; import com.flipkart.foxtrot.common.enums.CountPrecision; import com.flipkart.foxtrot.common.query.Filter; +import com.flipkart.foxtrot.common.stats.Stat; +import com.flipkart.foxtrot.common.util.CollectionUtils; +import com.google.common.collect.Sets; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -27,6 +32,8 @@ import javax.validation.constraints.NotNull; import java.util.List; +import java.util.Objects; +import java.util.Set; /** * User: Santanu Sinha (santanu.sinha@flipkart.com) @@ -41,8 +48,13 @@ public class GroupRequest extends ActionRequest { @NotEmpty private String table; + // Kept for backward compatibility private String uniqueCountOn; + private String aggregationField; + + private Set stats; + @NotNull @NotEmpty private List nesting; @@ -53,46 +65,29 @@ public GroupRequest() { super(Opcodes.GROUP); } - public GroupRequest(List filters, String table, String uniqueCountOn, List nesting) { + public GroupRequest(List filters, String table, String uniqueCountOn, + String aggregationField, Set stats, + List nesting, CountPrecision precision) { super(Opcodes.GROUP, filters); this.table = table; this.uniqueCountOn = uniqueCountOn; + this.aggregationField = aggregationField; + this.stats = stats; this.nesting = nesting; + this.precision = precision; } public T accept(ActionRequestVisitor visitor) { return visitor.visit(this); } - public String getTable() { - return table; - } - - public void setTable(String table) { - this.table = table; - } - - public String getUniqueCountOn() { - return uniqueCountOn; - } - - public void setUniqueCountOn(String uniqueCountOn) { - this.uniqueCountOn = uniqueCountOn; - } - - public List getNesting() { - return nesting; - } - - public void setNesting(List nesting) { - this.nesting = nesting; - } - @Override public String toString() { return new ToStringBuilder(this).appendSuper(super.toString()) .append("table", table) + .append("stats", stats) .append("uniqueCountOn", uniqueCountOn) + .append("aggregationField", aggregationField) .append("nesting", nesting) .toString(); } diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index ef37c6b82..83751d5f4 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -31,6 +31,7 @@ import com.flipkart.foxtrot.common.query.general.NotInFilter; import com.flipkart.foxtrot.common.query.numeric.*; import com.flipkart.foxtrot.common.query.string.ContainsFilter; +import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.common.util.CollectionUtils; import com.flipkart.foxtrot.common.visitor.CountPrecisionThresholdVisitorAdapter; import com.flipkart.foxtrot.core.common.Action; @@ -44,6 +45,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.flipkart.foxtrot.core.table.TableMetadataManager; +import com.google.api.client.repackaged.com.google.common.base.Strings; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.SneakyThrows; @@ -52,9 +54,11 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.search.aggregations.AbstractAggregationBuilder; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.terms.Terms; import org.elasticsearch.search.aggregations.metrics.cardinality.Cardinality; +import org.elasticsearch.search.aggregations.metrics.cardinality.CardinalityAggregationBuilder; import org.joda.time.Interval; import java.util.*; @@ -104,11 +108,17 @@ public String getRequestCacheKey() { } } - if (null != query.getUniqueCountOn()) { - filterHashKey += 31 * query.getUniqueCountOn() + if (null != query.getAggregationField()) { + filterHashKey += 31 * query.getAggregationField() .hashCode(); } + if(!CollectionUtils.isNullOrEmpty(query.getStats())){ + for(Stat stat : query.getStats()){ + filterHashKey += 31 * stat.hashCode(); + } + } + for (int i = 0; i < query.getNesting() .size(); i++) { filterHashKey += 31 * query.getNesting() @@ -138,7 +148,12 @@ public void validateImpl(GroupRequest parameter) { if (parameter.getUniqueCountOn() != null && parameter.getUniqueCountOn() .isEmpty()) { - validationErrors.add("unique field cannot be empty (can be null)"); + validationErrors.add("uniqueCountOn cannot be empty (can be null)"); + } + + if (parameter.getAggregationField() != null && parameter.getAggregationField() + .isEmpty()) { + validationErrors.add("aggregation field cannot be empty (can be null)"); } validateCardinality(parameter); @@ -717,18 +732,42 @@ private Long getValidCount(Long count) { : count; } - private AbstractAggregationBuilder buildAggregation(GroupRequest parameter) { + private AbstractAggregationBuilder buildAggregation(GroupRequest groupRequest) { return Utils.buildTermsAggregation(getParameter().getNesting() .stream() .map(x -> new ResultSort(x, ResultSort.Order.asc)) - .collect(Collectors.toList()), - !CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn()) - ? Sets.newHashSet( - Utils.buildCardinalityAggregation(getParameter().getUniqueCountOn(), - parameter.accept(new CountPrecisionThresholdVisitorAdapter( - elasticsearchTuningConfig.getPrecisionThreshold())))) - : Sets.newHashSet(), elasticsearchTuningConfig.getAggregationSize()); + .collect(Collectors.toList()),buildSubAggregation(getParameter()), + elasticsearchTuningConfig.getAggregationSize()); + + } + private Set buildSubAggregation(GroupRequest groupRequest) { + // Keep this for backward compatibility to support uniqueCountOn attribute coming from old requests + if(!Strings.isNullOrEmpty(groupRequest.getUniqueCountOn())){ + return Sets.newHashSet(buildCardinalityAggregation(groupRequest.getUniqueCountOn(), groupRequest)); + } + + if(Strings.isNullOrEmpty(groupRequest.getAggregationField())){ + return Sets.newHashSet(); + } + + boolean isNumericField = Utils.isNumericField(getTableMetadataManager(), groupRequest.getTable(), + groupRequest.getAggregationField()); + final AbstractAggregationBuilder groupAggStats; + if (isNumericField) { + groupAggStats = Utils.buildStatsAggregation(groupRequest.getAggregationField(), + groupRequest.getStats()); + } else { + groupAggStats = buildCardinalityAggregation(groupRequest.getAggregationField(), groupRequest); + } + return Sets.newHashSet(groupAggStats); + } + + private CardinalityAggregationBuilder buildCardinalityAggregation(String aggregationField, + GroupRequest groupRequest) { + return Utils.buildCardinalityAggregation(aggregationField, + groupRequest.accept(new CountPrecisionThresholdVisitorAdapter( + elasticsearchTuningConfig.getPrecisionThreshold()))); } private Map getMap(List fields, Aggregations aggregations) { @@ -740,12 +779,17 @@ private Map getMap(List fields, Aggregations aggregation Map levelCount = Maps.newHashMap(); for (Terms.Bucket bucket : terms.getBuckets()) { if (fields.size() == 1) { - if (!CollectionUtils.isNullOrEmpty(getParameter().getUniqueCountOn())) { + if (!Strings.isNullOrEmpty(getParameter().getUniqueCountOn())) { String key = Utils.sanitizeFieldForAggregation(getParameter().getUniqueCountOn()); Cardinality cardinality = bucket.getAggregations() .get(key); levelCount.put(String.valueOf(bucket.getKey()), cardinality.getValue()); } + else if (!Strings.isNullOrEmpty(getParameter().getAggregationField())) { + String metricKey = Utils.getExtendedStatsAggregationKey(getParameter().getAggregationField()); + levelCount.put(String.valueOf(bucket.getKey()), Utils.toStats( + bucket.getAggregations().getAsMap().get(metricKey))); + } else { levelCount.put(String.valueOf(bucket.getKey()), bucket.getDocCount()); } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java index 73ac89b16..1f2c58090 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java @@ -22,11 +22,14 @@ import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.query.general.EqualsFilter; import com.flipkart.foxtrot.common.query.numeric.GreaterThanFilter; +import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.core.TestUtils; import com.flipkart.foxtrot.core.exception.ErrorCode; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchQueryStore; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; @@ -85,7 +88,7 @@ public void testGroupActionSingleFieldNoFilter() throws FoxtrotException, JsonPr response.put("android", 7L); response.put("ios", 4L); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -108,7 +111,7 @@ public void testGroupActionSingleFieldEmptyFieldNoFilter() throws FoxtrotExcepti public void testGroupActionSingleFieldSpecialCharactersNoFilter() throws FoxtrotException, JsonProcessingException { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); - groupRequest.setNesting(Arrays.asList("")); + groupRequest.setNesting(Collections.singletonList("")); try { getQueryExecutor().execute(groupRequest); @@ -128,11 +131,11 @@ public void testGroupActionSingleFieldHavingSpecialCharactersWithFilter() throws equalsFilter.setField("device"); equalsFilter.setValue("nexus"); groupRequest.setFilters(Collections.singletonList(equalsFilter)); - groupRequest.setNesting(Arrays.asList("!@#$%^&*()")); + groupRequest.setNesting(Collections.singletonList("!@#$%^&*()")); Map response = Maps.newHashMap(); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -145,13 +148,13 @@ public void testGroupActionSingleFieldWithFilter() throws FoxtrotException, Json equalsFilter.setField("device"); equalsFilter.setValue("nexus"); groupRequest.setFilters(Collections.singletonList(equalsFilter)); - groupRequest.setNesting(Arrays.asList("os")); + groupRequest.setNesting(Collections.singletonList("os")); Map response = Maps.newHashMap(); response.put("android", 5L); response.put("ios", 1L); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -172,7 +175,7 @@ public void testGroupActionTwoFieldsNoFilter() throws FoxtrotException, JsonProc put("iphone", 1L); }}); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -196,7 +199,7 @@ public void testGroupActionTwoFieldsWithFilter() throws FoxtrotException, JsonPr put("ipad", 1L); }}); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -238,7 +241,7 @@ public void testGroupActionMultipleFieldsNoFilter() throws FoxtrotException, Jso }}); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -275,7 +278,198 @@ public void testGroupActionMultipleFieldsWithFilter() throws FoxtrotException, J put("ipad", iPadResponse); }}); - GroupResponse actualResult = GroupResponse.class.cast(getQueryExecutor().execute(groupRequest)); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } + + @Test + @SuppressWarnings("unchecked") + public void testGroupActionDistinctCountAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setUniqueCountOn("device"); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", 1L); + put("2", 2L); + put("3", 2L); + }}); + response.put("ios",new HashMap() {{ + put("1", 1L); + put("2", 2L); + }}); + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) response.get("android")).get("1"), + ((Map) actualResult.getResult().get("android")).get("1")); + assertEquals(((Map) response.get("android")).get("2"), + ((Map) actualResult.getResult().get("android")).get("2")); + assertEquals(((Map) response.get("android")).get("3"), + ((Map) actualResult.getResult().get("android")).get("3")); + assertEquals(((Map) response.get("ios")).get("1"), + ((Map) actualResult.getResult().get("ios")).get("1")); + assertEquals(((Map) response.get("ios")).get("2"), + ((Map) actualResult.getResult().get("ios")).get("2")); + } + + @Test + @SuppressWarnings("unchecked") + public void testGroupActionMaxAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setAggregationField("battery"); + groupRequest.setStats(Sets.newHashSet(Stat.MAX)); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", ImmutableMap.of("max",48.0)); + put("2", ImmutableMap.of("max",99.0)); + put("3", ImmutableMap.of("max",87.0)); + }}); + response.put("ios",new HashMap() {{ + put("1", ImmutableMap.of("max",24.0)); + put("2", ImmutableMap.of("max",56.0)); + }}); + + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("max"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("max")); + assertEquals(((Map) ((Map) response.get("android")).get("2")).get("max"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("2")).get("max")); + assertEquals(((Map) ((Map) response.get("android")).get("3")).get("max"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("3")).get("max")); + assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("max"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("1")).get("max")); + assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("max"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("2")).get("max")); + } + + @Test + @SuppressWarnings("unchecked") + public void testGroupActionAvgAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setAggregationField("battery"); + groupRequest.setStats(Sets.newHashSet(Stat.AVG)); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", ImmutableMap.of("avg",36.0)); + put("2", ImmutableMap.of("avg",84.33333333333333)); + put("3", ImmutableMap.of("avg",80.5)); + }}); + response.put("ios",new HashMap() {{ + put("1", ImmutableMap.of("avg",24.0)); + put("2", ImmutableMap.of("avg",45.0)); + }}); + + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("avg")); + assertEquals(((Map) ((Map) response.get("android")).get("2")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("2")).get("avg")); + assertEquals(((Map) ((Map) response.get("android")).get("3")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("3")).get("avg")); + assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("1")).get("avg")); + assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("2")).get("avg")); + } + + @Test + @SuppressWarnings("unchecked") + public void testGroupActionSumAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setAggregationField("battery"); + groupRequest.setStats(Sets.newHashSet(Stat.SUM)); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", ImmutableMap.of("sum",72.0)); + put("2", ImmutableMap.of("sum",253.0)); + put("3", ImmutableMap.of("sum",161.0)); + }}); + response.put("ios",new HashMap() {{ + put("1", ImmutableMap.of("sum",24.0)); + put("2", ImmutableMap.of("sum",135.0)); + }}); + + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("sum")); + assertEquals(((Map) ((Map) response.get("android")).get("2")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("2")).get("sum")); + assertEquals(((Map) ((Map) response.get("android")).get("3")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("3")).get("sum")); + assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("1")).get("sum")); + assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("2")).get("sum")); + } + + @Test + @SuppressWarnings("unchecked") + public void testGroupActionCountAndSumAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setAggregationField("battery"); + groupRequest.setStats(Sets.newHashSet(Stat.SUM, Stat.COUNT)); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", ImmutableMap.of("avg",36.0, "min",24.0, + "max",48.0, "count",2, "sum",72.0)); + put("2", ImmutableMap.of("avg",84.33333333333333, "min",76.0, + "max",99.0, "count", 3, "sum",253.0)); + put("3", ImmutableMap.of("avg",80.5, "min",74.0, + "max",87.0, "count",2, "sum",161.0)); + }}); + response.put("ios",new HashMap() {{ + put("1", ImmutableMap.of("avg",24.0, "min",24.0, + "max",24.0, "count",1, "sum",24.0)); + put("2", ImmutableMap.of("avg",45.0, "min", 35.0, + "max",56.0, "count", 3, "sum",135.0)); + }}); + + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("avg")); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("sum"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("sum")); + assertEquals(((Map) ((Map) response.get("android")).get("2")).get("min"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("2")).get("min")); + assertEquals(((Map) ((Map) response.get("android")).get("3")).get("max"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("3")).get("max")); + assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("max"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("1")).get("max")); + assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("avg"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("2")).get("avg")); + } } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/FqlQueryType.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/FqlQueryType.java index c82de948b..2913d346b 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/FqlQueryType.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/FqlQueryType.java @@ -10,5 +10,9 @@ public enum FqlQueryType { DESC, SHOWTABLES, COUNT, + SUM, + MAX, + MIN, + AVG, DISTINCT } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index b5461b0b6..bea9bc69d 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -14,10 +14,14 @@ import com.flipkart.foxtrot.common.query.general.*; import com.flipkart.foxtrot.common.query.numeric.*; import com.flipkart.foxtrot.common.query.string.ContainsFilter; +import com.flipkart.foxtrot.common.stats.AnalyticsRequestFlags; +import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.common.stats.StatsRequest; import com.flipkart.foxtrot.common.stats.StatsTrendRequest; import com.flipkart.foxtrot.common.trend.TrendRequest; import com.flipkart.foxtrot.core.exception.FqlParsingException; +import com.flipkart.foxtrot.sql.constants.Constants; +import com.flipkart.foxtrot.sql.constants.FqlFunctionType; import com.flipkart.foxtrot.sql.extendedsql.ExtendedSqlStatement; import com.flipkart.foxtrot.sql.extendedsql.desc.Describe; import com.flipkart.foxtrot.sql.extendedsql.showtables.ShowTables; @@ -26,6 +30,7 @@ import com.flipkart.foxtrot.sql.query.FqlShowTablesQuery; import com.flipkart.foxtrot.sql.util.QueryUtils; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; import io.dropwizard.util.Duration; import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.*; @@ -42,6 +47,7 @@ import java.io.StringReader; import java.util.List; +import java.util.Set; public class QueryTranslator extends SqlElementVisitor { private static final Logger logger = LoggerFactory.getLogger(QueryTranslator.class.getSimpleName()); @@ -244,7 +250,7 @@ private ActionRequest createGroupActionRequest() { group.setTable(tableName); group.setNesting(groupBycolumnsList); group.setFilters(filters); - setUniqueCountOn(group); + setGroupAggregation(group); return group; } @@ -305,13 +311,17 @@ private ResultSort generateResultSort(List orderByElements) { return resultSortColumn; } - private void setUniqueCountOn(GroupRequest group) { + private void setGroupAggregation(GroupRequest group) { if(calledAction instanceof CountRequest) { CountRequest countRequest = (CountRequest)this.calledAction; boolean distinct = countRequest.isDistinct(); if(distinct) { group.setUniqueCountOn(countRequest.getField()); } + } else if (calledAction instanceof StatsRequest){ + StatsRequest statsRequest = (StatsRequest) this.calledAction; + group.setStats(statsRequest.getStats()); + group.setAggregationField(statsRequest.getField()); } } @@ -358,7 +368,25 @@ public void visit(SelectExpressionItem selectExpressionItem) { actionRequest = parseHistogramRequest(function.getParameters()); break; case COUNT: - actionRequest = parseCountRequest(function.getParameters(), function.isAllColumns(), function.isDistinct()); + actionRequest = parseCountRequest(function.getParameters(), + function.isAllColumns(), + function.isDistinct()); + break; + case SUM: + actionRequest = parseStatsFunction(function.getParameters() + .getExpressions(), Sets.newHashSet(Stat.SUM)); + break; + case MIN: + actionRequest = parseStatsFunction(function.getParameters() + .getExpressions(), Sets.newHashSet(Stat.MIN)); + break; + case MAX: + actionRequest = parseStatsFunction(function.getParameters() + .getExpressions(), Sets.newHashSet(Stat.MAX)); + break; + case AVG: + actionRequest = parseStatsFunction(function.getParameters() + .getExpressions(), Sets.newHashSet(Stat.AVG)); break; case DESC: case SELECT: @@ -378,21 +406,33 @@ public void visit(SelectExpressionItem selectExpressionItem) { } private FqlQueryType getType(String function) { - if(function.equalsIgnoreCase("trend")) { + if(function.equalsIgnoreCase(FqlFunctionType.TREND)){ return FqlQueryType.TREND; } - if(function.equalsIgnoreCase("statstrend")) { + if(function.equalsIgnoreCase(FqlFunctionType.STATSTREND)) { return FqlQueryType.STATSTREND; } - if(function.equalsIgnoreCase("stats")) { + if(function.equalsIgnoreCase(FqlFunctionType.STATS)) { return FqlQueryType.STATS; } - if(function.equalsIgnoreCase("histogram")) { + if(function.equalsIgnoreCase(FqlFunctionType.HISTOGRAM)) { return FqlQueryType.HISTOGRAM; } - if(function.equalsIgnoreCase("count")) { + if(function.equalsIgnoreCase(FqlFunctionType.COUNT)) { return FqlQueryType.COUNT; } + if(function.equalsIgnoreCase(FqlFunctionType.SUM)) { + return FqlQueryType.SUM; + } + if(function.equalsIgnoreCase(FqlFunctionType.AVG)) { + return FqlQueryType.AVG; + } + if(function.equalsIgnoreCase(FqlFunctionType.MIN)) { + return FqlQueryType.MIN; + } + if(function.equalsIgnoreCase(FqlFunctionType.MAX)) { + return FqlQueryType.MAX; + } return FqlQueryType.SELECT; } @@ -425,6 +465,16 @@ private StatsTrendRequest parseStatsTrendFunction(List expressions) { return statsTrendRequest; } + /* + When asked for specific stats then add those stats and skip percentiles to save on execution time + */ + private StatsRequest parseStatsFunction(List expressions, Set stats) { + StatsRequest statsRequest = parseStatsFunction(expressions); + statsRequest.setStats(stats); + statsRequest.setFlags(Sets.newHashSet(AnalyticsRequestFlags.STATS_SKIP_PERCENTILES)); + return statsRequest; + } + private StatsRequest parseStatsFunction(List expressions) { if(expressions == null || expressions.isEmpty() || expressions.size() > 1) { throw new FqlParsingException("stats function has following format: stats(fieldname)"); diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/Constants.java similarity index 84% rename from foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java rename to foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/Constants.java index 715e3e22c..74f939d09 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/Constants.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/Constants.java @@ -1,4 +1,4 @@ -package com.flipkart.foxtrot.sql; +package com.flipkart.foxtrot.sql.constants; /** * Created by rishabh.goyal on 17/11/14. diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/FqlFunctionType.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/FqlFunctionType.java new file mode 100644 index 000000000..d54d905c6 --- /dev/null +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/constants/FqlFunctionType.java @@ -0,0 +1,19 @@ +package com.flipkart.foxtrot.sql.constants; + +public class FqlFunctionType { + + private FqlFunctionType(){ + throw new IllegalStateException("Utility Class"); + } + + public static final String TREND = "trend"; + public static final String STATSTREND = "statstrend"; + public static final String STATS = "stats"; + public static final String HISTOGRAM = "histogram"; + public static final String COUNT = "count"; + public static final String SUM = "sum"; + public static final String AVG = "avg"; + public static final String MIN = "min"; + public static final String MAX = "max"; + +} diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index 8a2de15a9..5638c6082 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -98,4 +98,11 @@ public void test() throws Exception { } } + + @Test + public void testAggregationQueryParse(){ + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select sum(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + queryTranslator.translate(sql); + } } From 6d61f73c6e4e3a60a18bc94ae03775b37c6562d2 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 08:36:19 +0530 Subject: [PATCH 02/12] fix import error --- .../flipkart/foxtrot/core/querystore/actions/GroupAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index 83751d5f4..253d19f7c 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -45,7 +45,7 @@ import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; import com.flipkart.foxtrot.core.querystore.query.ElasticSearchQueryGenerator; import com.flipkart.foxtrot.core.table.TableMetadataManager; -import com.google.api.client.repackaged.com.google.common.base.Strings; +import com.google.common.base.Strings; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import lombok.SneakyThrows; From aed0132811f7645e0bba01b9acf8fc9ab2b3f533 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 08:40:14 +0530 Subject: [PATCH 03/12] fix test case --- .../java/com/flipkart/foxtrot/common/group/GroupRequest.java | 5 ----- .../com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java index 7c00a1bfe..1bd428534 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java @@ -15,16 +15,12 @@ */ package com.flipkart.foxtrot.common.group; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; import com.flipkart.foxtrot.common.ActionRequest; import com.flipkart.foxtrot.common.ActionRequestVisitor; import com.flipkart.foxtrot.common.Opcodes; import com.flipkart.foxtrot.common.enums.CountPrecision; import com.flipkart.foxtrot.common.query.Filter; import com.flipkart.foxtrot.common.stats.Stat; -import com.flipkart.foxtrot.common.util.CollectionUtils; -import com.google.common.collect.Sets; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -32,7 +28,6 @@ import javax.validation.constraints.NotNull; import java.util.List; -import java.util.Objects; import java.util.Set; /** diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java index 47bee0453..b7d284d9c 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java @@ -39,7 +39,7 @@ public void testCardinalityEmailBuild() throws JsonProcessingException { Assert.assertEquals("Blocked query as it might have screwed up the cluster", email.getSubject()); Assert.assertEquals( "Blocked Query: {\"opcode\":\"group\",\"filters\":[],\"bypassCache\":false,\"table\":\"test-table\"," + - "\"uniqueCountOn\":null,\"nesting\":[\"os\",\"deviceId\"],\"precision\":null}\n" + + "\"uniqueCountOn\":null,\"aggregationField\":null,\"stats\":null,\"nesting\":[\"os\",\"deviceId\"],\"precision\":null}\n" + "Suspect field: deviceId\n" + "Probability of screwing up the cluster: 0.75", email.getContent()); From c6d67158d86354970d9f8957be7374eb16ed056e Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 09:13:24 +0530 Subject: [PATCH 04/12] fix request cache key for group req --- .../foxtrot/core/querystore/actions/GroupAction.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index 253d19f7c..e9f48db99 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -108,6 +108,11 @@ public String getRequestCacheKey() { } } + if (null != query.getUniqueCountOn()) { + filterHashKey += 31 * query.getUniqueCountOn() + .hashCode(); + } + if (null != query.getAggregationField()) { filterHashKey += 31 * query.getAggregationField() .hashCode(); From c4bbb3eb03d304a140adc73909315e927be2d3c5 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 09:26:53 +0530 Subject: [PATCH 05/12] treat sum, avg, min, max fql functions as stats query --- .../main/java/com/flipkart/foxtrot/sql/QueryTranslator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index bea9bc69d..e7fb58858 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -206,7 +206,10 @@ public FqlQuery translate(String sql) { case STATSTREND: request = createStatsTrendActionRequest(); break; - + case SUM: + case AVG: + case MIN: + case MAX: case STATS: request = createStatsActionRequest(); break; From abf7bac776cb6c87c30ace229aa7f61457a4121c Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 10:05:44 +0530 Subject: [PATCH 06/12] add more test cases for fql parsing --- .../flipkart/foxtrot/sql/QueryTranslator.java | 6 +- .../com/flipkart/foxtrot/sql/ParseTest.java | 131 +++++++++++++++++- 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index e7fb58858..0ab17a80a 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -317,9 +317,11 @@ private ResultSort generateResultSort(List orderByElements) { private void setGroupAggregation(GroupRequest group) { if(calledAction instanceof CountRequest) { CountRequest countRequest = (CountRequest)this.calledAction; - boolean distinct = countRequest.isDistinct(); - if(distinct) { + if(countRequest.isDistinct()) { group.setUniqueCountOn(countRequest.getField()); + } else { + group.setStats(Sets.newHashSet(Stat.COUNT)); + group.setAggregationField(countRequest.getField()); } } else if (calledAction instanceof StatsRequest){ StatsRequest statsRequest = (StatsRequest) this.calledAction; diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index 5638c6082..dd94ba7e6 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -3,8 +3,10 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; +import com.flipkart.foxtrot.common.group.GroupRequest; import com.flipkart.foxtrot.common.query.Query; import com.flipkart.foxtrot.common.query.general.MissingFilter; +import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.sql.query.FqlActionQuery; import com.google.common.collect.ImmutableList; import org.junit.Assert; @@ -100,9 +102,134 @@ public void test() throws Exception { } @Test - public void testAggregationQueryParse(){ + public void testGroupAggregationSumQueryParsing() { QueryTranslator queryTranslator = new QueryTranslator(); String sql = "select sum(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; - queryTranslator.translate(sql); + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); + Assert.assertEquals(1, groupRequest.getStats() + .size()); + Assert.assertTrue(groupRequest.getStats() + .contains(Stat.SUM)); + + } + + @Test + public void testGroupAggregationCountDistinctQueryParsing() { + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select count(distinct eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertEquals("eventData.amount", groupRequest.getUniqueCountOn()); + Assert.assertNull(groupRequest.getStats()); + + } + + @Test + public void testGroupAggregationCountQueryParsing() { + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select count(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertEquals("eventData.amount", groupRequest.getUniqueCountOn()); + Assert.assertEquals(1, groupRequest.getStats() + .size()); + Assert.assertNull(groupRequest.getUniqueCountOn()); + Assert.assertTrue(groupRequest.getStats() + .contains(Stat.COUNT)); + } + + @Test + public void testGroupAggregationAvgQueryParsing() { + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select avg(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); + Assert.assertEquals(1, groupRequest.getStats() + .size()); + Assert.assertNull(groupRequest.getUniqueCountOn()); + Assert.assertTrue(groupRequest.getStats() + .contains(Stat.AVG)); + + } + + @Test + public void testGroupAggregationMinQueryParsing() { + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select min(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); + Assert.assertEquals(1, groupRequest.getStats() + .size()); + Assert.assertNull(groupRequest.getUniqueCountOn()); + Assert.assertTrue(groupRequest.getStats() + .contains(Stat.MIN)); + + } + + @Test + public void testGroupAggregationMaxQueryParsing() { + QueryTranslator queryTranslator = new QueryTranslator(); + String sql = "select max(eventData.amount) from europa where eventType = 'AWESOME_EVENT' group by date.hourOfDay"; + FqlQuery fqlQuery = queryTranslator.translate(sql); + Assert.assertTrue(fqlQuery instanceof FqlActionQuery); + FqlActionQuery actionQuery = (FqlActionQuery) fqlQuery; + Assert.assertTrue(actionQuery.getActionRequest() instanceof GroupRequest); + GroupRequest groupRequest = (GroupRequest) actionQuery.getActionRequest(); + + Assert.assertEquals(1, groupRequest.getNesting() + .size()); + Assert.assertTrue(groupRequest.getNesting() + .contains("date.hourOfDay")); + Assert.assertNull(groupRequest.getUniqueCountOn()); + Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); + Assert.assertEquals(1, groupRequest.getStats() + .size()); + Assert.assertTrue(groupRequest.getStats() + .contains(Stat.MAX)); + } } From a454c525cda7b531655288c02d13f2c1c80c1e27 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 10:12:10 +0530 Subject: [PATCH 07/12] add more test cases for fql parsing --- .../src/test/java/com/flipkart/foxtrot/sql/ParseTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index dd94ba7e6..3cecf67e3 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -156,7 +156,7 @@ public void testGroupAggregationCountQueryParsing() { .size()); Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); - Assert.assertEquals("eventData.amount", groupRequest.getUniqueCountOn()); + Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); Assert.assertEquals(1, groupRequest.getStats() .size()); Assert.assertNull(groupRequest.getUniqueCountOn()); From c3b5bc679974a539038a50d8aad31c428502b6dd Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 10:26:48 +0530 Subject: [PATCH 08/12] add test case for count group aggregation --- .../querystore/actions/GroupActionTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java index 1f2c58090..ca890ab9b 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java @@ -427,6 +427,44 @@ public void testGroupActionSumAggregation(){ .get("2")).get("sum")); } + @Test + @SuppressWarnings("unchecked") + public void testGroupActionCountAggregation(){ + GroupRequest groupRequest = new GroupRequest(); + groupRequest.setTable(TestUtils.TEST_TABLE_NAME); + groupRequest.setNesting(Arrays.asList("os", "version")); + groupRequest.setAggregationField("battery"); + groupRequest.setStats(Sets.newHashSet( Stat.COUNT)); + + Map response = Maps.newHashMap(); + response.put("android",new HashMap() {{ + put("1", ImmutableMap.of("count",2 )); + put("2", ImmutableMap.of("count", 3)); + put("3", ImmutableMap.of("count",2)); + }}); + response.put("ios",new HashMap() {{ + put("1", ImmutableMap.of("count",1)); + put("2", ImmutableMap.of("count", 3)); + }}); + + GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); + assertEquals(((Map) ((Map) response.get("android")).get("1")).get("count"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("1")).get("count")); + assertEquals(((Map) ((Map) response.get("android")).get("2")).get("count"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("2")).get("count")); + assertEquals(((Map) ((Map) response.get("android")).get("3")).get("count"), + ((Map) ((Map) actualResult.getResult().get("android")) + .get("3")).get("count")); + assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("count"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("1")).get("count")); + assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("count"), + ((Map) ((Map) actualResult.getResult().get("ios")) + .get("2")).get("count")); + } + @Test @SuppressWarnings("unchecked") public void testGroupActionCountAndSumAggregation(){ From 000333934d92c5dd107c842cc3d4f2d810ef5223 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 23 Jun 2020 10:34:14 +0530 Subject: [PATCH 09/12] change count to long in test case --- .../core/querystore/actions/GroupActionTest.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java index ca890ab9b..d1d8f3f11 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java @@ -438,13 +438,13 @@ public void testGroupActionCountAggregation(){ Map response = Maps.newHashMap(); response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("count",2 )); - put("2", ImmutableMap.of("count", 3)); - put("3", ImmutableMap.of("count",2)); + put("1", ImmutableMap.of("count",2L )); + put("2", ImmutableMap.of("count", 3L)); + put("3", ImmutableMap.of("count",2L)); }}); response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("count",1)); - put("2", ImmutableMap.of("count", 3)); + put("1", ImmutableMap.of("count",1L)); + put("2", ImmutableMap.of("count", 3L)); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); From 6cc20b537d2d42c8c164eb00331d57e12d3ee38a Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Tue, 4 Aug 2020 12:51:07 +0530 Subject: [PATCH 10/12] change stats header for group response in flattener --- .../core/querystore/actions/Utils.java | 16 ++--- .../sql/responseprocessors/Flattener.java | 61 ++++++++++++++++++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index ce5057284..c16f711d7 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -43,14 +43,14 @@ public class Utils { private static final double[] DEFAULT_PERCENTILES = {1d, 5d, 25, 50d, 75d, 95d, 99d}; private static final double DEFAULT_COMPRESSION = 100.0; private static final int PRECISION_THRESHOLD = 500; - private static final String COUNT = "count"; - private static final String AVG = "avg"; - private static final String SUM = "sum"; - private static final String MIN = "min"; - private static final String MAX = "max"; - private static final String SUM_OF_SQUARES = "sum_of_squares"; - private static final String VARIANCE = "variance"; - private static final String STD_DEVIATION = "std_deviation"; + public static final String COUNT = "count"; + public static final String AVG = "avg"; + public static final String SUM = "sum"; + public static final String MIN = "min"; + public static final String MAX = "max"; + public static final String SUM_OF_SQUARES = "sum_of_squares"; + public static final String VARIANCE = "variance"; + public static final String STD_DEVIATION = "std_deviation"; private static final EnumSet NUMERIC_FIELD_TYPES = EnumSet.of(FieldType.INTEGER, FieldType.LONG, FieldType.FLOAT, FieldType.DOUBLE); diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java index b5d18ae5b..77c8eda27 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java @@ -13,10 +13,12 @@ import com.flipkart.foxtrot.common.query.MultiQueryResponse; import com.flipkart.foxtrot.common.query.MultiTimeQueryResponse; import com.flipkart.foxtrot.common.query.QueryResponse; +import com.flipkart.foxtrot.common.stats.Stat.StatVisitor; import com.flipkart.foxtrot.common.stats.StatsResponse; import com.flipkart.foxtrot.common.stats.StatsTrendResponse; import com.flipkart.foxtrot.common.trend.TrendResponse; import com.flipkart.foxtrot.core.exception.FqlParsingException; +import com.flipkart.foxtrot.core.querystore.actions.Utils; import com.flipkart.foxtrot.sql.responseprocessors.model.FieldHeader; import com.flipkart.foxtrot.sql.responseprocessors.model.FlatRepresentation; import com.flipkart.foxtrot.sql.responseprocessors.model.MetaData; @@ -49,6 +51,9 @@ public void visit(GroupResponse groupResponse) { Map fieldNames = Maps.newTreeMap(); Map dataFields = generateFieldMappings(null, objectMapper.valueToTree(groupResponse.getResult()), separator); GroupRequest groupRequest = (GroupRequest)request; + + String statsHeader = getStatsHeader(groupRequest); + List> rows = Lists.newArrayList(); for(Map.Entry groupData : dataFields.entrySet()) { String[] values = groupData.getKey() @@ -64,19 +69,69 @@ public void visit(GroupResponse groupResponse) { } fieldNames.put(fieldName, lengthMax(fieldNames.get(fieldName), values[i])); } - row.put(COUNT, groupData.getValue() + row.put(statsHeader, groupData.getValue() .getData()); rows.add(row); } - fieldNames.put(COUNT, 10); + fieldNames.put(statsHeader, 10); List headers = Lists.newArrayList(); for(String fieldName : groupRequest.getNesting()) { headers.add(new FieldHeader(fieldName, fieldNames.get(fieldName))); } - headers.add(new FieldHeader(COUNT, 10)); + headers.add(new FieldHeader(statsHeader, 10)); flatRepresentation = new FlatRepresentation("group", headers, rows); } + private String getStatsHeader(GroupRequest groupRequest) { + String statsHeader = Utils.COUNT; + if(Objects.nonNull(groupRequest.getStats()) && groupRequest.getStats().stream().findFirst().isPresent()){ + statsHeader = groupRequest.getStats().stream() + .findFirst() + .get().visit(new StatVisitor() { + @Override + public String visitCount() { + return Utils.COUNT; + } + + @Override + public String visitMin() { + return Utils.MIN; + } + + @Override + public String visitMax() { + return Utils.MAX; + } + + @Override + public String visitAvg() { + return Utils.AVG; + } + + @Override + public String visitSum() { + return Utils.SUM; + } + + @Override + public String visitSumOfSquares() { + return Utils.SUM_OF_SQUARES; + } + + @Override + public String visitVariance() { + return Utils.VARIANCE; + } + + @Override + public String visitStdDeviation() { + return Utils.STD_DEVIATION; + } + }); + } + return statsHeader; + } + @Override public void visit(HistogramResponse histogramResponse) { List> rows = Lists.newArrayList(); From 2ee1f180789a9f5fac667b6338aa9fce65d91e6f Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Mon, 17 Aug 2020 19:24:16 +0530 Subject: [PATCH 11/12] change aggregation type in group request --- .../foxtrot/common/group/GroupRequest.java | 8 +- .../core/querystore/actions/GroupAction.java | 11 +- .../core/querystore/actions/Utils.java | 45 +++ .../foxtrot/core/alerts/EmailBuilderTest.java | 2 +- .../querystore/actions/GroupActionTest.java | 267 +++++++----------- .../flipkart/foxtrot/sql/QueryTranslator.java | 66 +++-- .../sql/responseprocessors/Flattener.java | 49 +--- .../com/flipkart/foxtrot/sql/ParseTest.java | 32 +-- 8 files changed, 225 insertions(+), 255 deletions(-) diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java index 1bd428534..57ab739b3 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java @@ -48,7 +48,7 @@ public class GroupRequest extends ActionRequest { private String aggregationField; - private Set stats; + private Stat aggregationType; @NotNull @NotEmpty @@ -61,13 +61,13 @@ public GroupRequest() { } public GroupRequest(List filters, String table, String uniqueCountOn, - String aggregationField, Set stats, + String aggregationField, Stat aggregationType, List nesting, CountPrecision precision) { super(Opcodes.GROUP, filters); this.table = table; this.uniqueCountOn = uniqueCountOn; this.aggregationField = aggregationField; - this.stats = stats; + this.aggregationType = aggregationType; this.nesting = nesting; this.precision = precision; } @@ -80,7 +80,7 @@ public T accept(ActionRequestVisitor visitor) { public String toString() { return new ToStringBuilder(this).appendSuper(super.toString()) .append("table", table) - .append("stats", stats) + .append("stats", aggregationType) .append("uniqueCountOn", uniqueCountOn) .append("aggregationField", aggregationField) .append("nesting", nesting) diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java index e9f48db99..a2a72b56b 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/GroupAction.java @@ -67,6 +67,7 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static com.flipkart.foxtrot.core.querystore.actions.Utils.statsString; import static com.flipkart.foxtrot.core.util.ElasticsearchQueryUtils.QUERY_SIZE; /** @@ -118,10 +119,8 @@ public String getRequestCacheKey() { .hashCode(); } - if(!CollectionUtils.isNullOrEmpty(query.getStats())){ - for(Stat stat : query.getStats()){ - filterHashKey += 31 * stat.hashCode(); - } + if(null != query.getAggregationType()){ + filterHashKey += 31 * query.getAggregationType().hashCode(); } for (int i = 0; i < query.getNesting() @@ -761,7 +760,7 @@ private Set buildSubAggregation(GroupRequest groupRequest) { final AbstractAggregationBuilder groupAggStats; if (isNumericField) { groupAggStats = Utils.buildStatsAggregation(groupRequest.getAggregationField(), - groupRequest.getStats()); + Collections.singleton(groupRequest.getAggregationType())); } else { groupAggStats = buildCardinalityAggregation(groupRequest.getAggregationField(), groupRequest); } @@ -793,7 +792,7 @@ private Map getMap(List fields, Aggregations aggregation else if (!Strings.isNullOrEmpty(getParameter().getAggregationField())) { String metricKey = Utils.getExtendedStatsAggregationKey(getParameter().getAggregationField()); levelCount.put(String.valueOf(bucket.getKey()), Utils.toStats( - bucket.getAggregations().getAsMap().get(metricKey))); + bucket.getAggregations().get(metricKey)).get(statsString(getParameter().getAggregationType()))); } else { levelCount.put(String.valueOf(bucket.getKey()), bucket.getDocCount()); diff --git a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java index c16f711d7..5aeaa835f 100644 --- a/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java +++ b/foxtrot-core/src/main/java/com/flipkart/foxtrot/core/querystore/actions/Utils.java @@ -7,6 +7,7 @@ import com.flipkart.foxtrot.common.TableFieldMapping; import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.stats.Stat; +import com.flipkart.foxtrot.common.stats.Stat.StatVisitor; import com.flipkart.foxtrot.common.util.CollectionUtils; import com.flipkart.foxtrot.core.exception.FoxtrotExceptions; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchUtils; @@ -353,4 +354,48 @@ public static boolean isNumericField(TableMetadataManager tableMetadataManager, return null != fieldMetadata && NUMERIC_FIELD_TYPES.contains(fieldMetadata.getType()); } + public static String statsString(Stat aggregationType) { + return aggregationType + .visit(new StatVisitor() { + @Override + public String visitCount() { + return Utils.COUNT; + } + + @Override + public String visitMin() { + return Utils.MIN; + } + + @Override + public String visitMax() { + return Utils.MAX; + } + + @Override + public String visitAvg() { + return Utils.AVG; + } + + @Override + public String visitSum() { + return Utils.SUM; + } + + @Override + public String visitSumOfSquares() { + return Utils.SUM_OF_SQUARES; + } + + @Override + public String visitVariance() { + return Utils.VARIANCE; + } + + @Override + public String visitStdDeviation() { + return Utils.STD_DEVIATION; + } + }); + } } diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java index b7d284d9c..fc9029742 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/alerts/EmailBuilderTest.java @@ -39,7 +39,7 @@ public void testCardinalityEmailBuild() throws JsonProcessingException { Assert.assertEquals("Blocked query as it might have screwed up the cluster", email.getSubject()); Assert.assertEquals( "Blocked Query: {\"opcode\":\"group\",\"filters\":[],\"bypassCache\":false,\"table\":\"test-table\"," + - "\"uniqueCountOn\":null,\"aggregationField\":null,\"stats\":null,\"nesting\":[\"os\",\"deviceId\"],\"precision\":null}\n" + + "\"uniqueCountOn\":null,\"aggregationField\":null,\"aggregationType\":null,\"nesting\":[\"os\",\"deviceId\"],\"precision\":null}\n" + "Suspect field: deviceId\n" + "Probability of screwing up the cluster: 0.75", email.getContent()); diff --git a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java index d1d8f3f11..516b3e11c 100644 --- a/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java +++ b/foxtrot-core/src/test/java/com/flipkart/foxtrot/core/querystore/actions/GroupActionTest.java @@ -15,6 +15,10 @@ */ package com.flipkart.foxtrot.core.querystore.actions; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doReturn; + import com.fasterxml.jackson.core.JsonProcessingException; import com.flipkart.foxtrot.common.Document; import com.flipkart.foxtrot.common.group.GroupRequest; @@ -27,19 +31,16 @@ import com.flipkart.foxtrot.core.exception.ErrorCode; import com.flipkart.foxtrot.core.exception.FoxtrotException; import com.flipkart.foxtrot.core.querystore.impl.ElasticsearchQueryStore; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; -import java.util.*; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.doReturn; - /** * Created by rishabh.goyal on 28/04/14. */ @@ -56,14 +57,14 @@ public static void setUp() throws Exception { .execute() .actionGet(); getTableMetadataManager().getFieldMappings(TestUtils.TEST_TABLE_NAME, true, true); - ((ElasticsearchQueryStore)getQueryStore()).getCardinalityConfig() + ((ElasticsearchQueryStore) getQueryStore()).getCardinalityConfig() .setMaxCardinality(15000); getTableMetadataManager().updateEstimationData(TestUtils.TEST_TABLE_NAME, 1397658117000L); } @Ignore @Test - public void testGroupActionSingleQueryException() throws FoxtrotException, JsonProcessingException { + public void testGroupActionSingleQueryException() throws FoxtrotException { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Collections.singletonList("os")); @@ -123,7 +124,8 @@ public void testGroupActionSingleFieldSpecialCharactersNoFilter() throws Foxtrot } @Test - public void testGroupActionSingleFieldHavingSpecialCharactersWithFilter() throws FoxtrotException, JsonProcessingException { + public void testGroupActionSingleFieldHavingSpecialCharactersWithFilter() + throws FoxtrotException, JsonProcessingException { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); @@ -240,7 +242,6 @@ public void testGroupActionMultipleFieldsNoFilter() throws FoxtrotException, Jso put("iphone", iPhoneResponse); }}); - GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(response, actualResult.getResult()); } @@ -284,230 +285,174 @@ public void testGroupActionMultipleFieldsWithFilter() throws FoxtrotException, J @Test @SuppressWarnings("unchecked") - public void testGroupActionDistinctCountAggregation(){ + public void testGroupActionDistinctCountAggregation() { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Arrays.asList("os", "version")); groupRequest.setUniqueCountOn("device"); Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ + response.put("android", new HashMap() {{ put("1", 1L); put("2", 2L); put("3", 2L); }}); - response.put("ios",new HashMap() {{ + response.put("ios", new HashMap() {{ put("1", 1L); put("2", 2L); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); assertEquals(((Map) response.get("android")).get("1"), - ((Map) actualResult.getResult().get("android")).get("1")); + ((Map) actualResult.getResult() + .get("android")).get("1")); assertEquals(((Map) response.get("android")).get("2"), - ((Map) actualResult.getResult().get("android")).get("2")); + ((Map) actualResult.getResult() + .get("android")).get("2")); assertEquals(((Map) response.get("android")).get("3"), - ((Map) actualResult.getResult().get("android")).get("3")); + ((Map) actualResult.getResult() + .get("android")).get("3")); assertEquals(((Map) response.get("ios")).get("1"), - ((Map) actualResult.getResult().get("ios")).get("1")); + ((Map) actualResult.getResult() + .get("ios")).get("1")); assertEquals(((Map) response.get("ios")).get("2"), - ((Map) actualResult.getResult().get("ios")).get("2")); + ((Map) actualResult.getResult() + .get("ios")).get("2")); } @Test @SuppressWarnings("unchecked") - public void testGroupActionMaxAggregation(){ + public void testGroupActionMaxAggregation() { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Arrays.asList("os", "version")); groupRequest.setAggregationField("battery"); - groupRequest.setStats(Sets.newHashSet(Stat.MAX)); + groupRequest.setAggregationType(Stat.MAX); Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("max",48.0)); - put("2", ImmutableMap.of("max",99.0)); - put("3", ImmutableMap.of("max",87.0)); + response.put("android", new HashMap() {{ + put("1", 48.0); + put("2", 99.0); + put("3",87.0); }}); - response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("max",24.0)); - put("2", ImmutableMap.of("max",56.0)); + response.put("ios", new HashMap() {{ + put("1", 24.0); + put("2",56.0); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("max"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("max")); - assertEquals(((Map) ((Map) response.get("android")).get("2")).get("max"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("2")).get("max")); - assertEquals(((Map) ((Map) response.get("android")).get("3")).get("max"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("3")).get("max")); - assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("max"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("1")).get("max")); - assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("max"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("2")).get("max")); + assertEquals(((Map) response.get("android")).get("1"), ((Map) actualResult.getResult() + .get("android")).get("1")); + assertEquals(((Map) response.get("android")).get("2"), ((Map) actualResult.getResult() + .get("android")).get("2")); + assertEquals(((Map) response.get("android")).get("3"), ((Map) actualResult.getResult() + .get("android")).get("3")); + assertEquals(((Map) response.get("ios")).get("1"), ((Map) actualResult.getResult() + .get("ios")).get("1")); + assertEquals(((Map) response.get("ios")).get("2"), ((Map) actualResult.getResult() + .get("ios")).get("2")); } @Test @SuppressWarnings("unchecked") - public void testGroupActionAvgAggregation(){ + public void testGroupActionAvgAggregation() { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Arrays.asList("os", "version")); groupRequest.setAggregationField("battery"); - groupRequest.setStats(Sets.newHashSet(Stat.AVG)); + groupRequest.setAggregationType(Stat.AVG); Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("avg",36.0)); - put("2", ImmutableMap.of("avg",84.33333333333333)); - put("3", ImmutableMap.of("avg",80.5)); + response.put("android", new HashMap() {{ + put("1", 36.0); + put("2", 84.33333333333333); + put("3", 80.5); }}); - response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("avg",24.0)); - put("2", ImmutableMap.of("avg",45.0)); + response.put("ios", new HashMap() {{ + put("1", 24.0); + put("2", 45.0); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("avg")); - assertEquals(((Map) ((Map) response.get("android")).get("2")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("2")).get("avg")); - assertEquals(((Map) ((Map) response.get("android")).get("3")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("3")).get("avg")); - assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("1")).get("avg")); - assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("2")).get("avg")); + assertEquals(((Map) response.get("android")).get("1"), + ((Map) actualResult.getResult() + .get("android")).get("1")); + assertEquals(((Map) response.get("android")).get("2"), + ((Map) actualResult.getResult() + .get("android")).get("2")); + assertEquals(((Map) response.get("android")).get("3"), + ((Map) actualResult.getResult() + .get("android")).get("3")); + assertEquals(((Map) response.get("ios")).get("1"), + ((Map) actualResult.getResult() + .get("ios")).get("1")); + assertEquals(((Map) response.get("ios")).get("2"), ((Map) actualResult.getResult() + .get("ios")).get("2")); } @Test @SuppressWarnings("unchecked") - public void testGroupActionSumAggregation(){ + public void testGroupActionSumAggregation() { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Arrays.asList("os", "version")); groupRequest.setAggregationField("battery"); - groupRequest.setStats(Sets.newHashSet(Stat.SUM)); + groupRequest.setAggregationType(Stat.SUM); Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("sum",72.0)); - put("2", ImmutableMap.of("sum",253.0)); - put("3", ImmutableMap.of("sum",161.0)); + response.put("android", new HashMap() {{ + put("1",72.0); + put("2", 253.0); + put("3",161.0); }}); - response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("sum",24.0)); - put("2", ImmutableMap.of("sum",135.0)); + response.put("ios", new HashMap() {{ + put("1", 24.0); + put("2", 135.0); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("sum")); - assertEquals(((Map) ((Map) response.get("android")).get("2")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("2")).get("sum")); - assertEquals(((Map) ((Map) response.get("android")).get("3")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("3")).get("sum")); - assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("1")).get("sum")); - assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("2")).get("sum")); + assertEquals(((Map) response.get("android")).get("1"), ((Map) actualResult.getResult() + .get("android")).get("1")); + assertEquals(((Map) response.get("android")).get("2"), ((Map) actualResult.getResult() + .get("android")).get("2")); + assertEquals(((Map) response.get("android")).get("3"), ((Map) actualResult.getResult() + .get("android")).get("3")); + assertEquals(((Map) response.get("ios")).get("1"), ((Map) actualResult.getResult() + .get("ios")).get("1")); + assertEquals(((Map) response.get("ios")).get("2"), ((Map) actualResult.getResult() + .get("ios")).get("2")); } @Test @SuppressWarnings("unchecked") - public void testGroupActionCountAggregation(){ + public void testGroupActionCountAggregation() { GroupRequest groupRequest = new GroupRequest(); groupRequest.setTable(TestUtils.TEST_TABLE_NAME); groupRequest.setNesting(Arrays.asList("os", "version")); groupRequest.setAggregationField("battery"); - groupRequest.setStats(Sets.newHashSet( Stat.COUNT)); + groupRequest.setAggregationType(Stat.COUNT); Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("count",2L )); - put("2", ImmutableMap.of("count", 3L)); - put("3", ImmutableMap.of("count",2L)); + response.put("android", new HashMap() {{ + put("1", 2L); + put("2",3L); + put("3", 2L); }}); - response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("count",1L)); - put("2", ImmutableMap.of("count", 3L)); + response.put("ios", new HashMap() {{ + put("1", 1L); + put("2", 3L); }}); GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("count"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("count")); - assertEquals(((Map) ((Map) response.get("android")).get("2")).get("count"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("2")).get("count")); - assertEquals(((Map) ((Map) response.get("android")).get("3")).get("count"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("3")).get("count")); - assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("count"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("1")).get("count")); - assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("count"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("2")).get("count")); + assertEquals(((Map) response.get("android")).get("1"), ((Map) actualResult.getResult() + .get("android")).get("1")); + assertEquals(((Map) response.get("android")).get("2"), ((Map) actualResult.getResult() + .get("android")).get("2")); + assertEquals(((Map) response.get("android")).get("3"), ((Map) actualResult.getResult() + .get("android")).get("3")); + assertEquals(((Map) response.get("ios")).get("1"), ((Map) actualResult.getResult() + .get("ios")).get("1")); + assertEquals(((Map) response.get("ios")).get("2"), ((Map) actualResult.getResult() + .get("ios")).get("2")); } - @Test - @SuppressWarnings("unchecked") - public void testGroupActionCountAndSumAggregation(){ - GroupRequest groupRequest = new GroupRequest(); - groupRequest.setTable(TestUtils.TEST_TABLE_NAME); - groupRequest.setNesting(Arrays.asList("os", "version")); - groupRequest.setAggregationField("battery"); - groupRequest.setStats(Sets.newHashSet(Stat.SUM, Stat.COUNT)); - - Map response = Maps.newHashMap(); - response.put("android",new HashMap() {{ - put("1", ImmutableMap.of("avg",36.0, "min",24.0, - "max",48.0, "count",2, "sum",72.0)); - put("2", ImmutableMap.of("avg",84.33333333333333, "min",76.0, - "max",99.0, "count", 3, "sum",253.0)); - put("3", ImmutableMap.of("avg",80.5, "min",74.0, - "max",87.0, "count",2, "sum",161.0)); - }}); - response.put("ios",new HashMap() {{ - put("1", ImmutableMap.of("avg",24.0, "min",24.0, - "max",24.0, "count",1, "sum",24.0)); - put("2", ImmutableMap.of("avg",45.0, "min", 35.0, - "max",56.0, "count", 3, "sum",135.0)); - }}); - - GroupResponse actualResult = (GroupResponse) getQueryExecutor().execute(groupRequest); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("avg")); - assertEquals(((Map) ((Map) response.get("android")).get("1")).get("sum"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("1")).get("sum")); - assertEquals(((Map) ((Map) response.get("android")).get("2")).get("min"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("2")).get("min")); - assertEquals(((Map) ((Map) response.get("android")).get("3")).get("max"), - ((Map) ((Map) actualResult.getResult().get("android")) - .get("3")).get("max")); - assertEquals(((Map) ((Map) response.get("ios")).get("1")).get("max"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("1")).get("max")); - assertEquals(((Map) ((Map) response.get("ios")).get("2")).get("avg"), - ((Map) ((Map) actualResult.getResult().get("ios")) - .get("2")).get("avg")); - } } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java index 0ab17a80a..6b46f909d 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/QueryTranslator.java @@ -11,8 +11,17 @@ import com.flipkart.foxtrot.common.query.Query; import com.flipkart.foxtrot.common.query.ResultSort; import com.flipkart.foxtrot.common.query.datetime.LastFilter; -import com.flipkart.foxtrot.common.query.general.*; -import com.flipkart.foxtrot.common.query.numeric.*; +import com.flipkart.foxtrot.common.query.general.EqualsFilter; +import com.flipkart.foxtrot.common.query.general.ExistsFilter; +import com.flipkart.foxtrot.common.query.general.InFilter; +import com.flipkart.foxtrot.common.query.general.MissingFilter; +import com.flipkart.foxtrot.common.query.general.NotEqualsFilter; +import com.flipkart.foxtrot.common.query.general.NotInFilter; +import com.flipkart.foxtrot.common.query.numeric.BetweenFilter; +import com.flipkart.foxtrot.common.query.numeric.GreaterEqualFilter; +import com.flipkart.foxtrot.common.query.numeric.GreaterThanFilter; +import com.flipkart.foxtrot.common.query.numeric.LessEqualFilter; +import com.flipkart.foxtrot.common.query.numeric.LessThanFilter; import com.flipkart.foxtrot.common.query.string.ContainsFilter; import com.flipkart.foxtrot.common.stats.AnalyticsRequestFlags; import com.flipkart.foxtrot.common.stats.Stat; @@ -32,23 +41,44 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import io.dropwizard.util.Duration; +import java.io.StringReader; +import java.util.List; +import java.util.Set; import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.*; +import net.sf.jsqlparser.expression.DateValue; +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Function; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.TimeValue; import net.sf.jsqlparser.expression.operators.conditional.AndExpression; -import net.sf.jsqlparser.expression.operators.relational.*; +import net.sf.jsqlparser.expression.operators.relational.Between; +import net.sf.jsqlparser.expression.operators.relational.EqualsTo; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.GreaterThan; +import net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.expression.operators.relational.LikeExpression; +import net.sf.jsqlparser.expression.operators.relational.MinorThan; +import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals; +import net.sf.jsqlparser.expression.operators.relational.NotEqualsTo; import net.sf.jsqlparser.parser.CCJSqlParserManager; import net.sf.jsqlparser.schema.Column; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.select.*; +import net.sf.jsqlparser.statement.select.OrderByElement; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; +import net.sf.jsqlparser.statement.select.SelectItem; import org.elasticsearch.common.Strings; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.StringReader; -import java.util.List; -import java.util.Set; - public class QueryTranslator extends SqlElementVisitor { private static final Logger logger = LoggerFactory.getLogger(QueryTranslator.class.getSimpleName()); private static final MetaStatementMatcher metastatementMatcher = new MetaStatementMatcher(); @@ -84,15 +114,15 @@ public void visit(PlainSelect plainSelect) { plainSelect.getFromItem() .accept(this); //Populate table name List groupByItems = plainSelect.getGroupByColumnReferences(); - if(null != groupByItems) { - for(Expression groupByItem : CollectionUtils.nullSafeList(groupByItems)) { - queryType = FqlQueryType.GROUP; - if(groupByItem instanceof Column) { - Column column = (Column)groupByItem; - groupBycolumnsList.add(column.getFullyQualifiedName()); + if (null != groupByItems) { + for (Expression groupByItem : CollectionUtils.nullSafeList(groupByItems)) { + queryType = FqlQueryType.GROUP; + if (groupByItem instanceof Column) { + Column column = (Column) groupByItem; + groupBycolumnsList.add(column.getFullyQualifiedName()); + } } } - } if(FqlQueryType.SELECT == queryType) { List orderByElements = plainSelect.getOrderByElements(); resultSort = generateResultSort(orderByElements); @@ -320,12 +350,12 @@ private void setGroupAggregation(GroupRequest group) { if(countRequest.isDistinct()) { group.setUniqueCountOn(countRequest.getField()); } else { - group.setStats(Sets.newHashSet(Stat.COUNT)); + group.setAggregationType(Stat.COUNT); group.setAggregationField(countRequest.getField()); } } else if (calledAction instanceof StatsRequest){ StatsRequest statsRequest = (StatsRequest) this.calledAction; - group.setStats(statsRequest.getStats()); + group.setAggregationType(statsRequest.getStats().stream().findFirst().orElse(Stat.COUNT)); group.setAggregationField(statsRequest.getField()); } } diff --git a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java index 77c8eda27..08b90cee3 100644 --- a/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java +++ b/foxtrot-sql/src/main/java/com/flipkart/foxtrot/sql/responseprocessors/Flattener.java @@ -13,6 +13,7 @@ import com.flipkart.foxtrot.common.query.MultiQueryResponse; import com.flipkart.foxtrot.common.query.MultiTimeQueryResponse; import com.flipkart.foxtrot.common.query.QueryResponse; +import com.flipkart.foxtrot.common.stats.Stat; import com.flipkart.foxtrot.common.stats.Stat.StatVisitor; import com.flipkart.foxtrot.common.stats.StatsResponse; import com.flipkart.foxtrot.common.stats.StatsTrendResponse; @@ -29,6 +30,7 @@ import java.util.stream.Collectors; import static com.flipkart.foxtrot.common.Opcodes.COUNT; +import static com.flipkart.foxtrot.core.querystore.actions.Utils.statsString; import static com.flipkart.foxtrot.sql.responseprocessors.FlatteningUtils.generateFieldMappings; import static com.flipkart.foxtrot.sql.responseprocessors.FlatteningUtils.genericParse; @@ -84,54 +86,13 @@ public void visit(GroupResponse groupResponse) { private String getStatsHeader(GroupRequest groupRequest) { String statsHeader = Utils.COUNT; - if(Objects.nonNull(groupRequest.getStats()) && groupRequest.getStats().stream().findFirst().isPresent()){ - statsHeader = groupRequest.getStats().stream() - .findFirst() - .get().visit(new StatVisitor() { - @Override - public String visitCount() { - return Utils.COUNT; - } - - @Override - public String visitMin() { - return Utils.MIN; - } - - @Override - public String visitMax() { - return Utils.MAX; - } - - @Override - public String visitAvg() { - return Utils.AVG; - } - - @Override - public String visitSum() { - return Utils.SUM; - } - - @Override - public String visitSumOfSquares() { - return Utils.SUM_OF_SQUARES; - } - - @Override - public String visitVariance() { - return Utils.VARIANCE; - } - - @Override - public String visitStdDeviation() { - return Utils.STD_DEVIATION; - } - }); + if(Objects.nonNull(groupRequest.getAggregationType())){ + statsHeader = statsString(groupRequest.getAggregationType()); } return statsHeader; } + @Override public void visit(HistogramResponse histogramResponse) { List> rows = Lists.newArrayList(); diff --git a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java index 3cecf67e3..31bafbc5d 100644 --- a/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java +++ b/foxtrot-sql/src/test/java/com/flipkart/foxtrot/sql/ParseTest.java @@ -116,10 +116,8 @@ public void testGroupAggregationSumQueryParsing() { Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); - Assert.assertEquals(1, groupRequest.getStats() - .size()); - Assert.assertTrue(groupRequest.getStats() - .contains(Stat.SUM)); + Assert.assertNotNull(groupRequest.getAggregationType()); + Assert.assertEquals(groupRequest.getAggregationType(),Stat.SUM); } @@ -138,7 +136,7 @@ public void testGroupAggregationCountDistinctQueryParsing() { Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); Assert.assertEquals("eventData.amount", groupRequest.getUniqueCountOn()); - Assert.assertNull(groupRequest.getStats()); + Assert.assertNull(groupRequest.getAggregationType()); } @@ -157,11 +155,9 @@ public void testGroupAggregationCountQueryParsing() { Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); - Assert.assertEquals(1, groupRequest.getStats() - .size()); + Assert.assertNotNull(groupRequest.getAggregationType()); Assert.assertNull(groupRequest.getUniqueCountOn()); - Assert.assertTrue(groupRequest.getStats() - .contains(Stat.COUNT)); + Assert.assertEquals(groupRequest.getAggregationType(),Stat.COUNT); } @Test @@ -179,11 +175,10 @@ public void testGroupAggregationAvgQueryParsing() { Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); - Assert.assertEquals(1, groupRequest.getStats() - .size()); + Assert.assertNotNull(groupRequest.getAggregationType()); + Assert.assertNull(groupRequest.getUniqueCountOn()); - Assert.assertTrue(groupRequest.getStats() - .contains(Stat.AVG)); + Assert.assertEquals(groupRequest.getAggregationType(),Stat.AVG); } @@ -202,11 +197,9 @@ public void testGroupAggregationMinQueryParsing() { Assert.assertTrue(groupRequest.getNesting() .contains("date.hourOfDay")); Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); - Assert.assertEquals(1, groupRequest.getStats() - .size()); + Assert.assertNotNull(groupRequest.getAggregationType()); Assert.assertNull(groupRequest.getUniqueCountOn()); - Assert.assertTrue(groupRequest.getStats() - .contains(Stat.MIN)); + Assert.assertEquals(groupRequest.getAggregationType(),Stat.MIN); } @@ -226,10 +219,7 @@ public void testGroupAggregationMaxQueryParsing() { .contains("date.hourOfDay")); Assert.assertNull(groupRequest.getUniqueCountOn()); Assert.assertEquals("eventData.amount", groupRequest.getAggregationField()); - Assert.assertEquals(1, groupRequest.getStats() - .size()); - Assert.assertTrue(groupRequest.getStats() - .contains(Stat.MAX)); + Assert.assertEquals(groupRequest.getAggregationType(),Stat.MAX); } } From f8177499b8b44f07ae391d244d9578c3930d6ec1 Mon Sep 17 00:00:00 2001 From: Jitendra Dhawan Date: Wed, 19 Aug 2020 13:35:11 +0530 Subject: [PATCH 12/12] rename string representation --- .../java/com/flipkart/foxtrot/common/group/GroupRequest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java index 57ab739b3..fdd72c20a 100644 --- a/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java +++ b/foxtrot-common/src/main/java/com/flipkart/foxtrot/common/group/GroupRequest.java @@ -80,7 +80,7 @@ public T accept(ActionRequestVisitor visitor) { public String toString() { return new ToStringBuilder(this).appendSuper(super.toString()) .append("table", table) - .append("stats", aggregationType) + .append("aggregationType", aggregationType) .append("uniqueCountOn", uniqueCountOn) .append("aggregationField", aggregationField) .append("nesting", nesting)