From d245719b9ae4198010fd26aac1928770770015cf Mon Sep 17 00:00:00 2001 From: "zihe.liu" Date: Wed, 10 Jul 2024 14:31:14 +0800 Subject: [PATCH 1/2] s Signed-off-by: zihe.liu --- .../java/com/starrocks/catalog/Function.java | 12 ++ .../starrocks/http/rest/QueryDumpAction.java | 70 ++------- .../sql/optimizer/dump/QueryDumper.java | 100 +++++++++++++ .../sql/optimizer/function/MetaFunctions.java | 18 +++ .../rewrite/ScalarOperatorEvaluator.java | 5 +- .../rewrite/scalar/FoldConstantsRule.java | 14 +- .../starrocks/http/QueryDumpActionTest.java | 136 ++++++++++++++++++ .../starrocks/http/StarRocksHttpTestCase.java | 12 ++ .../sql/optimizer/QueryDumperTest.java | 127 ++++++++++++++++ .../sql/plan/ConstantExpressionTest.java | 66 +++++++++ test/sql/test_function/R/test_get_query_dump | 106 ++++++++++++++ test/sql/test_function/T/test_get_query_dump | 80 +++++++++++ 12 files changed, 685 insertions(+), 61 deletions(-) create mode 100644 fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumper.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/http/QueryDumpActionTest.java create mode 100644 fe/fe-core/src/test/java/com/starrocks/sql/optimizer/QueryDumperTest.java create mode 100644 test/sql/test_function/R/test_get_query_dump create mode 100644 test/sql/test_function/T/test_get_query_dump diff --git a/fe/fe-core/src/main/java/com/starrocks/catalog/Function.java b/fe/fe-core/src/main/java/com/starrocks/catalog/Function.java index 2eaa425a52bce..1ff1d636f6502 100644 --- a/fe/fe-core/src/main/java/com/starrocks/catalog/Function.java +++ b/fe/fe-core/src/main/java/com/starrocks/catalog/Function.java @@ -150,6 +150,9 @@ public enum CompareMode { private boolean isNullable = true; private Vector> defaultArgExprs; + + private boolean isMetaFunction = false; + // Only used for serialization protected Function() { } @@ -225,6 +228,7 @@ public Function(Function other) { isPolymorphic = other.isPolymorphic; couldApplyDictOptimize = other.couldApplyDictOptimize; isNullable = other.isNullable; + isMetaFunction = other.isMetaFunction; } public FunctionName getFunctionName() { @@ -341,6 +345,14 @@ public boolean isPolymorphic() { return isPolymorphic; } + public boolean isMetaFunction() { + return isMetaFunction; + } + + public void setMetaFunction(boolean metaFunction) { + isMetaFunction = metaFunction; + } + public long getFunctionId() { return functionId; } diff --git a/fe/fe-core/src/main/java/com/starrocks/http/rest/QueryDumpAction.java b/fe/fe-core/src/main/java/com/starrocks/http/rest/QueryDumpAction.java index 1e2f17d426e56..35b98318c6986 100644 --- a/fe/fe-core/src/main/java/com/starrocks/http/rest/QueryDumpAction.java +++ b/fe/fe-core/src/main/java/com/starrocks/http/rest/QueryDumpAction.java @@ -15,25 +15,15 @@ package com.starrocks.http.rest; import com.google.common.base.Strings; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.starrocks.catalog.Database; import com.starrocks.catalog.InternalCatalog; import com.starrocks.common.DdlException; +import com.starrocks.common.Pair; import com.starrocks.http.ActionController; import com.starrocks.http.BaseRequest; import com.starrocks.http.BaseResponse; import com.starrocks.http.IllegalArgException; -import com.starrocks.persist.gson.GsonUtils; import com.starrocks.qe.ConnectContext; -import com.starrocks.qe.StmtExecutor; -import com.starrocks.server.GlobalStateMgr; -import com.starrocks.sql.ast.StatementBase; -import com.starrocks.sql.optimizer.dump.DumpInfo; -import com.starrocks.sql.optimizer.dump.QueryDumpDeserializer; -import com.starrocks.sql.optimizer.dump.QueryDumpInfo; -import com.starrocks.sql.optimizer.dump.QueryDumpSerializer; -import com.starrocks.sql.parser.SqlParser; +import com.starrocks.sql.optimizer.dump.QueryDumper; import io.netty.handler.codec.http.HttpMethod; import io.netty.handler.codec.http.HttpResponseStatus; import org.apache.commons.lang3.StringUtils; @@ -50,25 +40,17 @@ public class QueryDumpAction extends RestBaseAction { private static final Logger LOG = LogManager.getLogger(QueryDumpAction.class); - private static final String DB = "db"; + public static final String URL = "/api/query_dump"; + private static final String DB = "db"; private static final String MOCK = "mock"; - private static final Gson GSON = new GsonBuilder() - .addSerializationExclusionStrategy(new GsonUtils.HiddenAnnotationExclusionStrategy()) - .addDeserializationExclusionStrategy(new GsonUtils.HiddenAnnotationExclusionStrategy()) - .enableComplexMapKeySerialization() - .disableHtmlEscaping() - .registerTypeAdapter(QueryDumpInfo.class, new QueryDumpSerializer()) - .registerTypeAdapter(QueryDumpInfo.class, new QueryDumpDeserializer()) - .create(); - public QueryDumpAction(ActionController controller) { super(controller); } public static void registerAction(ActionController controller) throws IllegalArgException { - controller.registerHandler(HttpMethod.POST, "/api/query_dump", new QueryDumpAction(controller)); + controller.registerHandler(HttpMethod.POST, URL, new QueryDumpAction(controller)); } @Override @@ -78,52 +60,24 @@ public void executeWithoutPassword(BaseRequest request, BaseResponse response) t boolean enableMock = request.getSingleParameter(MOCK) == null || "true".equalsIgnoreCase(StringUtils.trim(request.getSingleParameter(MOCK))); + String catalogName = ""; + String dbName = ""; if (!Strings.isNullOrEmpty(catalogDbName)) { String[] catalogDbNames = catalogDbName.split("\\."); - String catalogName = InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME; + catalogName = InternalCatalog.DEFAULT_INTERNAL_CATALOG_NAME; if (catalogDbNames.length == 2) { catalogName = catalogDbNames[0]; } - String dbName = catalogDbNames[catalogDbNames.length - 1]; - context.setCurrentCatalog(catalogName); - Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); - if (db == null) { - response.getContent().append("Database [" + dbName + "] does not exists"); - sendResult(request, response, HttpResponseStatus.NOT_FOUND); - return; - } - context.setDatabase(db.getFullName()); + dbName = catalogDbNames[catalogDbNames.length - 1]; } context.setIsHTTPQueryDump(true); String query = request.getContent(); - if (Strings.isNullOrEmpty(query)) { - response.getContent().append("not valid parameter"); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } - StatementBase parsedStmt; - try { - parsedStmt = SqlParser.parse(query, context.getSessionVariable()).get(0); - StmtExecutor executor = new StmtExecutor(context, parsedStmt); - executor.execute(); - } catch (Exception e) { - LOG.warn("execute query failed. ", e); - response.getContent().append("execute query failed. " + e.getMessage()); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - return; - } + Pair statusAndRes = QueryDumper.dumpQuery(catalogName, dbName, query, enableMock); - DumpInfo dumpInfo = context.getDumpInfo(); - if (dumpInfo != null) { - dumpInfo.setDesensitizedInfo(enableMock); - response.getContent().append(GSON.toJson(dumpInfo, QueryDumpInfo.class)); - sendResult(request, response); - } else { - response.getContent().append("not use cbo planner, try again."); - sendResult(request, response, HttpResponseStatus.BAD_REQUEST); - } + response.getContent().append(statusAndRes.second); + sendResult(request, response, statusAndRes.first); } } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumper.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumper.java new file mode 100644 index 0000000000000..e504ca27451d0 --- /dev/null +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/dump/QueryDumper.java @@ -0,0 +1,100 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer.dump; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.starrocks.catalog.Database; +import com.starrocks.common.Pair; +import com.starrocks.persist.gson.GsonUtils; +import com.starrocks.qe.ConnectContext; +import com.starrocks.qe.StmtExecutor; +import com.starrocks.server.GlobalStateMgr; +import com.starrocks.sql.ast.StatementBase; +import com.starrocks.sql.parser.SqlParser; +import io.netty.handler.codec.http.HttpResponseStatus; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class QueryDumper { + private static final Logger LOG = LogManager.getLogger(QueryDumper.class); + + private static final Gson GSON = new GsonBuilder() + .addSerializationExclusionStrategy(new GsonUtils.HiddenAnnotationExclusionStrategy()) + .addDeserializationExclusionStrategy(new GsonUtils.HiddenAnnotationExclusionStrategy()) + .enableComplexMapKeySerialization() + .disableHtmlEscaping() + .registerTypeAdapter(QueryDumpInfo.class, new QueryDumpSerializer()) + .registerTypeAdapter(QueryDumpInfo.class, new QueryDumpDeserializer()) + .create(); + + public static Pair dumpQuery(String catalogName, String dbName, String query, + boolean enableMock) { + ConnectContext context = ConnectContext.get(); + if (context == null) { + return Pair.create(HttpResponseStatus.BAD_REQUEST, + "There is no ConnectContext for this thread: " + Thread.currentThread().getName()); + } + + final String prevCatalog = context.getCurrentCatalog(); + final String prevDb = context.getDatabase(); + final boolean prevIsHTTPQueryDump = context.isHTTPQueryDump(); + + try { + if (StringUtils.isEmpty(query)) { + return Pair.create(HttpResponseStatus.BAD_REQUEST, "query is empty"); + } + + if (!StringUtils.isEmpty(catalogName)) { + context.setCurrentCatalog(catalogName); + } + + if (!StringUtils.isEmpty(dbName)) { + Database db = GlobalStateMgr.getCurrentState().getMetadataMgr().getDb(catalogName, dbName); + if (db == null) { + return Pair.create(HttpResponseStatus.NOT_FOUND, + String.format("Database [%s.%s] does not exists", catalogName, dbName)); + } + context.setDatabase(db.getFullName()); + } + + context.setIsHTTPQueryDump(true); + + StatementBase parsedStmt; + try { + parsedStmt = SqlParser.parse(query, context.getSessionVariable()).get(0); + StmtExecutor executor = new StmtExecutor(context, parsedStmt); + executor.execute(); + } catch (Exception e) { + LOG.warn("execute query failed. ", e); + return Pair.create(HttpResponseStatus.BAD_REQUEST, "execute query failed. " + e.getMessage()); + } + + DumpInfo dumpInfo = context.getDumpInfo(); + if (dumpInfo != null) { + dumpInfo.setDesensitizedInfo(enableMock); + String dumpStr = GSON.toJson(dumpInfo, QueryDumpInfo.class); + return Pair.create(HttpResponseStatus.OK, dumpStr); + } else { + return Pair.create(HttpResponseStatus.BAD_REQUEST, "not use cbo planner, try again."); + } + } finally { + context.setCurrentCatalog(prevCatalog); + context.setDatabase(prevDb); + context.setIsHTTPQueryDump(prevIsHTTPQueryDump); + } + } +} diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java index 4e70b4f66dbb0..512777d5b9c32 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/function/MetaFunctions.java @@ -44,8 +44,10 @@ import com.starrocks.sql.analyzer.Authorizer; import com.starrocks.sql.optimizer.CachingMvPlanContextBuilder; import com.starrocks.sql.optimizer.OptExpression; +import com.starrocks.sql.optimizer.dump.QueryDumper; import com.starrocks.sql.optimizer.operator.scalar.ConstantOperator; import com.starrocks.sql.optimizer.rewrite.ConstantFunction; +import io.netty.handler.codec.http.HttpResponseStatus; import org.apache.commons.collections4.MapUtils; import org.apache.commons.collections4.SetUtils; import org.apache.commons.lang3.tuple.Pair; @@ -346,4 +348,20 @@ public static ConstantOperator inspectMvPlan(ConstantOperator mvName, ConstantOp return ConstantOperator.createVarchar("failed"); } } + + @ConstantFunction(name = "get_query_dump", argTypes = {VARCHAR, BOOLEAN}, returnType = VARCHAR, isMetaFunction = true) + public static ConstantOperator getQueryDump(ConstantOperator query, ConstantOperator enableMock) { + com.starrocks.common.Pair statusAndRes = + QueryDumper.dumpQuery("", "", query.getVarchar(), enableMock.getBoolean()); + if (statusAndRes.first != HttpResponseStatus.OK) { + ErrorReport.reportSemanticException(ErrorCode.ERR_INVALID_PARAMETER, "get_query_dump: " + statusAndRes.second); + } + return ConstantOperator.createVarchar(statusAndRes.second); + } + + @ConstantFunction(name = "get_query_dump", argTypes = {VARCHAR}, returnType = VARCHAR, isMetaFunction = true) + public static ConstantOperator getQueryDump(ConstantOperator query) { + return getQueryDump(query, ConstantOperator.createBoolean(false)); + } + } diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java index 2deaffab12447..5184bc89b9b11 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/ScalarOperatorEvaluator.java @@ -117,7 +117,10 @@ public Function getMetaFunction(FunctionName name, Type[] args) { if (invoker == null || !invoker.isMetaFunction) { return null; } - return new Function(name, Lists.newArrayList(args), Type.VARCHAR, false); + + Function function = new Function(name, Lists.newArrayList(args), Type.VARCHAR, false); + function.setMetaFunction(true); + return function; } public ScalarOperator evaluation(CallOperator root) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java index 1d86e3a97e4c7..b0b3435c1e6a3 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. - package com.starrocks.sql.optimizer.rewrite.scalar; import com.starrocks.analysis.BinaryType; import com.starrocks.catalog.Type; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.optimizer.operator.scalar.BinaryPredicateOperator; import com.starrocks.sql.optimizer.operator.scalar.CallOperator; import com.starrocks.sql.optimizer.operator.scalar.CastOperator; @@ -48,9 +48,19 @@ public FoldConstantsRule(boolean needMonotonicFunc) { @Override public ScalarOperator visitCall(CallOperator call, ScalarOperatorRewriteContext context) { - if (call.isAggregate() || notAllConstant(call.getChildren())) { + if (call.isAggregate()) { + return call; + } + + if (notAllConstant(call.getChildren())) { + if (call.getFunction().isMetaFunction()) { + String errMsg = String.format("Meta function %s does not support non-constant arguments", + call.getFunction().getFunctionName()); + throw new SemanticException(errMsg); + } return call; } + return ScalarOperatorEvaluator.INSTANCE.evaluation(call, needMonotonicFunc); } diff --git a/fe/fe-core/src/test/java/com/starrocks/http/QueryDumpActionTest.java b/fe/fe-core/src/test/java/com/starrocks/http/QueryDumpActionTest.java new file mode 100644 index 0000000000000..fa11f8c4d0617 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/http/QueryDumpActionTest.java @@ -0,0 +1,136 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.http; + +import com.starrocks.http.rest.QueryDumpAction; +import com.starrocks.metric.MetricRepo; +import com.starrocks.service.ExecuteEnv; +import io.netty.handler.codec.http.HttpResponseStatus; +import okhttp3.HttpUrl; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; +import org.junit.Test; + +import java.io.IOException; + +import static org.assertj.core.api.Assertions.assertThat; + +public class QueryDumpActionTest extends StarRocksHttpTestCase { + + @Override + protected void doSetUp() { + MetricRepo.init(); + ExecuteEnv.setup(); + } + + @Test + public void testSuccess() throws Exception { + setUpWithCatalog(); + + try (Response response = postQueryDump(DB_NAME, true, "select * from testTbl")) { + assertThat(response.isSuccessful()).isTrue(); + String body = response.body().string(); + assertThat(body) + .contains("\"statement\":\"SELECT *\\nFROM db_mock_000.tbl_mock_001\"") + .containsPattern("\"column_statistics\":\\{.+\\}") + .containsPattern("\"table_row_count\":\\{.+\\}"); + } + + try (Response response = postQueryDump(DB_NAME, false, "select * from testTbl")) { + assertThat(response.isSuccessful()).isTrue(); + String body = response.body().string(); + assertThat(body) + .containsPattern("\"column_statistics\":\\{.+\\}") + .containsPattern("\"table_row_count\":\\{.+\\}"); + } + + try (Response response = postQueryDump(DB_NAME, null, "select * from testTbl")) { + assertThat(response.isSuccessful()).isTrue(); + String body = response.body().string(); + assertThat(body) + .containsPattern("\"column_statistics\":\\{.+\\}") + .containsPattern("\"table_row_count\":\\{.+\\}"); + } + + try (Response response = postQueryDump("default_catalog." + DB_NAME, false, "select * from testTbl")) { + assertThat(response.isSuccessful()).isTrue(); + String body = response.body().string(); + assertThat(body) + .containsPattern("\"column_statistics\":\\{.+\\}") + .containsPattern("\"table_row_count\":\\{.+\\}"); + } + } + + @Test + public void testInvalidQuery() throws Exception { + setUpWithCatalog(); + + try (Response response = postQueryDump(DB_NAME, false, "")) { + String body = response.body().string(); + assertThat(response.code()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + assertThat(body).contains("query is empty"); + } + + try (Response response = postQueryDump("default_catalog." + DB_NAME, false, "invalid-query")) { + String body = response.body().string(); + assertThat(response.code()).isEqualTo(HttpResponseStatus.BAD_REQUEST.code()); + assertThat(body).contains("execute query failed. Getting syntax error"); + } + } + + @Test + public void testInvalidDatabase() throws Exception { + setUpWithCatalog(); + + try (Response response = postQueryDump("default_catalog.no_db", false, "select * from testTbl")) { + String body = response.body().string(); + assertThat(response.code()).isEqualTo(HttpResponseStatus.NOT_FOUND.code()); + assertThat(body).contains("Database [default_catalog.no_db] does not exists"); + } + try (Response response = postQueryDump("", false, "select * from testTbl")) { + String body = response.body().string(); + assertThat(response.isSuccessful()).isTrue(); + assertThat(body).contains("Getting analyzing error. Detail message: No database selected."); + } + try (Response response = postQueryDump(null, false, "select * from testTbl")) { + String body = response.body().string(); + assertThat(response.isSuccessful()).isTrue(); + assertThat(body).contains("Getting analyzing error. Detail message: No database selected."); + } + } + + private Response postQueryDump(String db, Boolean enableMock, String query) throws IOException { + RequestBody body = RequestBody.create(query, null); + + HttpUrl.Builder builder = HttpUrl.parse(BASE_URL + QueryDumpAction.URL).newBuilder(); + if (db != null) { + builder.addQueryParameter("db", db); + } + if (enableMock != null) { + builder.addQueryParameter("mock", enableMock.toString()); + } + HttpUrl url = builder.build(); + + Request request = new Request.Builder() + .addHeader("Authorization", rootAuth) + .url(url) + .post(body) + .build(); + + return networkClient.newCall(request).execute(); + } + +} diff --git a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java index 6fcc09fcb1e2a..63dd096836a2f 100644 --- a/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java +++ b/fe/fe-core/src/test/java/com/starrocks/http/StarRocksHttpTestCase.java @@ -35,6 +35,7 @@ package com.starrocks.http; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Lists; import com.starrocks.alter.MaterializedViewHandler; import com.starrocks.alter.SchemaChangeHandler; @@ -51,12 +52,14 @@ import com.starrocks.catalog.PartitionInfo; import com.starrocks.catalog.Replica; import com.starrocks.catalog.SinglePartitionInfo; +import com.starrocks.catalog.TableProperty; import com.starrocks.catalog.TabletInvertedIndex; import com.starrocks.catalog.TabletMeta; import com.starrocks.catalog.Type; import com.starrocks.common.DdlException; import com.starrocks.common.ExceptionChecker.ThrowingRunnable; import com.starrocks.common.jmockit.Deencapsulation; +import com.starrocks.common.util.PropertyAnalyzer; import com.starrocks.load.Load; import com.starrocks.persist.EditLog; import com.starrocks.server.GlobalStateMgr; @@ -335,6 +338,7 @@ private static GlobalStateMgr newDelegateGlobalStateMgr() { //EasyMock.expect(globalStateMgr.getAuth()).andReturn(starrocksAuth).anyTimes(); Database db = new Database(testDbId, "testDb"); OlapTable table = newTable(TABLE_NAME); + table.setTableProperty(new TableProperty(ImmutableMap.of(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM, "1"))); db.registerTableUnlocked(table); OlapTable table1 = newTable(TABLE_NAME + 1); db.registerTableUnlocked(table1); @@ -479,6 +483,14 @@ MaterializedViewHandler getRollupHandler() { globalStateMgr.getTabletInvertedIndex(); minTimes = 0; result = tabletInvertedIndex; + + globalStateMgr.isLeader(); + minTimes = 0; + result = true; + + globalStateMgr.isSafeMode(); + minTimes = 0; + result = true; } }; diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/QueryDumperTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/QueryDumperTest.java new file mode 100644 index 0000000000000..76b7e21f8d975 --- /dev/null +++ b/fe/fe-core/src/test/java/com/starrocks/sql/optimizer/QueryDumperTest.java @@ -0,0 +1,127 @@ +// Copyright 2021-present StarRocks, Inc. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.starrocks.sql.optimizer; + +import com.starrocks.common.Pair; +import com.starrocks.qe.ConnectContext; +import com.starrocks.sql.optimizer.dump.DumpInfo; +import com.starrocks.sql.optimizer.dump.QueryDumper; +import com.starrocks.sql.plan.PlanTestBase; +import io.netty.handler.codec.http.HttpResponseStatus; +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class QueryDumperTest extends PlanTestBase { + private DumpInfo prevDumpInfo = null; + + @BeforeClass + public static void beforeClass() throws Exception { + PlanTestBase.beforeClass(); + } + + @Before + public void before() { + prevDumpInfo = connectContext.getDumpInfo(); + connectContext.setDumpInfo(null); + } + + @After + public void after() { + // After dumping query, the connect context should be reset to the previous state. + assertThat(connectContext.isHTTPQueryDump()).isFalse(); + assertThat(connectContext.getCurrentCatalog()).isEqualTo("default_catalog"); + assertThat(connectContext.getDatabase()).isEqualTo("test"); + + connectContext.setDumpInfo(prevDumpInfo); + } + + @Test + public void testDumpQueryInvalidConnectContext() { + ConnectContext context = ConnectContext.get(); + try { + ConnectContext.remove(); + + Pair statusAndRes = QueryDumper.dumpQuery("catalog", "db", "", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.BAD_REQUEST); + assertThat(statusAndRes.second).contains("There is no ConnectContext for this thread:"); + + } finally { + context.setThreadLocalInfo(); + } + } + + @Test + public void testDumpQueryInvalidQuery() { + { + + Pair statusAndRes = QueryDumper.dumpQuery("default_catalog", "test", "", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.BAD_REQUEST); + assertThat(statusAndRes.second).isEqualTo("query is empty"); + } + + { + Pair statusAndRes = QueryDumper.dumpQuery("default_catalog", "test", "no-query", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.BAD_REQUEST); + assertThat(statusAndRes.second).contains("execute query failed. Getting syntax error"); + } + } + + @Test + public void testDumpQueryInvalidCatalog() { + Pair statusAndRes = + QueryDumper.dumpQuery("no-catalog", "test", "select count(v1) from t0", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.NOT_FOUND); + assertThat(statusAndRes.second).isEqualTo("Database [no-catalog.test] does not exists"); + } + + @Test + public void testDumpQueryInvalidDatabase() { + Pair statusAndRes = + QueryDumper.dumpQuery("default_catalog", "no-db", "select count(v1) from t0", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.NOT_FOUND); + assertThat(statusAndRes.second).isEqualTo("Database [default_catalog.no-db] does not exists"); + } + + @Test + public void testDumpQuerySuccess() { + { + Pair statusAndRes = + QueryDumper.dumpQuery("default_catalog", "test", "select count(v1) from t0", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.OK); + // Non-empty column_statistics. + assertThat(statusAndRes.second).containsPattern("\"column_statistics\":\\{.+\\}"); + } + { + Pair statusAndRes = + QueryDumper.dumpQuery("", "", "select count(v1) from t0", false); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.OK); + // Non-empty column_statistics. + assertThat(statusAndRes.second).containsPattern("\"column_statistics\":\\{.+\\}"); + } + { + Pair statusAndRes = + QueryDumper.dumpQuery("default_catalog", "test", "select count(v1) from t0", true); + assertThat(statusAndRes.first).isEqualTo(HttpResponseStatus.OK); + // Non-empty column_statistics. + assertThat(statusAndRes.second) + .containsPattern("\"column_statistics\":\\{.+\\}") + .contains("SELECT count(tbl_mock_001.mock_002)"); + } + } +} diff --git a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java index 34b71debc4fe5..16943b224f765 100644 --- a/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java +++ b/fe/fe-core/src/test/java/com/starrocks/sql/plan/ConstantExpressionTest.java @@ -18,7 +18,9 @@ package com.starrocks.sql.plan; import com.starrocks.qe.SqlModeHelper; +import com.starrocks.sql.analyzer.SemanticException; import com.starrocks.sql.common.StarRocksPlannerException; +import com.starrocks.sql.optimizer.dump.DumpInfo; import com.starrocks.sql.parser.ParsingException; import org.junit.Assert; import org.junit.BeforeClass; @@ -479,4 +481,68 @@ public void testNumericLiteralComparison() throws Exception { connectContext.getSessionVariable().setSqlMode(prevSqlMode); } } + + @Test + public void testGetQueryDump() throws Exception { + DumpInfo prevDumpInfo = connectContext.getDumpInfo(); + + try { + connectContext.setDumpInfo(null); + + // Non-constant arguments. + { + String sql = "SELECT get_query_dump(lower('select count(v1) from t0')) from t0"; + Assert.assertThrows("Meta function get_query_dump does not support non-constant arguments", + SemanticException.class, () -> getFragmentPlan(sql)); + } + + // Success cases. + { + String sql = "SELECT get_query_dump('select count(v1) from t0', false) from t0"; + String plan = getFragmentPlan(sql); + assertContains(plan, "{\"statement\":\"select count(v1) from t0\""); + } + + { + String sql = "SELECT get_query_dump('select count(v1) from t0', true) from t0"; + String plan = getFragmentPlan(sql); + assertContains(plan, "{\"statement\":\"SELECT count(tbl_mock_001.mock_002)..."); + } + + { + String sql = "SELECT get_query_dump('select count(v1) from t0') from t0"; + String plan = getFragmentPlan(sql); + assertContains(plan, "{\"statement\":\"select count(v1) from t0\""); + } + + { + String sql = "SELECT get_query_dump(concat('select count(v1)', ' from t0')) from t0"; + String plan = getFragmentPlan(sql); + assertContains(plan, "{\"statement\":\"select count(v1) from t0\""); + } + + // Failed cases. + { + String sql = "SELECT get_query_dump('') from t0"; + Assert.assertThrows("Invalid parameter get_query_dump: query is empty", + StarRocksPlannerException.class, () -> getFragmentPlan(sql)); + } + { + String sql = "SELECT get_query_dump('not-a-query') from t0"; + Assert.assertThrows("Invalid parameter get_query_dump: execute query failed.", + StarRocksPlannerException.class, () -> getFragmentPlan(sql)); + } + + // Success cases after failed cases. + { + String sql = "SELECT get_query_dump(concat('select count(v1)', ' from t0')) from t0"; + String plan = getFragmentPlan(sql); + assertContains(plan, "{\"statement\":\"select count(v1) from t0\""); + } + + } finally { + connectContext.setDumpInfo(prevDumpInfo); + } + + } } diff --git a/test/sql/test_function/R/test_get_query_dump b/test/sql/test_function/R/test_get_query_dump new file mode 100644 index 0000000000000..14d36695516a4 --- /dev/null +++ b/test/sql/test_function/R/test_get_query_dump @@ -0,0 +1,106 @@ +-- name: test_get_query_dump +CREATE TABLE __row_util_base ( + k1 bigint NULL +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +DISTRIBUTED BY HASH(`k1`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); +-- result: +-- !result +insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1)); +-- result: +-- !result +insert into __row_util_base select * from __row_util_base; -- 20000 +insert into __row_util_base select * from __row_util_base; -- 40000 + +CREATE TABLE __row_util ( + idx bigint NULL +) ENGINE=OLAP +DUPLICATE KEY(`idx`) +DISTRIBUTED BY HASH(`idx`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); +-- result: +-- !result +insert into __row_util select row_number() over() as idx from __row_util_base; +-- result: +-- !result +CREATE TABLE t1 ( + k1 bigint NULL, + c_int_1_seq bigint NULL, + c_int_2_seq string NULL + +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +DISTRIBUTED BY HASH(`k1`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); +-- result: +-- !result +insert into t1 +select + idx, + idx, + concat('abc1-', idx) +from __row_util; +-- result: +-- !result +select get_query_dump(''); +-- result: +E: (6013, 'SemanticException: Getting analyzing error. Detail message: Invalid parameter get_query_dump: query is empty.') +-- !result +select get_query_dump('invalid-query'); +-- result: +E: (6013, "SemanticException: Getting analyzing error. Detail message: Invalid parameter get_query_dump: execute query failed. Getting syntax error at line 1, column 0. Detail message: Unexpected input 'invalid', the most similar input is {'INSTALL', 'UNINSTALL', 'KILL', 'LOAD', 'ADMIN', '(', ';'}..") +-- !result +select get_query_dump(lower('select * from t1')); +-- result: +E: (1064, 'Getting analyzing error. Detail message: Meta function get_query_dump does not support non-constant arguments.') +-- !result +with + query_dump as (select get_query_dump('select * from t1') as x), + w1 as ( + select 'statement' as k, length(get_json_string(x, '$.statement')) > 2 as not_empty from query_dump + UNION ALL + select 'table_meta' as k, length(get_json_string(x, '$.table_meta')) > 2 as not_empty from query_dump + UNION ALL + select 'table_row_count' as k, length(get_json_string(x, '$.table_row_count')) > 2 as not_empty from query_dump + UNION ALL + select 'column_statistics' as k, length(get_json_string(x, '$.column_statistics')) > 2 as not_empty from query_dump + ) +select * from w1 order by k; +-- result: +column_statistics 1 +statement 1 +table_meta 1 +table_row_count 1 +-- !result +with + query_dump as (select get_query_dump(concat('select *', ' from t1')) as x), + w1 as ( + select 'statement' as k, length(get_json_string(x, '$.statement')) > 2 as not_empty from query_dump + UNION ALL + select 'table_meta' as k, length(get_json_string(x, '$.table_meta')) > 2 as not_empty from query_dump + UNION ALL + select 'table_row_count' as k, length(get_json_string(x, '$.table_row_count')) > 2 as not_empty from query_dump + UNION ALL + select 'column_statistics' as k, length(get_json_string(x, '$.column_statistics')) > 2 as not_empty from query_dump + ) +select * from w1 order by k; +-- result: +column_statistics 1 +statement 1 +table_meta 1 +table_row_count 1 +-- !result +with + query_dump as (select get_query_dump('select * from t1', true) as x) +select get_json_string(x, '$.statement') from query_dump; +-- result: +SELECT * +FROM db_mock_000.tbl_mock_001 +-- !result \ No newline at end of file diff --git a/test/sql/test_function/T/test_get_query_dump b/test/sql/test_function/T/test_get_query_dump new file mode 100644 index 0000000000000..21978e6f7c18a --- /dev/null +++ b/test/sql/test_function/T/test_get_query_dump @@ -0,0 +1,80 @@ +-- name: test_get_query_dump + +-- Prepare data. +CREATE TABLE __row_util_base ( + k1 bigint NULL +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +DISTRIBUTED BY HASH(`k1`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); +insert into __row_util_base select generate_series from TABLE(generate_series(0, 10000 - 1)); +insert into __row_util_base select * from __row_util_base; -- 20000 +insert into __row_util_base select * from __row_util_base; -- 40000 + +CREATE TABLE __row_util ( + idx bigint NULL +) ENGINE=OLAP +DUPLICATE KEY(`idx`) +DISTRIBUTED BY HASH(`idx`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); +insert into __row_util select row_number() over() as idx from __row_util_base; + +CREATE TABLE t1 ( + k1 bigint NULL, + c_int_1_seq bigint NULL, + c_int_2_seq string NULL + +) ENGINE=OLAP +DUPLICATE KEY(`k1`) +DISTRIBUTED BY HASH(`k1`) BUCKETS 32 +PROPERTIES ( + "replication_num" = "1" +); + +insert into t1 +select + idx, + idx, + concat('abc1-', idx) +from __row_util; + +-- Invalid cases. +select get_query_dump(''); +select get_query_dump('invalid-query'); +select get_query_dump(lower('select * from t1')); + +-- Valid cases. +with + query_dump as (select get_query_dump('select * from t1') as x), + w1 as ( + select 'statement' as k, length(get_json_string(x, '$.statement')) > 2 as not_empty from query_dump + UNION ALL + select 'table_meta' as k, length(get_json_string(x, '$.table_meta')) > 2 as not_empty from query_dump + UNION ALL + select 'table_row_count' as k, length(get_json_string(x, '$.table_row_count')) > 2 as not_empty from query_dump + UNION ALL + select 'column_statistics' as k, length(get_json_string(x, '$.column_statistics')) > 2 as not_empty from query_dump + ) +select * from w1 order by k; + + +with + query_dump as (select get_query_dump(concat('select *', ' from t1')) as x), + w1 as ( + select 'statement' as k, length(get_json_string(x, '$.statement')) > 2 as not_empty from query_dump + UNION ALL + select 'table_meta' as k, length(get_json_string(x, '$.table_meta')) > 2 as not_empty from query_dump + UNION ALL + select 'table_row_count' as k, length(get_json_string(x, '$.table_row_count')) > 2 as not_empty from query_dump + UNION ALL + select 'column_statistics' as k, length(get_json_string(x, '$.column_statistics')) > 2 as not_empty from query_dump + ) +select * from w1 order by k; + +with + query_dump as (select get_query_dump('select * from t1', true) as x) +select get_json_string(x, '$.statement') from query_dump; From feec785fd2efd22c0c3214d21c17ea75639dac2e Mon Sep 17 00:00:00 2001 From: "zihe.liu" Date: Wed, 10 Jul 2024 15:22:28 +0800 Subject: [PATCH 2/2] s Signed-off-by: zihe.liu --- .../sql/optimizer/rewrite/scalar/FoldConstantsRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java index b0b3435c1e6a3..c4d70eb8aad3f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/optimizer/rewrite/scalar/FoldConstantsRule.java @@ -53,7 +53,7 @@ public ScalarOperator visitCall(CallOperator call, ScalarOperatorRewriteContext } if (notAllConstant(call.getChildren())) { - if (call.getFunction().isMetaFunction()) { + if (call.getFunction() != null && call.getFunction().isMetaFunction()) { String errMsg = String.format("Meta function %s does not support non-constant arguments", call.getFunction().getFunctionName()); throw new SemanticException(errMsg);