From 2a1eacbc22e4fdb5ff1ebdc6e112754b40e29d35 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 11 Dec 2024 21:02:29 +0800 Subject: [PATCH 01/13] backup --- .../persistence/pipe/PipeTaskInfo.java | 7 ++++++- .../thrift/ConfigNodeRPCServiceProcessor.java | 12 ++++++++++++ .../db/protocol/client/ConfigNodeClient.java | 14 ++++++++++++++ .../pipe/agent/task/meta/PipeMeta.java | 11 +++++++++++ .../pipe/agent/task/meta/PipeMetaKeeper.java | 8 ++++++++ .../src/main/thrift/confignode.thrift | 19 +++++++++++++++++++ 6 files changed, 70 insertions(+), 1 deletion(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 70d23a47cc7e..5db8fae8abc0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleLeaderChangePlan; @@ -341,9 +342,13 @@ private void checkBeforeDropPipeInternal(final String pipeName) { } public boolean isPipeExisted(final String pipeName) { + return isPipeExisted(pipeName, SystemConstant.SQL_DIALECT_TREE_VALUE); + } + + public boolean isPipeExisted(final String pipeName, final String dialect) { acquireReadLock(); try { - return pipeMetaKeeper.containsPipeMeta(pipeName); + return pipeMetaKeeper.containsPipeMeta(pipeName, dialect); } finally { releaseReadLock(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index ce360da47cfc..b7a83639ff35 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -198,6 +198,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1113,11 +1115,21 @@ public TSStatus startPipe(String pipeName) { return configManager.startPipe(pipeName); } + @Override + public TSStatus startPipeExtended(TStartPipeReq req) throws TException { + return null; + } + @Override public TSStatus stopPipe(String pipeName) { return configManager.stopPipe(pipeName); } + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) throws TException { + return null; + } + @Override public TSStatus dropPipe(String pipeName) { return configManager.dropPipe( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java index e9bfb6ce6472..878a7a5201e1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/protocol/client/ConfigNodeClient.java @@ -161,6 +161,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TSystemConfigurationResp; import org.apache.iotdb.confignode.rpc.thrift.TTestOperation; @@ -1052,12 +1054,24 @@ public TSStatus startPipe(String pipeName) throws TException { () -> client.startPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus startPipeExtended(TStartPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.startPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus stopPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( () -> client.stopPipe(pipeName), status -> !updateConfigNodeLeader(status)); } + @Override + public TSStatus stopPipeExtended(TStopPipeReq req) throws TException { + return executeRemoteCallWithRetry( + () -> client.stopPipeExtended(req), status -> !updateConfigNodeLeader(status)); + } + @Override public TSStatus dropPipe(String pipeName) throws TException { return executeRemoteCallWithRetry( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 997278010e9f..77216c09fe9d 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -19,7 +19,10 @@ package org.apache.iotdb.commons.pipe.agent.task.meta; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; + import org.apache.tsfile.utils.PublicBAOS; +import org.checkerframework.checker.nullness.qual.NonNull; import java.io.DataOutputStream; import java.io.FileInputStream; @@ -61,6 +64,14 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } + public boolean matchDialect(@NonNull final String dialect) { + return dialect.equalsIgnoreCase( + getStaticMeta() + .getExtractorParameters() + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE)); + } + public ByteBuffer serialize() throws IOException { final PublicBAOS byteArrayOutputStream = new PublicBAOS(); final DataOutputStream outputStream = new DataOutputStream(byteArrayOutputStream); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index f8a8c9e55ac9..6a2e7d1bb7d6 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -85,6 +85,14 @@ public boolean containsPipeMeta(String pipeName) { return pipeNameToPipeMetaMap.containsKey(pipeName); } + public boolean containsPipeMeta(String pipeName, String dialect) { + final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); + if (Objects.isNull(pipeMeta)) { + return false; + } + return pipeMeta.matchDialect(dialect); + } + public Iterable getPipeMetaList() { return pipeNameToPipeMetaMap.values(); } diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 8f2110b1672a..8e7b6ff5a8a5 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -750,11 +750,23 @@ struct TAlterPipeReq { 6: optional map extractorAttributes 7: optional bool isReplaceAllExtractorAttributes 8: optional bool ifExistsCondition + 9: optional string sqlDialect +} + +struct TStartPipeReq { + 1: required string pipeName + 2: optional string sqlDialect +} + +struct TStopPipeReq { + 1: required string pipeName + 2: optional string sqlDialect } struct TDropPipeReq { 1: required string pipeName 2: optional bool ifExistsCondition + 3: optional string sqlDialect } // Deprecated, restored for compatibility @@ -767,6 +779,7 @@ struct TPipeSinkInfo { struct TShowPipeReq { 1: optional string pipeName 2: optional bool whereClause + 3: optional string sqlDialect } struct TShowPipeResp { @@ -1664,9 +1677,15 @@ service IConfigNodeRPCService { /** Start Pipe */ common.TSStatus startPipe(string pipeName) + /** Start Pipe */ + common.TSStatus startPipeExtended(TStartPipeReq req) + /** Stop Pipe */ common.TSStatus stopPipe(string pipeName) + /** Stop Pipe */ + common.TSStatus stopPipeExtended(TStopPipeReq req) + /** Drop Pipe */ common.TSStatus dropPipe(string pipeName) From abb453f26cbad8d0b4b13a6c57eefab8dce19725 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 11 Dec 2024 22:06:12 +0800 Subject: [PATCH 02/13] setup --- .../response/pipe/task/PipeTableResp.java | 7 +++ .../confignode/manager/ConfigManager.java | 10 ++-- .../iotdb/confignode/manager/IManager.java | 23 +++++--- .../coordinator/task/PipeTaskCoordinator.java | 52 ++++++++++++++++++- .../persistence/pipe/PipeTaskInfo.java | 16 ++---- .../impl/pipe/task/AlterPipeProcedureV2.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 24 ++++++--- .../pipe/agent/task/meta/PipeMeta.java | 5 +- .../pipe/agent/task/meta/PipeMetaKeeper.java | 4 +- 9 files changed, 105 insertions(+), 40 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 020690f24651..0e658aeeee45 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -94,6 +94,13 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { } } + public PipeTableResp filter( + final Boolean whereClause, final String pipeName, final String sqlDialect) { + final PipeTableResp resp = filter(whereClause, pipeName); + resp.allPipeMeta.removeIf(meta -> !meta.matchSqlDialect(sqlDialect)); + return resp; + } + public TGetAllPipeInfoResp convertToTGetAllPipeInfoResp() throws IOException { final List pipeInformationByteBuffers = new ArrayList<>(); for (final PipeMeta pipeMeta : allPipeMeta) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java index 43383319f85c..9c69a8741ddf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/ConfigManager.java @@ -224,6 +224,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TTimeSlotList; @@ -2198,18 +2200,18 @@ public TSStatus alterPipe(TAlterPipeReq req) { } @Override - public TSStatus startPipe(String pipeName) { + public TSStatus startPipe(TStartPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().startPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().startPipe(req) : status; } @Override - public TSStatus stopPipe(String pipeName) { + public TSStatus stopPipe(TStopPipeReq req) { TSStatus status = confirmLeader(); return status.getCode() == TSStatusCode.SUCCESS_STATUS.getStatusCode() - ? pipeManager.getPipeTaskCoordinator().stopPipe(pipeName) + ? pipeManager.getPipeTaskCoordinator().stopPipe(req) : status; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java index 526c66612995..c572d6d51dfc 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/IManager.java @@ -145,6 +145,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicReq; import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TSubscribeReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.confignode.rpc.thrift.TUnsubscribeReq; @@ -694,34 +696,39 @@ TDataPartitionTableResp getOrCreateDataPartition( * Alter Pipe. * * @param req Info about Pipe - * @return TSStatus + * @return {@link TSStatusCode#SUCCESS_STATUS} if altered the pipe successfully, {@link + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus alterPipe(TAlterPipeReq req); /** * Start Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if started the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus startPipe(String pipeName); + TSStatus startPipe(TStartPipeReq req); /** * Stop Pipe. * - * @param pipeName name of Pipe + * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if stopped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ - TSStatus stopPipe(String pipeName); + TSStatus stopPipe(TStopPipeReq req); /** * Drop Pipe. * * @param req Info about Pipe * @return {@link TSStatusCode#SUCCESS_STATUS} if dropped the pipe successfully, {@link - * TSStatusCode#PIPE_ERROR} if encountered failure. + * TSStatusCode#PIPE_ERROR} if encountered failure, {@link TSStatusCode#PIPE_NOT_EXIST_ERROR} + * if the pipe does not exist. */ TSStatus dropPipe(TDropPipeReq req); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 7de39849e599..1516b07584ad 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -21,6 +21,7 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.consensus.request.read.pipe.task.ShowPipePlanV2; import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; import org.apache.iotdb.confignode.manager.ConfigManager; @@ -31,6 +32,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TGetAllPipeInfoResp; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TShowPipeResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.consensus.exception.ConsensusException; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -130,6 +133,19 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { + final String pipeName = req.getPipeName(); + final String sqlDialect = + req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to alter pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } final TSStatus status = configManager.getProcedureManager().alterPipe(req); if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to alter pipe {}. Result status: {}.", req.getPipeName(), status); @@ -151,6 +167,20 @@ public TSStatus startPipe(String pipeName) { return status; } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + public TSStatus startPipe(TStartPipeReq req) { + final String pipeName = req.getPipeName(); + final String sqlDialect = + req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to start pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return startPipe(pipeName); + } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(String pipeName) { final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); @@ -176,10 +206,26 @@ public TSStatus stopPipe(String pipeName) { return status; } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ + public TSStatus stopPipe(TStopPipeReq req) { + final String pipeName = req.getPipeName(); + final String sqlDialect = + req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + return RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to stop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + return stopPipe(pipeName); + } + /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName); + final String sqlDialect = + req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName, sqlDialect); TSStatus status = null; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().dropConsensusPipe(pipeName); @@ -204,8 +250,10 @@ public TSStatus dropPipe(TDropPipeReq req) { public TShowPipeResp showPipes(final TShowPipeReq req) { try { + final String sqlDialect = + req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName) + .filter(req.whereClause, req.pipeName, sqlDialect) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 5db8fae8abc0..920b484778f0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -186,23 +186,19 @@ private boolean checkBeforeCreatePipeInternal(final TCreatePipeReq createPipeReq throw new PipeException(exceptionMessage); } - public boolean checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) + public void checkAndUpdateRequestBeforeAlterPipe(final TAlterPipeReq alterPipeRequest) throws PipeException { acquireReadLock(); try { - return checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); + checkAndUpdateRequestBeforeAlterPipeInternal(alterPipeRequest); } finally { releaseReadLock(); } } - private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) + private void checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq alterPipeRequest) throws PipeException { if (!isPipeExisted(alterPipeRequest.getPipeName())) { - if (alterPipeRequest.isSetIfExistsCondition() && alterPipeRequest.isIfExistsCondition()) { - return false; - } - final String exceptionMessage = String.format( "Failed to alter pipe %s, the pipe does not exist", alterPipeRequest.getPipeName()); @@ -266,8 +262,6 @@ private boolean checkAndUpdateRequestBeforeAlterPipeInternal(final TAlterPipeReq .getAttribute()); } } - - return true; } public void checkBeforeStartPipe(final String pipeName) throws PipeException { @@ -345,10 +339,10 @@ public boolean isPipeExisted(final String pipeName) { return isPipeExisted(pipeName, SystemConstant.SQL_DIALECT_TREE_VALUE); } - public boolean isPipeExisted(final String pipeName, final String dialect) { + public boolean isPipeExisted(final String pipeName, final String sqlDialect) { acquireReadLock(); try { - return pipeMetaKeeper.containsPipeMeta(pipeName, dialect); + return pipeMetaKeeper.containsPipeMeta(pipeName, sqlDialect); } finally { releaseReadLock(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java index 8b6aceb9c300..ba7d40abc53f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/pipe/task/AlterPipeProcedureV2.java @@ -96,9 +96,7 @@ public boolean executeFromValidateTask(ConfigNodeProcedureEnv env) throws PipeEx // We should execute checkBeforeAlterPipe before checking the pipe plugin. This method will // update the alterPipeRequest based on the alterPipeRequest and existing pipe metadata. - if (!pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest)) { - return false; - } + pipeTaskInfo.get().checkAndUpdateRequestBeforeAlterPipe(alterPipeRequest); final PipeManager pipeManager = env.getConfigManager().getPipeManager(); pipeManager diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index b7a83639ff35..d2c2ffa0b0e0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -1112,28 +1113,37 @@ public TSStatus alterPipe(TAlterPipeReq req) { @Override public TSStatus startPipe(String pipeName) { - return configManager.startPipe(pipeName); + return configManager.startPipe( + new TStartPipeReq() + .setPipeName(pipeName) + .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); } @Override - public TSStatus startPipeExtended(TStartPipeReq req) throws TException { - return null; + public TSStatus startPipeExtended(TStartPipeReq req) { + return configManager.startPipe(req); } @Override public TSStatus stopPipe(String pipeName) { - return configManager.stopPipe(pipeName); + return configManager.stopPipe( + new TStopPipeReq() + .setPipeName(pipeName) + .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); } @Override - public TSStatus stopPipeExtended(TStopPipeReq req) throws TException { - return null; + public TSStatus stopPipeExtended(TStopPipeReq req) { + return configManager.stopPipe(req); } @Override public TSStatus dropPipe(String pipeName) { return configManager.dropPipe( - new TDropPipeReq().setPipeName(pipeName).setIfExistsCondition(false)); + new TDropPipeReq() + .setPipeName(pipeName) + .setIfExistsCondition(false) + .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 77216c09fe9d..cb660aec7939 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -22,7 +22,6 @@ import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.tsfile.utils.PublicBAOS; -import org.checkerframework.checker.nullness.qual.NonNull; import java.io.DataOutputStream; import java.io.FileInputStream; @@ -64,8 +63,8 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } - public boolean matchDialect(@NonNull final String dialect) { - return dialect.equalsIgnoreCase( + public boolean matchSqlDialect(final String sqlDialect) { + return sqlDialect.equalsIgnoreCase( getStaticMeta() .getExtractorParameters() .getStringOrDefault( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index 6a2e7d1bb7d6..2074cbaa404e 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -85,12 +85,12 @@ public boolean containsPipeMeta(String pipeName) { return pipeNameToPipeMetaMap.containsKey(pipeName); } - public boolean containsPipeMeta(String pipeName, String dialect) { + public boolean containsPipeMeta(String pipeName, String sqlDialect) { final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); if (Objects.isNull(pipeMeta)) { return false; } - return pipeMeta.matchDialect(dialect); + return pipeMeta.matchSqlDialect(sqlDialect); } public Iterable getPipeMetaList() { From 53b249b752c55eae44d09f81093325397d25c4c5 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Wed, 11 Dec 2024 22:41:52 +0800 Subject: [PATCH 03/13] adapt DN --- .../config/TableConfigTaskVisitor.java | 19 ++++++++++ .../config/TreeConfigTaskVisitor.java | 12 +++++++ .../executor/ClusterConfigTaskExecutor.java | 20 +++++++++-- .../config/sys/pipe/AlterPipeTask.java | 8 +++-- .../config/sys/pipe/DropPipeTask.java | 7 ++-- .../config/sys/pipe/ShowPipeTask.java | 3 +- .../config/sys/pipe/StartPipeTask.java | 7 ++-- .../config/sys/pipe/StopPipeTask.java | 7 ++-- .../plan/relational/sql/ast/AlterPipe.java | 36 ++++++++++++------- .../plan/relational/sql/ast/DropPipe.java | 21 ++++++++--- .../plan/relational/sql/ast/ShowPipes.java | 24 +++++++++---- .../plan/relational/sql/ast/StartPipe.java | 22 ++++++++---- .../plan/relational/sql/ast/StopPipe.java | 22 ++++++++---- .../metadata/pipe/AlterPipeStatement.java | 31 ++++++++++------ .../metadata/pipe/DropPipeStatement.java | 19 +++++++--- .../metadata/pipe/ShowPipesStatement.java | 18 +++++++--- .../metadata/pipe/StartPipeStatement.java | 17 ++++++--- .../metadata/pipe/StopPipeStatement.java | 17 ++++++--- 18 files changed, 231 insertions(+), 79 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 5c6178830c32..9fe76a32d519 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -721,30 +721,49 @@ protected IConfigTask visitAlterPipe(AlterPipe node, MPPQueryContext context) { .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); } + // Inject table model into the node + node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); + return new AlterPipeTask(node); } @Override protected IConfigTask visitDropPipe(DropPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + + // Inject table model into the node + node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); + return new DropPipeTask(node); } @Override protected IConfigTask visitStartPipe(StartPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + + // Inject table model into the node + node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); + return new StartPipeTask(node); } @Override protected IConfigTask visitStopPipe(StopPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); + + // Inject table model into the node + node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); + return new StopPipeTask(node); } @Override protected IConfigTask visitShowPipes(ShowPipes node, MPPQueryContext context) { context.setQueryType(QueryType.READ); + + // Inject table model into the node + node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); + return new ShowPipeTask(node); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 5b1e513089c8..9ae536375edc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -453,6 +453,9 @@ public IConfigTask visitShowAINodes( @Override public IConfigTask visitShowPipes( ShowPipesStatement showPipesStatement, MPPQueryContext context) { + // Inject tree model into the statement + showPipesStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); + return new ShowPipeTask(showPipesStatement); } @@ -502,17 +505,26 @@ public IConfigTask visitAlterPipe( .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); } + // Inject tree model into the statement + alterPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); + return new AlterPipeTask(alterPipeStatement); } @Override public IConfigTask visitStartPipe( StartPipeStatement startPipeStatement, MPPQueryContext context) { + // Inject tree model into the statement + startPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); + return new StartPipeTask(startPipeStatement); } @Override public IConfigTask visitStopPipe(StopPipeStatement stopPipeStatement, MPPQueryContext context) { + // Inject tree model into the statement + stopPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); + return new StopPipeTask(stopPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 286bfe8b5785..3c3c125b6187 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -133,6 +133,8 @@ import org.apache.iotdb.confignode.rpc.thrift.TShowTopicResp; import org.apache.iotdb.confignode.rpc.thrift.TShowVariablesResp; import org.apache.iotdb.confignode.rpc.thrift.TSpaceQuotaResp; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TThrottleQuotaResp; import org.apache.iotdb.confignode.rpc.thrift.TUnsetSchemaTemplateReq; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -2039,6 +2041,7 @@ public SettableFuture alterPipe(final AlterPipeStatement alter req.setExtractorAttributes(alterPipeStatement.getExtractorAttributes()); req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllExtractorAttributes()); req.setIfExistsCondition(alterPipeStatement.hasIfExistsCondition()); + req.setSqlDialect(alterPipeStatement.getSqlDialect()); final TSStatus tsStatus = configNodeClient.alterPipe(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); @@ -2068,7 +2071,11 @@ public SettableFuture startPipe(final StartPipeStatement start try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.startPipe(startPipeStatement.getPipeName()); + final TSStatus tsStatus = + configNodeClient.startPipeExtended( + new TStartPipeReq() + .setPipeName(startPipeStatement.getPipeName()) + .setSqlDialect(startPipeStatement.getSqlDialect())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2101,7 +2108,8 @@ public SettableFuture dropPipe(final DropPipeStatement dropPip configNodeClient.dropPipeExtended( new TDropPipeReq() .setPipeName(dropPipeStatement.getPipeName()) - .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition())); + .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition()) + .setSqlDialect(dropPipeStatement.getSqlDialect())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2130,7 +2138,12 @@ public SettableFuture stopPipe(final StopPipeStatement stopPip try (final ConfigNodeClient configNodeClient = CONFIG_NODE_CLIENT_MANAGER.borrowClient(ConfigNodeInfo.CONFIG_REGION_ID)) { - final TSStatus tsStatus = configNodeClient.stopPipe(stopPipeStatement.getPipeName()); + + final TSStatus tsStatus = + configNodeClient.stopPipeExtended( + new TStopPipeReq() + .setPipeName(stopPipeStatement.getPipeName()) + .setSqlDialect(stopPipeStatement.getSqlDialect())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2154,6 +2167,7 @@ public SettableFuture showPipes(final ShowPipesStatement showP if (showPipesStatement.getWhereClause()) { tShowPipeReq.setWhereClause(true); } + tShowPipeReq.setSqlDialect(showPipesStatement.getSqlDialect()); final List tShowPipeInfoList = configNodeClient.showPipe(tShowPipeReq).getPipeInfoList(); ShowPipeTask.buildTSBlock(tShowPipeInfoList, future); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index f6d0ebab2cbf..fba64b3bb67f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -37,13 +37,13 @@ public class AlterPipeTask implements IConfigTask { private final AlterPipeStatement alterPipeStatement; - public AlterPipeTask(AlterPipeStatement alterPipeStatement) { + public AlterPipeTask(final AlterPipeStatement alterPipeStatement) { // support now() function applyNowFunctionToExtractorAttributes(alterPipeStatement.getExtractorAttributes()); this.alterPipeStatement = alterPipeStatement; } - public AlterPipeTask(AlterPipe node) { + public AlterPipeTask(final AlterPipe node) { alterPipeStatement = new AlterPipeStatement(StatementType.ALTER_PIPE); alterPipeStatement.setPipeName(node.getPipeName()); alterPipeStatement.setIfExists(node.hasIfExistsCondition()); @@ -57,10 +57,12 @@ public AlterPipeTask(AlterPipe node) { alterPipeStatement.setReplaceAllExtractorAttributes(node.isReplaceAllExtractorAttributes()); alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); + + alterPipeStatement.setSqlDialect(node.getSqlDialect()); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.alterPipe(alterPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java index a28ca91f3c65..4bc32d64c005 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java @@ -32,18 +32,19 @@ public class DropPipeTask implements IConfigTask { private final DropPipeStatement dropPipeStatement; - public DropPipeTask(DropPipeStatement dropPipeStatement) { + public DropPipeTask(final DropPipeStatement dropPipeStatement) { this.dropPipeStatement = dropPipeStatement; } - public DropPipeTask(DropPipe node) { + public DropPipeTask(final DropPipe node) { dropPipeStatement = new DropPipeStatement(StatementType.DROP_PIPE); dropPipeStatement.setPipeName(node.getPipeName()); dropPipeStatement.setIfExists(node.hasIfExistsCondition()); + dropPipeStatement.setSqlDialect(node.getSqlDialect()); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.dropPipe(dropPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index 6481d0713ac2..d62631e794c5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -52,10 +52,11 @@ public ShowPipeTask(final ShowPipesStatement showPipesStatement) { this.showPipesStatement = showPipesStatement; } - public ShowPipeTask(ShowPipes node) { + public ShowPipeTask(final ShowPipes node) { showPipesStatement = new ShowPipesStatement(); showPipesStatement.setPipeName(node.getPipeName()); showPipesStatement.setWhereClause(node.hasWhereClause()); + showPipesStatement.setSqlDialect(node.getSqlDialect()); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java index 39ff162a43c7..e1de98b198f7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java @@ -32,17 +32,18 @@ public class StartPipeTask implements IConfigTask { private final StartPipeStatement startPipeStatement; - public StartPipeTask(StartPipeStatement startPipeStatement) { + public StartPipeTask(final StartPipeStatement startPipeStatement) { this.startPipeStatement = startPipeStatement; } - public StartPipeTask(StartPipe node) { + public StartPipeTask(final StartPipe node) { startPipeStatement = new StartPipeStatement(StatementType.START_PIPE); startPipeStatement.setPipeName(node.getPipeName()); + startPipeStatement.setSqlDialect(node.getSqlDialect()); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.startPipe(startPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java index 04806bef42f6..07918c37c541 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java @@ -32,17 +32,18 @@ public class StopPipeTask implements IConfigTask { private final StopPipeStatement stopPipeStatement; - public StopPipeTask(StopPipeStatement stopPipeStatement) { + public StopPipeTask(final StopPipeStatement stopPipeStatement) { this.stopPipeStatement = stopPipeStatement; } - public StopPipeTask(StopPipe node) { + public StopPipeTask(final StopPipe node) { stopPipeStatement = new StopPipeStatement(StatementType.STOP_PIPE); stopPipeStatement.setPipeName(node.getPipeName()); + stopPipeStatement.setSqlDialect(node.getSqlDialect()); } @Override - public ListenableFuture execute(IConfigTaskExecutor configTaskExecutor) + public ListenableFuture execute(final IConfigTaskExecutor configTaskExecutor) throws InterruptedException { return configTaskExecutor.stopPipe(stopPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java index 4d398b55fcc2..36c560d2dde6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java @@ -35,6 +35,7 @@ public class AlterPipe extends PipeStatement { private final boolean isReplaceAllExtractorAttributes; private final boolean isReplaceAllProcessorAttributes; private final boolean isReplaceAllConnectorAttributes; + private String sqlDialect; public AlterPipe( final String pipeName, @@ -55,6 +56,10 @@ public AlterPipe( this.isReplaceAllConnectorAttributes = isReplaceAllConnectorAttributes; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); + } + public String getPipeName() { return pipeName; } @@ -87,8 +92,12 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } + public String getSqlDialect() { + return sqlDialect; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } @@ -102,29 +111,31 @@ public int hashCode() { connectorAttributes, isReplaceAllExtractorAttributes, isReplaceAllProcessorAttributes, - isReplaceAllConnectorAttributes); + isReplaceAllConnectorAttributes, + sqlDialect); } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - AlterPipe alterPipe = (AlterPipe) obj; - return Objects.equals(pipeName, alterPipe.pipeName) - && Objects.equals(ifExistsCondition, alterPipe.ifExistsCondition) - && Objects.equals(extractorAttributes, alterPipe.extractorAttributes) - && Objects.equals(processorAttributes, alterPipe.processorAttributes) - && Objects.equals(connectorAttributes, alterPipe.connectorAttributes) + final AlterPipe that = (AlterPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition) + && Objects.equals(this.extractorAttributes, that.extractorAttributes) + && Objects.equals(this.processorAttributes, that.processorAttributes) + && Objects.equals(this.connectorAttributes, that.connectorAttributes) && Objects.equals( - isReplaceAllExtractorAttributes, alterPipe.isReplaceAllExtractorAttributes) + this.isReplaceAllExtractorAttributes, that.isReplaceAllExtractorAttributes) && Objects.equals( - isReplaceAllProcessorAttributes, alterPipe.isReplaceAllProcessorAttributes) + this.isReplaceAllProcessorAttributes, that.isReplaceAllProcessorAttributes) && Objects.equals( - isReplaceAllConnectorAttributes, alterPipe.isReplaceAllConnectorAttributes); + this.isReplaceAllConnectorAttributes, that.isReplaceAllConnectorAttributes) + && Objects.equals(this.sqlDialect, that.sqlDialect); } @Override @@ -138,6 +149,7 @@ public String toString() { .add("isReplaceAllExtractorAttributes", isReplaceAllExtractorAttributes) .add("isReplaceAllProcessorAttributes", isReplaceAllProcessorAttributes) .add("isReplaceAllConnectorAttributes", isReplaceAllConnectorAttributes) + .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java index 6952c06d13d6..9fa4a89a8cb6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java @@ -28,12 +28,17 @@ public class DropPipe extends PipeStatement { private final String pipeName; private final boolean ifExistsCondition; + private String sqlDialect; public DropPipe(final String pipeName, final boolean ifExistsCondition) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); this.ifExistsCondition = ifExistsCondition; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); + } + public String getPipeName() { return pipeName; } @@ -42,8 +47,12 @@ public boolean hasIfExistsCondition() { return ifExistsCondition; } + public String getSqlDialect() { + return sqlDialect; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } @@ -53,16 +62,17 @@ public int hashCode() { } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - DropPipe other = (DropPipe) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(ifExistsCondition, other.ifExistsCondition); + final DropPipe that = (DropPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition) + && Objects.equals(this.sqlDialect, that.sqlDialect); } @Override @@ -70,6 +80,7 @@ public String toString() { return toStringHelper(this) .add("pipeName", pipeName) .add("ifExistsCondition", ifExistsCondition) + .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java index e65079610931..72b816676272 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java @@ -24,17 +24,23 @@ import java.util.Objects; import static com.google.common.base.MoreObjects.toStringHelper; +import static java.util.Objects.requireNonNull; public class ShowPipes extends PipeStatement { private final String pipeName; private final boolean hasWhereClause; + private String sqlDialect; public ShowPipes(final @Nullable String pipeName, final boolean hasWhereClause) { this.pipeName = pipeName; this.hasWhereClause = hasWhereClause; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); + } + public String getPipeName() { return pipeName; } @@ -43,27 +49,32 @@ public boolean hasWhereClause() { return hasWhereClause; } + public String getSqlDialect() { + return sqlDialect; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } @Override public int hashCode() { - return Objects.hash(pipeName, hasWhereClause); + return Objects.hash(pipeName, hasWhereClause, sqlDialect); } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - ShowPipes other = (ShowPipes) obj; - return Objects.equals(pipeName, other.pipeName) - && Objects.equals(hasWhereClause, other.hasWhereClause); + final ShowPipes that = (ShowPipes) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.hasWhereClause, that.hasWhereClause) + && Objects.equals(this.sqlDialect, that.sqlDialect); } @Override @@ -71,6 +82,7 @@ public String toString() { return toStringHelper(this) .add("pipeName", pipeName) .add("hasWhereClause", hasWhereClause) + .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java index 540d6fef9ec8..743f71b473b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java @@ -27,39 +27,49 @@ public class StartPipe extends PipeStatement { private final String pipeName; + private String sqlDialect; public StartPipe(final String pipeName) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); + } + public String getPipeName() { return pipeName; } + public String getSqlDialect() { + return sqlDialect; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } @Override public int hashCode() { - return Objects.hash(pipeName); + return Objects.hash(pipeName, sqlDialect); } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StartPipe other = (StartPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StartPipe that = (StartPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.sqlDialect, that.sqlDialect); } @Override public String toString() { - return toStringHelper(this).add("pipeName", pipeName).toString(); + return toStringHelper(this).add("pipeName", pipeName).add("sqlDialect", sqlDialect).toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java index daa9ab32e02b..418c2d592b1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java @@ -27,39 +27,49 @@ public class StopPipe extends PipeStatement { private final String pipeName; + private String sqlDialect; public StopPipe(final String pipeName) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); + } + public String getPipeName() { return pipeName; } + public String getSqlDialect() { + return sqlDialect; + } + @Override - public R accept(AstVisitor visitor, C context) { + public R accept(final AstVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } @Override public int hashCode() { - return Objects.hash(pipeName); + return Objects.hash(pipeName, sqlDialect); } @Override - public boolean equals(Object obj) { + public boolean equals(final Object obj) { if (this == obj) { return true; } if (obj == null || getClass() != obj.getClass()) { return false; } - StopPipe other = (StopPipe) obj; - return Objects.equals(pipeName, other.pipeName); + final StopPipe that = (StopPipe) obj; + return Objects.equals(this.pipeName, that.pipeName) + && Objects.equals(this.sqlDialect, that.sqlDialect); } @Override public String toString() { - return toStringHelper(this).add("pipeName", pipeName).toString(); + return toStringHelper(this).add("pipeName", pipeName).add("sqlDialect", sqlDialect).toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java index de5dc3c1d594..6ec4f0ae9cee 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java @@ -44,8 +44,9 @@ public class AlterPipeStatement extends Statement implements IConfigStatement { private boolean isReplaceAllExtractorAttributes; private boolean isReplaceAllProcessorAttributes; private boolean isReplaceAllConnectorAttributes; + private String sqlDialect; - public AlterPipeStatement(StatementType alterPipeStatement) { + public AlterPipeStatement(final StatementType alterPipeStatement) { this.statementType = alterPipeStatement; } @@ -81,38 +82,46 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } - public void setPipeName(String pipeName) { + public String getSqlDialect() { + return sqlDialect; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } - public void setExtractorAttributes(Map extractorAttributes) { + public void setExtractorAttributes(final Map extractorAttributes) { this.extractorAttributes = extractorAttributes; } - public void setProcessorAttributes(Map processorAttributes) { + public void setProcessorAttributes(final Map processorAttributes) { this.processorAttributes = processorAttributes; } - public void setConnectorAttributes(Map connectorAttributes) { + public void setConnectorAttributes(final Map connectorAttributes) { this.connectorAttributes = connectorAttributes; } - public void setReplaceAllExtractorAttributes(boolean replaceAllExtractorAttributes) { + public void setReplaceAllExtractorAttributes(final boolean replaceAllExtractorAttributes) { isReplaceAllExtractorAttributes = replaceAllExtractorAttributes; } - public void setReplaceAllProcessorAttributes(boolean replaceAllProcessorAttributes) { + public void setReplaceAllProcessorAttributes(final boolean replaceAllProcessorAttributes) { isReplaceAllProcessorAttributes = replaceAllProcessorAttributes; } - public void setReplaceAllConnectorAttributes(boolean replaceAllConnectorAttributes) { + public void setReplaceAllConnectorAttributes(final boolean replaceAllConnectorAttributes) { isReplaceAllConnectorAttributes = replaceAllConnectorAttributes; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = sqlDialect; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -124,7 +133,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -134,7 +143,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java index a3403e00e689..49c7cc285b83 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java @@ -37,8 +37,9 @@ public class DropPipeStatement extends Statement implements IConfigStatement { private String pipeName; private boolean ifExistsCondition; + private String sqlDialect; - public DropPipeStatement(StatementType dropPipeStatement) { + public DropPipeStatement(final StatementType dropPipeStatement) { this.statementType = dropPipeStatement; } @@ -50,14 +51,22 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public String getSqlDialect() { + return sqlDialect; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setIfExists(boolean ifExistsCondition) { + public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = sqlDialect; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -69,7 +78,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -79,7 +88,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java index 54fe1099f39f..2e7c809bae5d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java @@ -39,6 +39,8 @@ public ShowPipesStatement() { private boolean whereClause; + private String sqlDialect; + public String getPipeName() { return pipeName; } @@ -47,21 +49,29 @@ public boolean getWhereClause() { return whereClause; } - public void setPipeName(String pipeName) { + public String getSqlDialect() { + return sqlDialect; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setWhereClause(boolean whereClause) { + public void setWhereClause(final boolean whereClause) { this.whereClause = whereClause; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = sqlDialect; + } + @Override public QueryType getQueryType() { return QueryType.READ; } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -71,7 +81,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java index 43b61a1fb445..cfcc6c2ede51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java @@ -36,8 +36,9 @@ public class StartPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private String sqlDialect; - public StartPipeStatement(StatementType startPipeStatement) { + public StartPipeStatement(final StatementType startPipeStatement) { this.statementType = startPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public String getSqlDialect() { + return sqlDialect; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = sqlDialect; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java index d463fda199d3..89e68f8daf4b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java @@ -36,8 +36,9 @@ public class StopPipeStatement extends Statement implements IConfigStatement { private String pipeName; + private String sqlDialect; - public StopPipeStatement(StatementType stopPipeStatement) { + public StopPipeStatement(final StatementType stopPipeStatement) { this.statementType = stopPipeStatement; } @@ -45,10 +46,18 @@ public String getPipeName() { return pipeName; } - public void setPipeName(String pipeName) { + public String getSqlDialect() { + return sqlDialect; + } + + public void setPipeName(final String pipeName) { this.pipeName = pipeName; } + public void setSqlDialect(final String sqlDialect) { + this.sqlDialect = sqlDialect; + } + @Override public QueryType getQueryType() { return QueryType.WRITE; @@ -60,7 +69,7 @@ public List getPaths() { } @Override - public TSStatus checkPermissionBeforeProcess(String userName) { + public TSStatus checkPermissionBeforeProcess(final String userName) { if (AuthorityChecker.SUPER_USER.equals(userName)) { return new TSStatus(TSStatusCode.SUCCESS_STATUS.getStatusCode()); } @@ -70,7 +79,7 @@ public TSStatus checkPermissionBeforeProcess(String userName) { } @Override - public R accept(StatementVisitor visitor, C context) { + public R accept(final StatementVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); } } From 30b2fafb35dbfb3dd22a42d6aabf06464eec3163 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 12 Dec 2024 12:13:09 +0800 Subject: [PATCH 04/13] add IT --- .../pipe/it/autocreate/IoTDBPipeAlterIT.java | 3 +- .../autocreate/IoTDBPipeSwitchStatusIT.java | 23 +- .../it/tablemodel/IoTDBPipeIsolationIT.java | 316 ++++++++++++++++++ .../tablemodel/IoTDBPipeSwitchStatusIT.java | 23 +- .../coordinator/task/PipeTaskCoordinator.java | 47 ++- .../persistence/pipe/PipeTaskInfo.java | 8 +- .../pipe/agent/task/meta/PipeMeta.java | 3 + 7 files changed, 376 insertions(+), 47 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java index 66bae32c5b10..a2f5c65cb73e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java @@ -26,7 +26,6 @@ import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; import org.apache.iotdb.it.framework.IoTDBTestRunner; import org.apache.iotdb.itbase.category.MultiClusterIT2AutoCreateSchema; -import org.apache.iotdb.itbase.env.BaseEnv; import org.junit.Assert; import org.junit.Test; @@ -106,7 +105,7 @@ public void testBasicAlterPipe() throws Exception { } // Alter pipe (modify) - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + try (final Connection connection = senderEnv.getConnection(); final Statement statement = connection.createStatement()) { statement.execute("alter pipe a2b modify source ('source.pattern'='root.test2')"); } catch (SQLException e) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java index cdcc41ad53fe..03e52a982e61 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeSwitchStatusIT.java @@ -266,11 +266,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -281,10 +284,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java new file mode 100644 index 000000000000..c6e888353d8f --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { + + @Test + public void testWritePipeIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "tree"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(treePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "table"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(tablePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // start pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended( + new TStartPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .startPipeExtended( + new TStartPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended( + new TStartPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .startPipeExtended( + new TStartPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + + // stop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .stopPipeExtended( + new TStopPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .stopPipeExtended( + new TStopPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .stopPipeExtended( + new TStopPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .stopPipeExtended( + new TStopPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + + // alter pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + treePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .alterPipe( + new TAlterPipeReq( + tablePipeName, + Collections.emptyMap(), + Collections.emptyMap(), + false, + false) + .setExtractorAttributes(Collections.emptyMap()) + .setIsReplaceAllExtractorAttributes(false) + .setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + + // drop pipe + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .dropPipeExtended( + new TDropPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), + client + .dropPipeExtended( + new TDropPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended( + new TDropPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended( + new TDropPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .getCode()); + } + } + + @Test + public void testReadPipeIsolation() { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(0, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Create table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java index 5c5cbcd7b950..4c80f0ae6731 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSwitchStatusIT.java @@ -303,11 +303,14 @@ public void testWrongPipeName() throws Exception { .setExtractorAttributes(extractorAttributes) .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("").getCode()); Assert.assertEquals( - TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.startPipe("*").getCode()); + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.startPipe("*").getCode()); List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); @@ -318,10 +321,14 @@ public void testWrongPipeName() throws Exception { Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("p").getCode()); - Assert.assertEquals(TSStatusCode.PIPE_ERROR.getStatusCode(), client.stopPipe("*").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p0").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("p").getCode()); + Assert.assertEquals( + TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipe("*").getCode()); showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; Assert.assertTrue( showPipeResult.stream().anyMatch((o) -> o.id.equals("p1") && o.state.equals("RUNNING"))); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 1516b07584ad..292e5319865f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -21,7 +21,6 @@ import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.pipe.agent.task.meta.PipeStaticMeta; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.confignode.consensus.request.read.pipe.task.ShowPipePlanV2; import org.apache.iotdb.confignode.consensus.response.pipe.task.PipeTableResp; import org.apache.iotdb.confignode.manager.ConfigManager; @@ -119,7 +118,7 @@ public boolean isLocked() { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus createPipe(TCreatePipeReq req) { - TSStatus status = null; + final TSStatus status; if (req.getPipeName().startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().createConsensusPipe(req); } else { @@ -134,8 +133,7 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = - req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final String sqlDialect = req.getSqlDialect(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { @@ -155,7 +153,7 @@ public TSStatus alterPipe(TAlterPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus startPipe(String pipeName) { - TSStatus status = null; + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().startConsensusPipe(pipeName); } else { @@ -170,8 +168,7 @@ public TSStatus startPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus startPipe(TStartPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = - req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final String sqlDialect = req.getSqlDialect(); if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, @@ -184,7 +181,7 @@ public TSStatus startPipe(TStartPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(String pipeName) { final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); - TSStatus status = null; + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().stopConsensusPipe(pipeName); } else { @@ -209,8 +206,7 @@ public TSStatus stopPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(TStopPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = - req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final String sqlDialect = req.getSqlDialect(); if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, @@ -223,10 +219,18 @@ public TSStatus stopPipe(TStopPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = - req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; - final boolean isPipeExistedBeforeDrop = pipeTaskInfo.isPipeExisted(pipeName, sqlDialect); - TSStatus status = null; + final String sqlDialect = req.getSqlDialect(); + final boolean isSetIfExistsCondition = + req.isSetIfExistsCondition() && req.isIfExistsCondition(); + if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + return isSetIfExistsCondition + ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) + : RpcUtils.getStatus( + TSStatusCode.PIPE_NOT_EXIST_ERROR, + String.format( + "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + } + final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().dropConsensusPipe(pipeName); } else { @@ -235,23 +239,12 @@ public TSStatus dropPipe(TDropPipeReq req) { if (status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { LOGGER.warn("Failed to drop pipe {}. Result status: {}.", pipeName, status); } - - final boolean isSetIfExistsCondition = - req.isSetIfExistsCondition() && req.isIfExistsCondition(); - // If the `IF EXISTS` condition is not set and the pipe does not exist before the delete - // operation, return an error status indicating that the pipe does not exist. - return isPipeExistedBeforeDrop || isSetIfExistsCondition - ? status - : RpcUtils.getStatus( - TSStatusCode.PIPE_NOT_EXIST_ERROR, - String.format( - "Failed to drop pipe %s. Failures: %s does not exist.", pipeName, pipeName)); + return status; } public TShowPipeResp showPipes(final TShowPipeReq req) { try { - final String sqlDialect = - req.isSetSqlDialect() ? req.getSqlDialect() : SystemConstant.SQL_DIALECT_TREE_VALUE; + final String sqlDialect = req.getSqlDialect(); return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) .filter(req.whereClause, req.pipeName, sqlDialect) .convertToTShowPipeResp(); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 920b484778f0..41ad81a45f20 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -37,7 +37,6 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleLeaderChangePlan; @@ -336,7 +335,12 @@ private void checkBeforeDropPipeInternal(final String pipeName) { } public boolean isPipeExisted(final String pipeName) { - return isPipeExisted(pipeName, SystemConstant.SQL_DIALECT_TREE_VALUE); + acquireReadLock(); + try { + return pipeMetaKeeper.containsPipeMeta(pipeName); + } finally { + releaseReadLock(); + } } public boolean isPipeExisted(final String pipeName, final String sqlDialect) { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index cb660aec7939..4a2924b3016a 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -64,6 +64,9 @@ public PipeTemporaryMeta getTemporaryMeta() { } public boolean matchSqlDialect(final String sqlDialect) { + if (Objects.isNull(sqlDialect)) { + return true; + } return sqlDialect.equalsIgnoreCase( getStaticMeta() .getExtractorParameters() From a023046bb7a099400d179377483164f5251ef908 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 16:33:07 +0800 Subject: [PATCH 05/13] apply code review --- .../it/tablemodel/IoTDBPipeIsolationIT.java | 56 ++++++------------- .../response/pipe/task/PipeTableResp.java | 4 +- .../coordinator/task/PipeTaskCoordinator.java | 20 +++---- .../persistence/pipe/PipeTaskInfo.java | 6 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 16 +----- .../config/TableConfigTaskVisitor.java | 19 ------- .../config/TreeConfigTaskVisitor.java | 12 ---- .../executor/ClusterConfigTaskExecutor.java | 10 ++-- .../config/sys/pipe/AlterPipeTask.java | 2 +- .../config/sys/pipe/DropPipeTask.java | 2 +- .../config/sys/pipe/ShowPipeTask.java | 2 +- .../config/sys/pipe/StartPipeTask.java | 2 +- .../config/sys/pipe/StopPipeTask.java | 2 +- .../plan/relational/sql/ast/AlterPipe.java | 16 +----- .../plan/relational/sql/ast/DropPipe.java | 13 +---- .../plan/relational/sql/ast/ShowPipes.java | 16 +----- .../plan/relational/sql/ast/StartPipe.java | 16 +----- .../plan/relational/sql/ast/StopPipe.java | 16 +----- .../metadata/pipe/AlterPipeStatement.java | 10 ++-- .../metadata/pipe/DropPipeStatement.java | 18 +++--- .../metadata/pipe/ShowPipesStatement.java | 10 ++-- .../metadata/pipe/StartPipeStatement.java | 10 ++-- .../metadata/pipe/StopPipeStatement.java | 10 ++-- .../pipe/agent/task/meta/PipeMeta.java | 19 +++++-- .../pipe/config/constant/SystemConstant.java | 6 ++ .../src/main/thrift/confignode.thrift | 10 ++-- 26 files changed, 110 insertions(+), 213 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index c6e888353d8f..c9bee8cf9560 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -98,53 +98,39 @@ public void testWritePipeIsolation() throws Exception { Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client - .startPipeExtended( - new TStartPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(true)) .getCode()); Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client - .startPipeExtended( - new TStartPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client - .startPipeExtended( - new TStartPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .startPipeExtended(new TStartPipeReq(treePipeName).setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client - .startPipeExtended( - new TStartPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(true)) .getCode()); // stop pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), - client - .stopPipeExtended( - new TStopPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) - .getCode()); + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(true)).getCode()); Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client - .stopPipeExtended( - new TStopPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .stopPipeExtended( - new TStopPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) - .getCode()); + client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(false)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .stopPipeExtended( - new TStopPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) - .getCode()); + client.stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(true)).getCode()); // alter pipe Assert.assertEquals( @@ -159,7 +145,7 @@ public void testWritePipeIsolation() throws Exception { false) .setExtractorAttributes(Collections.emptyMap()) .setIsReplaceAllExtractorAttributes(false) - .setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .setIsTableModel(true)) .getCode()); Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), @@ -173,7 +159,7 @@ public void testWritePipeIsolation() throws Exception { false) .setExtractorAttributes(Collections.emptyMap()) .setIsReplaceAllExtractorAttributes(false) - .setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), @@ -187,7 +173,7 @@ public void testWritePipeIsolation() throws Exception { false) .setExtractorAttributes(Collections.emptyMap()) .setIsReplaceAllExtractorAttributes(false) - .setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), @@ -201,34 +187,24 @@ public void testWritePipeIsolation() throws Exception { false) .setExtractorAttributes(Collections.emptyMap()) .setIsReplaceAllExtractorAttributes(false) - .setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) + .setIsTableModel(true)) .getCode()); // drop pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), - client - .dropPipeExtended( - new TDropPipeReq(treePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) - .getCode()); + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client - .dropPipeExtended( - new TDropPipeReq(tablePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) .getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .dropPipeExtended( - new TDropPipeReq(treePipeName).setSqlDialect(BaseEnv.TREE_SQL_DIALECT)) - .getCode()); + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(false)).getCode()); Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .dropPipeExtended( - new TDropPipeReq(tablePipeName).setSqlDialect(BaseEnv.TABLE_SQL_DIALECT)) - .getCode()); + client.dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(true)).getCode()); } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 0e658aeeee45..14174dcda475 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -95,9 +95,9 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { } public PipeTableResp filter( - final Boolean whereClause, final String pipeName, final String sqlDialect) { + final Boolean whereClause, final String pipeName, final boolean isTableModel) { final PipeTableResp resp = filter(whereClause, pipeName); - resp.allPipeMeta.removeIf(meta -> !meta.matchSqlDialect(sqlDialect)); + resp.allPipeMeta.removeIf(meta -> !meta.matchSqlDialect(isTableModel)); return resp; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index 292e5319865f..b7882b6814e7 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -133,10 +133,10 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = req.getSqlDialect(); + final boolean isTableModel = req.isIsTableModel(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -168,8 +168,8 @@ public TSStatus startPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus startPipe(TStartPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = req.getSqlDialect(); - if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + final boolean isTableModel = req.isIsTableModel(); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( @@ -206,8 +206,8 @@ public TSStatus stopPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(TStopPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = req.getSqlDialect(); - if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + final boolean isTableModel = req.isIsTableModel(); + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( @@ -219,10 +219,10 @@ public TSStatus stopPipe(TStopPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final String sqlDialect = req.getSqlDialect(); + final boolean isTableModel = req.isIsTableModel(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, sqlDialect)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -244,9 +244,9 @@ public TSStatus dropPipe(TDropPipeReq req) { public TShowPipeResp showPipes(final TShowPipeReq req) { try { - final String sqlDialect = req.getSqlDialect(); + final boolean isTableModel = req.isIsTableModel(); return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName, sqlDialect) + .filter(req.whereClause, req.pipeName, isTableModel) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 41ad81a45f20..41e6f815d153 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -37,6 +37,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant; +import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleLeaderChangePlan; @@ -343,10 +344,11 @@ public boolean isPipeExisted(final String pipeName) { } } - public boolean isPipeExisted(final String pipeName, final String sqlDialect) { + public boolean isPipeExisted(final String pipeName, final boolean isTableModel) { acquireReadLock(); try { - return pipeMetaKeeper.containsPipeMeta(pipeName, sqlDialect); + return pipeMetaKeeper.containsPipeMeta( + pipeName, SystemConstant.fromIsTableModel(isTableModel)); } finally { releaseReadLock(); } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java index d2c2ffa0b0e0..96811bd754cf 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/service/thrift/ConfigNodeRPCServiceProcessor.java @@ -37,7 +37,6 @@ import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.path.PathPatternTree; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.utils.AuthUtils; import org.apache.iotdb.commons.utils.StatusUtils; @@ -1113,10 +1112,7 @@ public TSStatus alterPipe(TAlterPipeReq req) { @Override public TSStatus startPipe(String pipeName) { - return configManager.startPipe( - new TStartPipeReq() - .setPipeName(pipeName) - .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); + return configManager.startPipe(new TStartPipeReq().setPipeName(pipeName)); } @Override @@ -1126,10 +1122,7 @@ public TSStatus startPipeExtended(TStartPipeReq req) { @Override public TSStatus stopPipe(String pipeName) { - return configManager.stopPipe( - new TStopPipeReq() - .setPipeName(pipeName) - .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); + return configManager.stopPipe(new TStopPipeReq().setPipeName(pipeName)); } @Override @@ -1140,10 +1133,7 @@ public TSStatus stopPipeExtended(TStopPipeReq req) { @Override public TSStatus dropPipe(String pipeName) { return configManager.dropPipe( - new TDropPipeReq() - .setPipeName(pipeName) - .setIfExistsCondition(false) - .setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE)); + new TDropPipeReq().setPipeName(pipeName).setIfExistsCondition(false)); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java index 9fe76a32d519..5c6178830c32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TableConfigTaskVisitor.java @@ -721,49 +721,30 @@ protected IConfigTask visitAlterPipe(AlterPipe node, MPPQueryContext context) { .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TABLE_VALUE); } - // Inject table model into the node - node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); - return new AlterPipeTask(node); } @Override protected IConfigTask visitDropPipe(DropPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - - // Inject table model into the node - node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); - return new DropPipeTask(node); } @Override protected IConfigTask visitStartPipe(StartPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - - // Inject table model into the node - node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); - return new StartPipeTask(node); } @Override protected IConfigTask visitStopPipe(StopPipe node, MPPQueryContext context) { context.setQueryType(QueryType.WRITE); - - // Inject table model into the node - node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); - return new StopPipeTask(node); } @Override protected IConfigTask visitShowPipes(ShowPipes node, MPPQueryContext context) { context.setQueryType(QueryType.READ); - - // Inject table model into the node - node.setSqlDialect(SystemConstant.SQL_DIALECT_TABLE_VALUE); - return new ShowPipeTask(node); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java index 9ae536375edc..5b1e513089c8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/TreeConfigTaskVisitor.java @@ -453,9 +453,6 @@ public IConfigTask visitShowAINodes( @Override public IConfigTask visitShowPipes( ShowPipesStatement showPipesStatement, MPPQueryContext context) { - // Inject tree model into the statement - showPipesStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); - return new ShowPipeTask(showPipesStatement); } @@ -505,26 +502,17 @@ public IConfigTask visitAlterPipe( .put(SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE); } - // Inject tree model into the statement - alterPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); - return new AlterPipeTask(alterPipeStatement); } @Override public IConfigTask visitStartPipe( StartPipeStatement startPipeStatement, MPPQueryContext context) { - // Inject tree model into the statement - startPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); - return new StartPipeTask(startPipeStatement); } @Override public IConfigTask visitStopPipe(StopPipeStatement stopPipeStatement, MPPQueryContext context) { - // Inject tree model into the statement - stopPipeStatement.setSqlDialect(SystemConstant.SQL_DIALECT_TREE_VALUE); - return new StopPipeTask(stopPipeStatement); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java index 3c3c125b6187..c26f25c10093 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/executor/ClusterConfigTaskExecutor.java @@ -2041,7 +2041,7 @@ public SettableFuture alterPipe(final AlterPipeStatement alter req.setExtractorAttributes(alterPipeStatement.getExtractorAttributes()); req.setIsReplaceAllExtractorAttributes(alterPipeStatement.isReplaceAllExtractorAttributes()); req.setIfExistsCondition(alterPipeStatement.hasIfExistsCondition()); - req.setSqlDialect(alterPipeStatement.getSqlDialect()); + req.setIsTableModel(alterPipeStatement.isTableModel()); final TSStatus tsStatus = configNodeClient.alterPipe(req); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); @@ -2075,7 +2075,7 @@ public SettableFuture startPipe(final StartPipeStatement start configNodeClient.startPipeExtended( new TStartPipeReq() .setPipeName(startPipeStatement.getPipeName()) - .setSqlDialect(startPipeStatement.getSqlDialect())); + .setIsTableModel(startPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2109,7 +2109,7 @@ public SettableFuture dropPipe(final DropPipeStatement dropPip new TDropPipeReq() .setPipeName(dropPipeStatement.getPipeName()) .setIfExistsCondition(dropPipeStatement.hasIfExistsCondition()) - .setSqlDialect(dropPipeStatement.getSqlDialect())); + .setIsTableModel(dropPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2143,7 +2143,7 @@ public SettableFuture stopPipe(final StopPipeStatement stopPip configNodeClient.stopPipeExtended( new TStopPipeReq() .setPipeName(stopPipeStatement.getPipeName()) - .setSqlDialect(stopPipeStatement.getSqlDialect())); + .setIsTableModel(stopPipeStatement.isTableModel())); if (TSStatusCode.SUCCESS_STATUS.getStatusCode() != tsStatus.getCode()) { future.setException(new IoTDBException(tsStatus.message, tsStatus.code)); } else { @@ -2167,7 +2167,7 @@ public SettableFuture showPipes(final ShowPipesStatement showP if (showPipesStatement.getWhereClause()) { tShowPipeReq.setWhereClause(true); } - tShowPipeReq.setSqlDialect(showPipesStatement.getSqlDialect()); + tShowPipeReq.setIsTableModel(showPipesStatement.isTableModel()); final List tShowPipeInfoList = configNodeClient.showPipe(tShowPipeReq).getPipeInfoList(); ShowPipeTask.buildTSBlock(tShowPipeInfoList, future); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java index fba64b3bb67f..159c6941628e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/AlterPipeTask.java @@ -58,7 +58,7 @@ public AlterPipeTask(final AlterPipe node) { alterPipeStatement.setReplaceAllProcessorAttributes(node.isReplaceAllProcessorAttributes()); alterPipeStatement.setReplaceAllConnectorAttributes(node.isReplaceAllConnectorAttributes()); - alterPipeStatement.setSqlDialect(node.getSqlDialect()); + alterPipeStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java index 4bc32d64c005..a10c6042241c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/DropPipeTask.java @@ -40,7 +40,7 @@ public DropPipeTask(final DropPipe node) { dropPipeStatement = new DropPipeStatement(StatementType.DROP_PIPE); dropPipeStatement.setPipeName(node.getPipeName()); dropPipeStatement.setIfExists(node.hasIfExistsCondition()); - dropPipeStatement.setSqlDialect(node.getSqlDialect()); + dropPipeStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java index d62631e794c5..c43a9953b584 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/ShowPipeTask.java @@ -56,7 +56,7 @@ public ShowPipeTask(final ShowPipes node) { showPipesStatement = new ShowPipesStatement(); showPipesStatement.setPipeName(node.getPipeName()); showPipesStatement.setWhereClause(node.hasWhereClause()); - showPipesStatement.setSqlDialect(node.getSqlDialect()); + showPipesStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java index e1de98b198f7..c118f0a3a58a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StartPipeTask.java @@ -39,7 +39,7 @@ public StartPipeTask(final StartPipeStatement startPipeStatement) { public StartPipeTask(final StartPipe node) { startPipeStatement = new StartPipeStatement(StatementType.START_PIPE); startPipeStatement.setPipeName(node.getPipeName()); - startPipeStatement.setSqlDialect(node.getSqlDialect()); + startPipeStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java index 07918c37c541..91764c43263d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/execution/config/sys/pipe/StopPipeTask.java @@ -39,7 +39,7 @@ public StopPipeTask(final StopPipeStatement stopPipeStatement) { public StopPipeTask(final StopPipe node) { stopPipeStatement = new StopPipeStatement(StatementType.STOP_PIPE); stopPipeStatement.setPipeName(node.getPipeName()); - stopPipeStatement.setSqlDialect(node.getSqlDialect()); + stopPipeStatement.setTableModel(true); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java index 36c560d2dde6..a40fab917a51 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/AlterPipe.java @@ -35,7 +35,6 @@ public class AlterPipe extends PipeStatement { private final boolean isReplaceAllExtractorAttributes; private final boolean isReplaceAllProcessorAttributes; private final boolean isReplaceAllConnectorAttributes; - private String sqlDialect; public AlterPipe( final String pipeName, @@ -56,10 +55,6 @@ public AlterPipe( this.isReplaceAllConnectorAttributes = isReplaceAllConnectorAttributes; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); - } - public String getPipeName() { return pipeName; } @@ -92,10 +87,6 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } - public String getSqlDialect() { - return sqlDialect; - } - @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitAlterPipe(this, context); @@ -111,8 +102,7 @@ public int hashCode() { connectorAttributes, isReplaceAllExtractorAttributes, isReplaceAllProcessorAttributes, - isReplaceAllConnectorAttributes, - sqlDialect); + isReplaceAllConnectorAttributes); } @Override @@ -134,8 +124,7 @@ public boolean equals(final Object obj) { && Objects.equals( this.isReplaceAllProcessorAttributes, that.isReplaceAllProcessorAttributes) && Objects.equals( - this.isReplaceAllConnectorAttributes, that.isReplaceAllConnectorAttributes) - && Objects.equals(this.sqlDialect, that.sqlDialect); + this.isReplaceAllConnectorAttributes, that.isReplaceAllConnectorAttributes); } @Override @@ -149,7 +138,6 @@ public String toString() { .add("isReplaceAllExtractorAttributes", isReplaceAllExtractorAttributes) .add("isReplaceAllProcessorAttributes", isReplaceAllProcessorAttributes) .add("isReplaceAllConnectorAttributes", isReplaceAllConnectorAttributes) - .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java index 9fa4a89a8cb6..1b3837ba2147 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/DropPipe.java @@ -28,17 +28,12 @@ public class DropPipe extends PipeStatement { private final String pipeName; private final boolean ifExistsCondition; - private String sqlDialect; public DropPipe(final String pipeName, final boolean ifExistsCondition) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); this.ifExistsCondition = ifExistsCondition; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); - } - public String getPipeName() { return pipeName; } @@ -47,10 +42,6 @@ public boolean hasIfExistsCondition() { return ifExistsCondition; } - public String getSqlDialect() { - return sqlDialect; - } - @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitDropPipe(this, context); @@ -71,8 +62,7 @@ public boolean equals(final Object obj) { } final DropPipe that = (DropPipe) obj; return Objects.equals(this.pipeName, that.pipeName) - && Objects.equals(this.ifExistsCondition, that.ifExistsCondition) - && Objects.equals(this.sqlDialect, that.sqlDialect); + && Objects.equals(this.ifExistsCondition, that.ifExistsCondition); } @Override @@ -80,7 +70,6 @@ public String toString() { return toStringHelper(this) .add("pipeName", pipeName) .add("ifExistsCondition", ifExistsCondition) - .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java index 72b816676272..c0dfe9379de1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/ShowPipes.java @@ -24,23 +24,17 @@ import java.util.Objects; import static com.google.common.base.MoreObjects.toStringHelper; -import static java.util.Objects.requireNonNull; public class ShowPipes extends PipeStatement { private final String pipeName; private final boolean hasWhereClause; - private String sqlDialect; public ShowPipes(final @Nullable String pipeName, final boolean hasWhereClause) { this.pipeName = pipeName; this.hasWhereClause = hasWhereClause; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); - } - public String getPipeName() { return pipeName; } @@ -49,10 +43,6 @@ public boolean hasWhereClause() { return hasWhereClause; } - public String getSqlDialect() { - return sqlDialect; - } - @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitShowPipes(this, context); @@ -60,7 +50,7 @@ public R accept(final AstVisitor visitor, final C context) { @Override public int hashCode() { - return Objects.hash(pipeName, hasWhereClause, sqlDialect); + return Objects.hash(pipeName, hasWhereClause); } @Override @@ -73,8 +63,7 @@ public boolean equals(final Object obj) { } final ShowPipes that = (ShowPipes) obj; return Objects.equals(this.pipeName, that.pipeName) - && Objects.equals(this.hasWhereClause, that.hasWhereClause) - && Objects.equals(this.sqlDialect, that.sqlDialect); + && Objects.equals(this.hasWhereClause, that.hasWhereClause); } @Override @@ -82,7 +71,6 @@ public String toString() { return toStringHelper(this) .add("pipeName", pipeName) .add("hasWhereClause", hasWhereClause) - .add("sqlDialect", sqlDialect) .toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java index 743f71b473b8..15eb5b2413dd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StartPipe.java @@ -27,24 +27,15 @@ public class StartPipe extends PipeStatement { private final String pipeName; - private String sqlDialect; public StartPipe(final String pipeName) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); - } - public String getPipeName() { return pipeName; } - public String getSqlDialect() { - return sqlDialect; - } - @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitStartPipe(this, context); @@ -52,7 +43,7 @@ public R accept(final AstVisitor visitor, final C context) { @Override public int hashCode() { - return Objects.hash(pipeName, sqlDialect); + return Objects.hash(pipeName); } @Override @@ -64,12 +55,11 @@ public boolean equals(final Object obj) { return false; } final StartPipe that = (StartPipe) obj; - return Objects.equals(this.pipeName, that.pipeName) - && Objects.equals(this.sqlDialect, that.sqlDialect); + return Objects.equals(this.pipeName, that.pipeName); } @Override public String toString() { - return toStringHelper(this).add("pipeName", pipeName).add("sqlDialect", sqlDialect).toString(); + return toStringHelper(this).add("pipeName", pipeName).toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java index 418c2d592b1f..6d1a629de882 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/StopPipe.java @@ -27,24 +27,15 @@ public class StopPipe extends PipeStatement { private final String pipeName; - private String sqlDialect; public StopPipe(final String pipeName) { this.pipeName = requireNonNull(pipeName, "pipe name can not be null"); } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = requireNonNull(sqlDialect, "sql dialect can not be null"); - } - public String getPipeName() { return pipeName; } - public String getSqlDialect() { - return sqlDialect; - } - @Override public R accept(final AstVisitor visitor, final C context) { return visitor.visitStopPipe(this, context); @@ -52,7 +43,7 @@ public R accept(final AstVisitor visitor, final C context) { @Override public int hashCode() { - return Objects.hash(pipeName, sqlDialect); + return Objects.hash(pipeName); } @Override @@ -64,12 +55,11 @@ public boolean equals(final Object obj) { return false; } final StopPipe that = (StopPipe) obj; - return Objects.equals(this.pipeName, that.pipeName) - && Objects.equals(this.sqlDialect, that.sqlDialect); + return Objects.equals(this.pipeName, that.pipeName); } @Override public String toString() { - return toStringHelper(this).add("pipeName", pipeName).add("sqlDialect", sqlDialect).toString(); + return toStringHelper(this).add("pipeName", pipeName).toString(); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java index 6ec4f0ae9cee..44065cf31850 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/AlterPipeStatement.java @@ -44,7 +44,7 @@ public class AlterPipeStatement extends Statement implements IConfigStatement { private boolean isReplaceAllExtractorAttributes; private boolean isReplaceAllProcessorAttributes; private boolean isReplaceAllConnectorAttributes; - private String sqlDialect; + private boolean isTableModel; public AlterPipeStatement(final StatementType alterPipeStatement) { this.statementType = alterPipeStatement; @@ -82,8 +82,8 @@ public boolean isReplaceAllConnectorAttributes() { return isReplaceAllConnectorAttributes; } - public String getSqlDialect() { - return sqlDialect; + public boolean isTableModel() { + return isTableModel; } public void setPipeName(final String pipeName) { @@ -118,8 +118,8 @@ public void setReplaceAllConnectorAttributes(final boolean replaceAllConnectorAt isReplaceAllConnectorAttributes = replaceAllConnectorAttributes; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = sqlDialect; + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java index 49c7cc285b83..c61a15d777d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/DropPipeStatement.java @@ -37,22 +37,22 @@ public class DropPipeStatement extends Statement implements IConfigStatement { private String pipeName; private boolean ifExistsCondition; - private String sqlDialect; + private boolean isTableModel; public DropPipeStatement(final StatementType dropPipeStatement) { this.statementType = dropPipeStatement; } - public boolean hasIfExistsCondition() { - return ifExistsCondition; - } - public String getPipeName() { return pipeName; } - public String getSqlDialect() { - return sqlDialect; + public boolean hasIfExistsCondition() { + return ifExistsCondition; + } + + public boolean isTableModel() { + return isTableModel; } public void setPipeName(final String pipeName) { @@ -63,8 +63,8 @@ public void setIfExists(final boolean ifExistsCondition) { this.ifExistsCondition = ifExistsCondition; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = sqlDialect; + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java index 2e7c809bae5d..658f36a0635f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/ShowPipesStatement.java @@ -39,7 +39,7 @@ public ShowPipesStatement() { private boolean whereClause; - private String sqlDialect; + private boolean isTableModel; public String getPipeName() { return pipeName; @@ -49,8 +49,8 @@ public boolean getWhereClause() { return whereClause; } - public String getSqlDialect() { - return sqlDialect; + public boolean isTableModel() { + return isTableModel; } public void setPipeName(final String pipeName) { @@ -61,8 +61,8 @@ public void setWhereClause(final boolean whereClause) { this.whereClause = whereClause; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = sqlDialect; + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java index cfcc6c2ede51..9023d77c06ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StartPipeStatement.java @@ -36,7 +36,7 @@ public class StartPipeStatement extends Statement implements IConfigStatement { private String pipeName; - private String sqlDialect; + private boolean isTableModel; public StartPipeStatement(final StatementType startPipeStatement) { this.statementType = startPipeStatement; @@ -46,16 +46,16 @@ public String getPipeName() { return pipeName; } - public String getSqlDialect() { - return sqlDialect; + public boolean isTableModel() { + return isTableModel; } public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = sqlDialect; + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java index 89e68f8daf4b..ea14a3dbd1b2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/statement/metadata/pipe/StopPipeStatement.java @@ -36,7 +36,7 @@ public class StopPipeStatement extends Statement implements IConfigStatement { private String pipeName; - private String sqlDialect; + private boolean isTableModel; public StopPipeStatement(final StatementType stopPipeStatement) { this.statementType = stopPipeStatement; @@ -46,16 +46,16 @@ public String getPipeName() { return pipeName; } - public String getSqlDialect() { - return sqlDialect; + public boolean isTableModel() { + return isTableModel; } public void setPipeName(final String pipeName) { this.pipeName = pipeName; } - public void setSqlDialect(final String sqlDialect) { - this.sqlDialect = sqlDialect; + public void setTableModel(final boolean tableModel) { + this.isTableModel = tableModel; } @Override diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 4a2924b3016a..e30d77424080 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -63,15 +63,24 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } + public boolean matchSqlDialect(final boolean isTableModel) { + return matchSqlDialect(SystemConstant.fromIsTableModel(isTableModel)); + } + public boolean matchSqlDialect(final String sqlDialect) { if (Objects.isNull(sqlDialect)) { return true; } - return sqlDialect.equalsIgnoreCase( - getStaticMeta() - .getExtractorParameters() - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE)); + return isDoubleLiving() + || sqlDialect.equalsIgnoreCase( + getStaticMeta() + .getExtractorParameters() + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE)); + } + + private boolean isDoubleLiving() { + return false; } public ByteBuffer serialize() throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java index 105390eab9ca..236bca06fb70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java @@ -52,6 +52,12 @@ public static PipeParameters addSystemKeysIfNecessary(final PipeParameters given return new PipeParameters(attributes); } + public static String fromIsTableModel(final boolean isTableModel) { + return isTableModel + ? SystemConstant.SQL_DIALECT_TABLE_VALUE + : SystemConstant.SQL_DIALECT_TREE_VALUE; + } + /////////////////////////////////// Private Constructor /////////////////////////////////// private SystemConstant() { diff --git a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index 8e7b6ff5a8a5..b15878cead43 100644 --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@ -750,23 +750,23 @@ struct TAlterPipeReq { 6: optional map extractorAttributes 7: optional bool isReplaceAllExtractorAttributes 8: optional bool ifExistsCondition - 9: optional string sqlDialect + 9: optional bool isTableModel } struct TStartPipeReq { 1: required string pipeName - 2: optional string sqlDialect + 2: optional bool isTableModel } struct TStopPipeReq { 1: required string pipeName - 2: optional string sqlDialect + 2: optional bool isTableModel } struct TDropPipeReq { 1: required string pipeName 2: optional bool ifExistsCondition - 3: optional string sqlDialect + 3: optional bool isTableModel } // Deprecated, restored for compatibility @@ -779,7 +779,7 @@ struct TPipeSinkInfo { struct TShowPipeReq { 1: optional string pipeName 2: optional bool whereClause - 3: optional string sqlDialect + 3: optional bool isTableModel } struct TShowPipeResp { From fdb2d4a48e563db4cbc61e18d7e3a0b5c4839542 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 16:35:31 +0800 Subject: [PATCH 06/13] minor improve --- .../coordinator/task/PipeTaskCoordinator.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index b7882b6814e7..da94f98527da 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -133,10 +133,9 @@ public TSStatus createPipe(TCreatePipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus alterPipe(TAlterPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isTableModel = req.isIsTableModel(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -168,8 +167,7 @@ public TSStatus startPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus startPipe(TStartPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isTableModel = req.isIsTableModel(); - if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( @@ -206,8 +204,7 @@ public TSStatus stopPipe(String pipeName) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus stopPipe(TStopPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isTableModel = req.isIsTableModel(); - if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { return RpcUtils.getStatus( TSStatusCode.PIPE_NOT_EXIST_ERROR, String.format( @@ -219,10 +216,9 @@ public TSStatus stopPipe(TStopPipeReq req) { /** Caller should ensure that the method is called in the lock {@link #lock()}. */ public TSStatus dropPipe(TDropPipeReq req) { final String pipeName = req.getPipeName(); - final boolean isTableModel = req.isIsTableModel(); final boolean isSetIfExistsCondition = req.isSetIfExistsCondition() && req.isIfExistsCondition(); - if (!pipeTaskInfo.isPipeExisted(pipeName, isTableModel)) { + if (!pipeTaskInfo.isPipeExisted(pipeName, req.isTableModel)) { return isSetIfExistsCondition ? RpcUtils.getStatus(TSStatusCode.SUCCESS_STATUS) : RpcUtils.getStatus( @@ -244,9 +240,8 @@ public TSStatus dropPipe(TDropPipeReq req) { public TShowPipeResp showPipes(final TShowPipeReq req) { try { - final boolean isTableModel = req.isIsTableModel(); return ((PipeTableResp) configManager.getConsensusManager().read(new ShowPipePlanV2())) - .filter(req.whereClause, req.pipeName, isTableModel) + .filter(req.whereClause, req.pipeName, req.isTableModel) .convertToTShowPipeResp(); } catch (final ConsensusException e) { LOGGER.warn("Failed in the read API executing the consensus layer due to: ", e); From 83fcc9d075cd86fdbbb18ce8a51eedb1af5e1c0f Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 17:11:28 +0800 Subject: [PATCH 07/13] basic support double living --- .../it/tablemodel/IoTDBPipeIsolationIT.java | 125 +++++++++++++++++- .../persistence/pipe/PipeTaskInfo.java | 2 +- .../pipe/agent/task/meta/PipeMeta.java | 20 ++- .../pipe/config/constant/SystemConstant.java | 2 +- 4 files changed, 141 insertions(+), 8 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index c9bee8cf9560..d1fc36fed1a0 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -58,6 +58,7 @@ public void testWritePipeIsolation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + // Create tree pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final Map extractorAttributes = new HashMap<>(); @@ -75,6 +76,7 @@ public void testWritePipeIsolation() throws Exception { Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); } + // Create table pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { final Map extractorAttributes = new HashMap<>(); @@ -94,7 +96,7 @@ public void testWritePipeIsolation() throws Exception { try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // start pipe + // Start pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client @@ -116,7 +118,7 @@ public void testWritePipeIsolation() throws Exception { .startPipeExtended(new TStartPipeReq(tablePipeName).setIsTableModel(true)) .getCode()); - // stop pipe + // Stop pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.stopPipeExtended(new TStopPipeReq(treePipeName).setIsTableModel(true)).getCode()); @@ -132,7 +134,7 @@ public void testWritePipeIsolation() throws Exception { TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.stopPipeExtended(new TStopPipeReq(tablePipeName).setIsTableModel(true)).getCode()); - // alter pipe + // Alter pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client @@ -190,7 +192,7 @@ public void testWritePipeIsolation() throws Exception { .setIsTableModel(true)) .getCode()); - // drop pipe + // Drop pipe Assert.assertEquals( TSStatusCode.PIPE_NOT_EXIST_ERROR.getStatusCode(), client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); @@ -289,4 +291,119 @@ public void testReadPipeIsolation() { fail(e.getMessage()); } } + + @Test + public void testDoubleLivingIsolation() throws Exception { + // TODO: consider 'mode.double-living' + + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "tree"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("capture.table", "true"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(treePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Create table pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "table"); + extractorAttributes.put("capture.tree", "true"); + extractorAttributes.put("capture.table", "true"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(tablePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (SQLException e) { + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 41e6f815d153..55ae6ad89e21 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -348,7 +348,7 @@ public boolean isPipeExisted(final String pipeName, final boolean isTableModel) acquireReadLock(); try { return pipeMetaKeeper.containsPipeMeta( - pipeName, SystemConstant.fromIsTableModel(isTableModel)); + pipeName, SystemConstant.fetchSqlDialectValue(isTableModel)); } finally { releaseReadLock(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index e30d77424080..91d70c326612 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -19,6 +19,7 @@ package org.apache.iotdb.commons.pipe.agent.task.meta; +import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.tsfile.utils.PublicBAOS; @@ -28,6 +29,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Objects; public class PipeMeta { @@ -64,7 +66,7 @@ public PipeTemporaryMeta getTemporaryMeta() { } public boolean matchSqlDialect(final boolean isTableModel) { - return matchSqlDialect(SystemConstant.fromIsTableModel(isTableModel)); + return matchSqlDialect(SystemConstant.fetchSqlDialectValue(isTableModel)); } public boolean matchSqlDialect(final String sqlDialect) { @@ -80,7 +82,21 @@ public boolean matchSqlDialect(final String sqlDialect) { } private boolean isDoubleLiving() { - return false; + // TODO: consider 'mode.double-living' + return getStaticMeta() + .getExtractorParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + false) + && getStaticMeta() + .getExtractorParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + false); } public ByteBuffer serialize() throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java index 236bca06fb70..81d3a1be11de 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java @@ -52,7 +52,7 @@ public static PipeParameters addSystemKeysIfNecessary(final PipeParameters given return new PipeParameters(attributes); } - public static String fromIsTableModel(final boolean isTableModel) { + public static String fetchSqlDialectValue(final boolean isTableModel) { return isTableModel ? SystemConstant.SQL_DIALECT_TABLE_VALUE : SystemConstant.SQL_DIALECT_TREE_VALUE; From e3426407527cf3f68ea2b2a8760fd0b05ee2816a Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 18:30:42 +0800 Subject: [PATCH 08/13] support 'mode.double-living' --- .../it/tablemodel/IoTDBPipeIsolationIT.java | 135 ++++++++++++++++-- .../customizer/parameter/PipeParameters.java | 4 + .../pipe/agent/task/meta/PipeMeta.java | 37 +++-- .../constant/PipeExtractorConstant.java | 3 + .../pipe/extractor/IoTDBExtractor.java | 70 +++++++++ 5 files changed, 222 insertions(+), 27 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index d1fc36fed1a0..c4fb40c4b8f8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -224,7 +224,7 @@ public void testReadPipeIsolation() { String.format( "create pipe %s with sink ('node-urls'='%s')", treePipeName, receiverDataNode.getIpAndPortString())); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -237,7 +237,7 @@ public void testReadPipeIsolation() { count++; } Assert.assertEquals(1, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -250,7 +250,7 @@ public void testReadPipeIsolation() { count++; } Assert.assertEquals(0, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -261,7 +261,7 @@ public void testReadPipeIsolation() { String.format( "create pipe %s with sink ('node-urls'='%s')", tablePipeName, receiverDataNode.getIpAndPortString())); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -274,7 +274,7 @@ public void testReadPipeIsolation() { count++; } Assert.assertEquals(1, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -287,15 +287,13 @@ public void testReadPipeIsolation() { count++; } Assert.assertEquals(1, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } } @Test - public void testDoubleLivingIsolation() throws Exception { - // TODO: consider 'mode.double-living' - + public void testCaptureTreeAndTableIsolation() throws Exception { final String treePipeName = "tree_a2b"; final String tablePipeName = "table_a2b"; @@ -330,7 +328,7 @@ public void testDoubleLivingIsolation() throws Exception { count++; } Assert.assertEquals(1, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -343,7 +341,7 @@ public void testDoubleLivingIsolation() throws Exception { count++; } Assert.assertEquals(1, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } @@ -376,7 +374,118 @@ public void testDoubleLivingIsolation() throws Exception { count++; } Assert.assertEquals(2, count); - } catch (SQLException e) { + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } + + @Test + public void testDoubleLivingIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "tree"); + extractorAttributes.put("mode.double-living", "true"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(treePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Create table pipe + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final Map extractorAttributes = new HashMap<>(); + final Map processorAttributes = new HashMap<>(); + final Map connectorAttributes = new HashMap<>(); + + extractorAttributes.put("__system.sql-dialect", "table"); + extractorAttributes.put("mode.double-living", "true"); + connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); + + final TSStatus status = + client.createPipe( + new TCreatePipeReq(tablePipeName, connectorAttributes) + .setExtractorAttributes(extractorAttributes) + .setProcessorAttributes(processorAttributes)); + Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (final SQLException e) { fail(e.getMessage()); } @@ -389,7 +498,7 @@ public void testDoubleLivingIsolation() throws Exception { count++; } Assert.assertEquals(2, count); - } catch (SQLException e) { + } catch (final SQLException e) { fail(e.getMessage()); } diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java index 3dcb2d19b0a7..dbcf8c522d00 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java @@ -75,6 +75,10 @@ public void addAttribute(final String key, String values) { attributes.put(KeyReducer.reduce(key), values); } + public void addRawAttribute(final String key, String values) { + attributes.put(key, values); + } + public String getString(final String key) { final String value = attributes.get(key); return value != null ? value : attributes.get(KeyReducer.reduce(key)); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 91d70c326612..effa04e9684c 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -73,7 +73,7 @@ public boolean matchSqlDialect(final String sqlDialect) { if (Objects.isNull(sqlDialect)) { return true; } - return isDoubleLiving() + return matchBoth() || sqlDialect.equalsIgnoreCase( getStaticMeta() .getExtractorParameters() @@ -81,22 +81,31 @@ public boolean matchSqlDialect(final String sqlDialect) { SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE)); } - private boolean isDoubleLiving() { - // TODO: consider 'mode.double-living' - return getStaticMeta() + private boolean matchBoth() { + return // 1. 'mode.double-living' is set to true + getStaticMeta() .getExtractorParameters() .getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - false) - && getStaticMeta() - .getExtractorParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - false); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE) + || + // 2. 'capture.tree' and 'capture.table' is set to true + (getStaticMeta() + .getExtractorParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + false) + && getStaticMeta() + .getExtractorParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + false)); } public ByteBuffer serialize() throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java index 7522cd444fe0..70a918bf4daf 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/PipeExtractorConstant.java @@ -115,6 +115,9 @@ public class PipeExtractorConstant { public static final String EXTRACTOR_MODE_SNAPSHOT_KEY = "extractor.mode.snapshot"; public static final String SOURCE_MODE_SNAPSHOT_KEY = "source.mode.snapshot"; public static final boolean EXTRACTOR_MODE_SNAPSHOT_DEFAULT_VALUE = false; + public static final String EXTRACTOR_MODE_DOUBLE_LIVING_KEY = "extractor.mode.double-living"; + public static final String SOURCE_MODE_DOUBLE_LIVING_KEY = "source.mode.double-living"; + public static final boolean EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE = false; public static final String EXTRACTOR_START_TIME_KEY = "extractor.start-time"; public static final String SOURCE_START_TIME_KEY = "source.start-time"; diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java index 2d3a1615f00e..04e284375d29 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java @@ -26,8 +26,10 @@ import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameterValidator; import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import java.util.Arrays; +import java.util.Objects; import java.util.concurrent.atomic.AtomicBoolean; import static org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant.EXTRACTOR_EXCLUSION_DEFAULT_VALUE; @@ -85,6 +87,71 @@ public void validate(final PipeParameterValidator validator) throws Exception { .getStringOrDefault( Arrays.asList(EXTRACTOR_EXCLUSION_KEY, SOURCE_EXCLUSION_KEY), EXTRACTOR_EXCLUSION_DEFAULT_VALUE)); + + // Validate double living + // NOTE: this function may modify pipe parameters + validateDoubleLiving(validator.getParameters()); + } + + private void validateDoubleLiving(final PipeParameters parameters) { + final boolean isDoubleLiving = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + // check 'capture.tree' + final Boolean isTreeModelDataAllowedToBeCaptured = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + if (Objects.nonNull(isTreeModelDataAllowedToBeCaptured) + && !isTreeModelDataAllowedToBeCaptured) { + throw new PipeParameterNotValidException( + "capture.tree can not be specified to false when double living is enabled"); + } + + // check 'capture.table' + final Boolean isTableModelDataAllowedToBeCaptured = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + if (Objects.nonNull(isTableModelDataAllowedToBeCaptured) + && !isTableModelDataAllowedToBeCaptured) { + throw new PipeParameterNotValidException( + "capture.table can not be specified to false when double living is enabled"); + } + + // check 'forwarding-pipe-requests' + final Boolean isForwardingPipeRequests = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY); + if (Objects.nonNull(isForwardingPipeRequests) && isForwardingPipeRequests) { + throw new PipeParameterNotValidException( + "forwarding-pipe-requests can not be specified to true when double living is enabled"); + } + + // modify pipe parameters + applyDoubleLiving(parameters); + } + } + + private void applyDoubleLiving(final PipeParameters parameters) { + // temporarily set 'capture.tree' = 'true' + parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); + parameters.addRawAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); + parameters.addRawAttribute(PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY, "true"); + // temporarily set 'capture.table' = 'true' + parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); + parameters.addRawAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); + parameters.addRawAttribute(PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY, "true"); + // temporarily set 'forwarding-pipe-requests' = 'false' + parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, "false"); + parameters.addRawAttribute( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, "false"); + parameters.addRawAttribute(PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY, "false"); } @Override @@ -99,6 +166,9 @@ public void customize( taskID = pipeName + "_" + regionId + "_" + creationTime; pipeTaskMeta = environment.getPipeTaskMeta(); + // apply double living before setting isForwardingPipeRequests + applyDoubleLiving(parameters); + isForwardingPipeRequests = parameters.getBooleanOrDefault( Arrays.asList( From 14cadf443aee99e3e895f97bf853c405c4a35c7c Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 18:36:08 +0800 Subject: [PATCH 09/13] fix IT --- .../pipe/it/tablemodel/IoTDBPipeAlterIT.java | 30 ++++++++++++------- .../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 30 ++++++++++++------- 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java index a81b02f7dd73..1665f9d2ce2b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java @@ -65,7 +65,8 @@ public void testBasicAlterPipe() throws Exception { long lastCreationTime; try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -96,7 +97,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -114,7 +116,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -149,7 +152,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -183,7 +187,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("STOPPED", showPipeResult.get(0).state); @@ -225,7 +230,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -257,7 +263,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -289,7 +296,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -323,7 +331,8 @@ public void testBasicAlterPipe() throws Exception { // show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); @@ -354,7 +363,8 @@ public void testBasicAlterPipe() throws Exception { // Show pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); // Check status Assert.assertEquals("RUNNING", showPipeResult.get(0).state); diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java index 24b1c92563b6..c618ead5a1b8 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java @@ -89,7 +89,8 @@ public void testValidPipeName() throws Exception { } } - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; for (final String pipeName : expectedPipeNames) { Assert.assertTrue( showPipeResult.stream() @@ -106,7 +107,7 @@ public void testValidPipeName() throws Exception { } } - showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + showPipeResult = client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -177,7 +178,8 @@ public void testRevertStageOrder() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(0, showPipeResult.size()); } } @@ -249,7 +251,8 @@ public void testMissingStage() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -315,7 +318,8 @@ public void testInvalidParameter() throws Exception { } catch (SQLException ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } @@ -522,7 +526,8 @@ public void testBrackets() throws Exception { } catch (Exception ignored) { } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(2, showPipeResult.size()); } } @@ -578,10 +583,12 @@ public void testShowPipeWithWrongPipeName() throws Exception { .setProcessorAttributes(processorAttributes)); Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(3, showPipeResult.size()); - showPipeResult = client.showPipe(new TShowPipeReq().setPipeName("p1")).pipeInfoList; + showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true).setPipeName("p1")).pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -589,7 +596,9 @@ public void testShowPipeWithWrongPipeName() throws Exception { // Show all pipes whose connector is also used by p1. // p1 and p2 share the same connector parameters, so they have the same connector. showPipeResult = - client.showPipe(new TShowPipeReq().setPipeName("p1").setWhereClause(true)).pipeInfoList; + client.showPipe( + new TShowPipeReq().setIsTableModel(true).setPipeName("p1").setWhereClause(true)) + .pipeInfoList; Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p1"))); Assert.assertTrue(showPipeResult.stream().anyMatch((o) -> o.id.equals("p2"))); Assert.assertFalse(showPipeResult.stream().anyMatch((o) -> o.id.equals("p3"))); @@ -678,7 +687,8 @@ public void testInclusionPattern() throws Exception { fail(e.getMessage()); } - final List showPipeResult = client.showPipe(new TShowPipeReq()).pipeInfoList; + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; Assert.assertEquals(1, showPipeResult.size()); } } From ccf64de276f2ada096d4c7d5a6c7a4c0c50ebf97 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Tue, 17 Dec 2024 21:09:54 +0800 Subject: [PATCH 10/13] improve double living & add IT --- .../tablemodel/IoTDBPipeDoubleLivingIT.java | 370 ++++++++++++++++++ .../it/tablemodel/IoTDBPipeIsolationIT.java | 213 ++-------- .../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 3 +- .../dataregion/IoTDBDataRegionExtractor.java | 39 +- .../datastructure/pattern/TablePattern.java | 21 +- .../datastructure/pattern/TreePattern.java | 22 +- .../pipe/extractor/IoTDBExtractor.java | 105 +++-- 7 files changed, 513 insertions(+), 260 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java new file mode 100644 index 000000000000..3f76bcf63990 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java @@ -0,0 +1,370 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.pipe.it.tablemodel; + +import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; +import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeInfo; +import org.apache.iotdb.confignode.rpc.thrift.TShowPipeReq; +import org.apache.iotdb.db.it.utils.TestUtils; +import org.apache.iotdb.it.env.cluster.node.DataNodeWrapper; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.MultiClusterIT2TableModel; +import org.apache.iotdb.itbase.env.BaseEnv; +import org.apache.iotdb.rpc.TSStatusCode; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({MultiClusterIT2TableModel.class}) +public class IoTDBPipeDoubleLivingIT extends AbstractPipeTableModelTestIT { + + @Test + public void testDoubleLivingInvalidParameter() throws Exception { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.table'='false'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'forwarding-pipe-requests'='true'," + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + final List showPipeResult = + client.showPipe(new TShowPipeReq().setIsTableModel(true)).pipeInfoList; + Assert.assertEquals(0, showPipeResult.size()); + } + } + + // combination of org.apache.iotdb.pipe.it.tablemodel.IoTDBPipeLifeCycleIT.testDoubleLiving and + // org.apache.iotdb.pipe.it.autocreate.IoTDBPipeLifeCycleIT.testDoubleLiving + @Test + public void testBasicDoubleLiving() { + boolean insertResult; + + final DataNodeWrapper senderDataNode = senderEnv.getDataNodeWrapper(0); + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + final Consumer handleFailure = + o -> { + TestUtils.executeNonQueryWithRetry(receiverEnv, "flush"); + TestUtils.executeNonQueryWithRetry(senderEnv, "flush"); + }; + + // insertion on sender + for (int i = 0; i < 100; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + TableModelUtils.createDataBaseAndTable(senderEnv, "test", "test"); + insertResult = TableModelUtils.insertData("test", "test", 0, 100, senderEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on sender + for (int i = 100; i < 200; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + senderEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + for (int i = 200; i < 300; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 100, 200, senderEnv); + if (!insertResult) { + return; + } + insertResult = TableModelUtils.insertData("test", "test", 200, 300, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(senderEnv, "flush")) { + return; + } + + try (final Connection connection = receiverEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'database-name'='test'," + + "'table-name'='test'," + + "'path'='root.db.d1.s1'," + + "'mode.double-living'='true')" + + " with sink (" + + "'batch.enable'='false'," + + "'node-urls'='%s')", + "p2", senderDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // insertion on receiver + for (int i = 300; i < 400; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 300, 400, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + final Set expectedResSet = new HashSet<>(); + for (int i = 0; i < 400; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 400, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 400, receiverEnv, handleFailure); + + // restart cluster + try { + TestUtils.restartCluster(senderEnv); + TestUtils.restartCluster(receiverEnv); + } catch (final Throwable e) { + e.printStackTrace(); + return; + } + + // insertion on receiver + for (int i = 400; i < 500; ++i) { + if (!TestUtils.tryExecuteNonQueryWithRetry( + receiverEnv, String.format("insert into root.db.d1(time, s1) values (%s, 1)", i))) { + return; + } + } + insertResult = TableModelUtils.insertData("test", "test", 400, 500, receiverEnv); + if (!insertResult) { + return; + } + if (!TestUtils.tryExecuteNonQueryWithRetry(receiverEnv, "flush")) { + return; + } + + // check result + for (int i = 400; i < 500; ++i) { + expectedResSet.add(i + ",1.0,"); + } + TestUtils.assertDataEventuallyOnEnv( + senderEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TestUtils.assertDataEventuallyOnEnv( + receiverEnv, "select * from root.**", "Time,root.db.d1.s1,", expectedResSet); + TableModelUtils.assertData("test", "test", 0, 500, senderEnv, handleFailure); + TableModelUtils.assertData("test", "test", 0, 500, receiverEnv, handleFailure); + } + + @Test + public void testDoubleLivingIsolation() throws Exception { + final String treePipeName = "tree_a2b"; + final String tablePipeName = "table_a2b"; + + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // Create tree pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(1, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Create table pipe + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'mode.double-living'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + // Show table pipe by table session + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + Assert.assertEquals(2, count); + } catch (final SQLException e) { + fail(e.getMessage()); + } + + try (final SyncConfigNodeIServiceClient client = + (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { + // Drop pipe + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); + Assert.assertEquals( + TSStatusCode.SUCCESS_STATUS.getStatusCode(), + client + .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) + .getCode()); + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index c4fb40c4b8f8..4e8ee1e1305e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -19,10 +19,8 @@ package org.apache.iotdb.pipe.it.tablemodel; -import org.apache.iotdb.common.rpc.thrift.TSStatus; import org.apache.iotdb.commons.client.sync.SyncConfigNodeIServiceClient; import org.apache.iotdb.confignode.rpc.thrift.TAlterPipeReq; -import org.apache.iotdb.confignode.rpc.thrift.TCreatePipeReq; import org.apache.iotdb.confignode.rpc.thrift.TDropPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TStartPipeReq; import org.apache.iotdb.confignode.rpc.thrift.TStopPipeReq; @@ -42,8 +40,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import static org.junit.Assert.fail; @@ -59,39 +55,27 @@ public void testWritePipeIsolation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); // Create tree pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "tree"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(treePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); } // Create table pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "table"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(tablePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s with sink ('node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); } try (final SyncConfigNodeIServiceClient client = @@ -300,71 +284,22 @@ public void testCaptureTreeAndTableIsolation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); // Create tree pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "tree"); - extractorAttributes.put("capture.tree", "true"); - extractorAttributes.put("capture.table", "true"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(treePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - // Show tree pipe by tree session try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + treePipeName, receiverDataNode.getIpAndPortString())); } catch (final SQLException e) { + e.printStackTrace(); fail(e.getMessage()); } - // Create table pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("capture.tree", "true"); - extractorAttributes.put("capture.table", "true"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(tablePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - // Show tree pipe by tree session try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { @@ -373,7 +308,7 @@ public void testCaptureTreeAndTableIsolation() throws Exception { while (resultSet.next()) { count++; } - Assert.assertEquals(2, count); + Assert.assertEquals(1, count); } catch (final SQLException e) { fail(e.getMessage()); } @@ -386,96 +321,28 @@ public void testCaptureTreeAndTableIsolation() throws Exception { while (resultSet.next()) { count++; } - Assert.assertEquals(2, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } - - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Drop pipe - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); - Assert.assertEquals( - TSStatusCode.SUCCESS_STATUS.getStatusCode(), - client - .dropPipeExtended(new TDropPipeReq(tablePipeName).setIsTableModel(false)) - .getCode()); - } - } - - @Test - public void testDoubleLivingIsolation() throws Exception { - final String treePipeName = "tree_a2b"; - final String tablePipeName = "table_a2b"; - - final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - - // Create tree pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "tree"); - extractorAttributes.put("mode.double-living", "true"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(treePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - - // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } Assert.assertEquals(1, count); } catch (final SQLException e) { fail(e.getMessage()); } - // Show table pipe by table session + // Create table pipe try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + tablePipeName, receiverDataNode.getIpAndPortString())); } catch (final SQLException e) { + e.printStackTrace(); fail(e.getMessage()); } - // Create table pipe - try (final SyncConfigNodeIServiceClient client = - (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - final Map extractorAttributes = new HashMap<>(); - final Map processorAttributes = new HashMap<>(); - final Map connectorAttributes = new HashMap<>(); - - extractorAttributes.put("__system.sql-dialect", "table"); - extractorAttributes.put("mode.double-living", "true"); - connectorAttributes.put("node-urls", receiverDataNode.getIpAndPortString()); - - final TSStatus status = - client.createPipe( - new TCreatePipeReq(tablePipeName, connectorAttributes) - .setExtractorAttributes(extractorAttributes) - .setProcessorAttributes(processorAttributes)); - Assert.assertEquals(TSStatusCode.SUCCESS_STATUS.getStatusCode(), status.getCode()); - } - // Show tree pipe by tree session try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java index c618ead5a1b8..09658a583b08 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeSyntaxIT.java @@ -125,7 +125,7 @@ public void testRevertParameterOrder() { String.format( "create pipe p1" + " with source ( " - + "'capture.table'='test'," + + "'capture.table'='true'," + "'database-name'='test'," + "'table-name'='test'," + "'mode.streaming'='true'," @@ -545,6 +545,7 @@ public void testShowPipeWithWrongPipeName() throws Exception { final Map processorAttributes = new HashMap<>(); final Map connectorAttributes = new HashMap<>(); + extractorAttributes.put("__system.sql-dialect", "table"); extractorAttributes.put("extractor.database-name", "test"); extractorAttributes.put("extractor.table-name", "test.*"); extractorAttributes.put("extractor.inclusion", "data.insert"); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java index e1c23e025c46..bee26df7fd1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java @@ -127,6 +127,15 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor { public void validate(final PipeParameterValidator validator) throws Exception { super.validate(validator); + final boolean isDoubleLiving = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + // Validate whether the pipe needs to extract table model data or tree model data final boolean isTreeDialect = validator @@ -135,21 +144,23 @@ public void validate(final PipeParameterValidator validator) throws Exception { SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); final boolean isTreeModelDataAllowedToBeCaptured = - validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - isTreeDialect); + isDoubleLiving + || validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + isTreeDialect); final boolean isTableModelDataAllowedToBeCaptured = - validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - !isTreeDialect); + isDoubleLiving + || validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + !isTreeDialect); if (!isTreeModelDataAllowedToBeCaptured && validator .getParameters() diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java index 3ba6c70adaeb..0f6bb2823588 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java @@ -98,15 +98,22 @@ public String getTablePattern() { */ public static TablePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTableModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - !sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + final boolean isTableModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + !sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String databaseNamePattern = sourceParameters.getStringOrDefault( Arrays.asList(EXTRACTOR_DATABASE_NAME_KEY, SOURCE_DATABASE_NAME_KEY), diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java index 7ee8f6dd88db..f03f0a291868 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TreePattern.java @@ -70,15 +70,23 @@ public boolean isRoot() { */ public static TreePattern parsePipePatternFromSourceParameters( final PipeParameters sourceParameters) { - final boolean isTreeModelDataAllowedToBeCaptured = + final boolean isDoubleLiving = sourceParameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - sourceParameters - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + + final boolean isTreeModelDataAllowedToBeCaptured = + isDoubleLiving + || sourceParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + sourceParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE)); final String path = sourceParameters.getStringByKeys(EXTRACTOR_PATH_KEY, SOURCE_PATH_KEY); diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java index 04e284375d29..6c0b81d3b938 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java @@ -89,7 +89,6 @@ public void validate(final PipeParameterValidator validator) throws Exception { EXTRACTOR_EXCLUSION_DEFAULT_VALUE)); // Validate double living - // NOTE: this function may modify pipe parameters validateDoubleLiving(validator.getParameters()); } @@ -100,58 +99,41 @@ private void validateDoubleLiving(final PipeParameters parameters) { PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - if (isDoubleLiving) { - // check 'capture.tree' - final Boolean isTreeModelDataAllowedToBeCaptured = - parameters.getBooleanByKeys( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); - if (Objects.nonNull(isTreeModelDataAllowedToBeCaptured) - && !isTreeModelDataAllowedToBeCaptured) { - throw new PipeParameterNotValidException( - "capture.tree can not be specified to false when double living is enabled"); - } - - // check 'capture.table' - final Boolean isTableModelDataAllowedToBeCaptured = - parameters.getBooleanByKeys( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); - if (Objects.nonNull(isTableModelDataAllowedToBeCaptured) - && !isTableModelDataAllowedToBeCaptured) { - throw new PipeParameterNotValidException( - "capture.table can not be specified to false when double living is enabled"); - } - - // check 'forwarding-pipe-requests' - final Boolean isForwardingPipeRequests = - parameters.getBooleanByKeys( - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY); - if (Objects.nonNull(isForwardingPipeRequests) && isForwardingPipeRequests) { - throw new PipeParameterNotValidException( - "forwarding-pipe-requests can not be specified to true when double living is enabled"); - } - - // modify pipe parameters - applyDoubleLiving(parameters); + if (!isDoubleLiving) { + return; + } + + // check 'capture.tree' + final Boolean isTreeModelDataAllowedToBeCaptured = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + if (Objects.nonNull(isTreeModelDataAllowedToBeCaptured) + && !isTreeModelDataAllowedToBeCaptured) { + throw new PipeParameterNotValidException( + "capture.tree can not be specified to false when double living is enabled"); + } + + // check 'capture.table' + final Boolean isTableModelDataAllowedToBeCaptured = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + if (Objects.nonNull(isTableModelDataAllowedToBeCaptured) + && !isTableModelDataAllowedToBeCaptured) { + throw new PipeParameterNotValidException( + "capture.table can not be specified to false when double living is enabled"); } - } - private void applyDoubleLiving(final PipeParameters parameters) { - // temporarily set 'capture.tree' = 'true' - parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); - parameters.addRawAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, "true"); - parameters.addRawAttribute(PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY, "true"); - // temporarily set 'capture.table' = 'true' - parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); - parameters.addRawAttribute(PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, "true"); - parameters.addRawAttribute(PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY, "true"); - // temporarily set 'forwarding-pipe-requests' = 'false' - parameters.addAttribute(PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, "false"); - parameters.addRawAttribute( - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, "false"); - parameters.addRawAttribute(PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY, "false"); + // check 'forwarding-pipe-requests' + final Boolean isForwardingPipeRequests = + parameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY); + if (Objects.nonNull(isForwardingPipeRequests) && isForwardingPipeRequests) { + throw new PipeParameterNotValidException( + "forwarding-pipe-requests can not be specified to true when double living is enabled"); + } } @Override @@ -166,15 +148,22 @@ public void customize( taskID = pipeName + "_" + regionId + "_" + creationTime; pipeTaskMeta = environment.getPipeTaskMeta(); - // apply double living before setting isForwardingPipeRequests - applyDoubleLiving(parameters); - - isForwardingPipeRequests = + final boolean isDoubleLiving = parameters.getBooleanOrDefault( Arrays.asList( - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, - PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), - PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { + isForwardingPipeRequests = false; + } else { + isForwardingPipeRequests = + parameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_KEY, + PipeExtractorConstant.SOURCE_FORWARDING_PIPE_REQUESTS_KEY), + PipeExtractorConstant.EXTRACTOR_FORWARDING_PIPE_REQUESTS_DEFAULT_VALUE); + } } @Override From c97731bba86057ca3887e611cab2211def6df10d Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 2 Jan 2025 19:31:16 +0800 Subject: [PATCH 11/13] fix review --- .../iotdb/pipe/api/customizer/parameter/PipeParameters.java | 4 ---- .../manager/pipe/coordinator/task/PipeTaskCoordinator.java | 4 ++-- .../iotdb/confignode/persistence/pipe/PipeTaskInfo.java | 2 +- .../apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java | 4 ++-- .../iotdb/commons/pipe/config/constant/SystemConstant.java | 2 +- 5 files changed, 6 insertions(+), 10 deletions(-) diff --git a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java index dbcf8c522d00..3dcb2d19b0a7 100644 --- a/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java +++ b/iotdb-api/pipe-api/src/main/java/org/apache/iotdb/pipe/api/customizer/parameter/PipeParameters.java @@ -75,10 +75,6 @@ public void addAttribute(final String key, String values) { attributes.put(KeyReducer.reduce(key), values); } - public void addRawAttribute(final String key, String values) { - attributes.put(key, values); - } - public String getString(final String key) { final String value = attributes.get(key); return value != null ? value : attributes.get(KeyReducer.reduce(key)); diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java index da94f98527da..4aaf3ab46c3f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/task/PipeTaskCoordinator.java @@ -151,7 +151,7 @@ public TSStatus alterPipe(TAlterPipeReq req) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus startPipe(String pipeName) { + private TSStatus startPipe(String pipeName) { final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { status = configManager.getProcedureManager().startConsensusPipe(pipeName); @@ -177,7 +177,7 @@ public TSStatus startPipe(TStartPipeReq req) { } /** Caller should ensure that the method is called in the lock {@link #lock()}. */ - public TSStatus stopPipe(String pipeName) { + private TSStatus stopPipe(String pipeName) { final boolean isStoppedByRuntimeException = pipeTaskInfo.isStoppedByRuntimeException(pipeName); final TSStatus status; if (pipeName.startsWith(PipeStaticMeta.CONSENSUS_PIPE_PREFIX)) { diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 55ae6ad89e21..6e6d10ce979f 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -348,7 +348,7 @@ public boolean isPipeExisted(final String pipeName, final boolean isTableModel) acquireReadLock(); try { return pipeMetaKeeper.containsPipeMeta( - pipeName, SystemConstant.fetchSqlDialectValue(isTableModel)); + pipeName, SystemConstant.getSqlDialectValue(isTableModel)); } finally { releaseReadLock(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index effa04e9684c..1badb0b0e7bd 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -66,7 +66,7 @@ public PipeTemporaryMeta getTemporaryMeta() { } public boolean matchSqlDialect(final boolean isTableModel) { - return matchSqlDialect(SystemConstant.fetchSqlDialectValue(isTableModel)); + return matchSqlDialect(SystemConstant.getSqlDialectValue(isTableModel)); } public boolean matchSqlDialect(final String sqlDialect) { @@ -91,7 +91,7 @@ private boolean matchBoth() { PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE) || - // 2. 'capture.tree' and 'capture.table' is set to true + // 2. 'capture.tree' and 'capture.table' are set to true (getStaticMeta() .getExtractorParameters() .getBooleanOrDefault( diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java index 81d3a1be11de..3fa32552b372 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java @@ -52,7 +52,7 @@ public static PipeParameters addSystemKeysIfNecessary(final PipeParameters given return new PipeParameters(attributes); } - public static String fetchSqlDialectValue(final boolean isTableModel) { + public static String getSqlDialectValue(final boolean isTableModel) { return isTableModel ? SystemConstant.SQL_DIALECT_TABLE_VALUE : SystemConstant.SQL_DIALECT_TREE_VALUE; From c79ccefa63f090f1dffbc1c8dc9e7078e08f2d88 Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 2 Jan 2025 20:25:25 +0800 Subject: [PATCH 12/13] improve --- .../tablemodel/IoTDBPipeDoubleLivingIT.java | 53 +----- .../it/tablemodel/IoTDBPipeIsolationIT.java | 166 +++++++----------- .../pipe/it/tablemodel/TableModelUtils.java | 17 ++ .../response/pipe/task/PipeTableResp.java | 2 +- .../persistence/pipe/PipeTaskInfo.java | 4 +- .../pipe/agent/task/meta/PipeMeta.java | 69 ++++---- .../pipe/agent/task/meta/PipeMetaKeeper.java | 4 +- .../pipe/config/constant/SystemConstant.java | 6 - .../pipe/extractor/IoTDBExtractor.java | 10 +- 9 files changed, 130 insertions(+), 201 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java index 3f76bcf63990..f6573728cf29 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeDoubleLivingIT.java @@ -36,7 +36,6 @@ import org.junit.runner.RunWith; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.HashSet; @@ -265,8 +264,8 @@ public void testBasicDoubleLiving() { @Test public void testDoubleLivingIsolation() throws Exception { - final String treePipeName = "tree_a2b"; - final String tablePipeName = "table_a2b"; + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -287,30 +286,10 @@ public void testDoubleLivingIsolation() throws Exception { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); // Create table pipe try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); @@ -329,30 +308,10 @@ public void testDoubleLivingIsolation() throws Exception { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(2, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(2, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index 4e8ee1e1305e..ed2644debc0b 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -36,7 +36,6 @@ import org.junit.runner.RunWith; import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.Collections; @@ -49,8 +48,8 @@ public class IoTDBPipeIsolationIT extends AbstractPipeTableModelTestIT { @Test public void testWritePipeIsolation() throws Exception { - final String treePipeName = "tree_a2b"; - final String tablePipeName = "table_a2b"; + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); @@ -196,12 +195,12 @@ public void testWritePipeIsolation() throws Exception { @Test public void testReadPipeIsolation() { - final String treePipeName = "tree_a2b"; - final String tablePipeName = "table_a2b"; + final String treePipeName = "treePipe"; + final String tablePipeName = "tablePipe"; final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - // Create tree pipe by tree session + // 1. Create tree pipe by tree session try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -213,32 +212,12 @@ public void testReadPipeIsolation() { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(0, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); - // Create table pipe by table session + // 2. Create table pipe by table session try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -250,30 +229,10 @@ public void testReadPipeIsolation() { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); } @Test @@ -283,7 +242,7 @@ public void testCaptureTreeAndTableIsolation() throws Exception { final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); - // Create tree pipe + // 1. Create tree pipe by tree session try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -301,32 +260,12 @@ public void testCaptureTreeAndTableIsolation() throws Exception { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(1, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); - // Create table pipe + // 2. Create table pipe by table session try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); final Statement statement = connection.createStatement()) { statement.execute( @@ -344,34 +283,14 @@ public void testCaptureTreeAndTableIsolation() throws Exception { } // Show tree pipe by tree session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(2, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); // Show table pipe by table session - try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); - final Statement statement = connection.createStatement()) { - final ResultSet resultSet = statement.executeQuery("show pipes"); - int count = 0; - while (resultSet.next()) { - count++; - } - Assert.assertEquals(2, count); - } catch (final SQLException e) { - fail(e.getMessage()); - } + Assert.assertEquals(2, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + // 3. Drop pipe try (final SyncConfigNodeIServiceClient client = (SyncConfigNodeIServiceClient) senderEnv.getLeaderConfigNodeConnection()) { - // Drop pipe Assert.assertEquals( TSStatusCode.SUCCESS_STATUS.getStatusCode(), client.dropPipeExtended(new TDropPipeReq(treePipeName).setIsTableModel(true)).getCode()); @@ -382,4 +301,55 @@ public void testCaptureTreeAndTableIsolation() throws Exception { .getCode()); } } + + @Test + public void testCaptureCornerCases() { + final DataNodeWrapper receiverDataNode = receiverEnv.getDataNodeWrapper(0); + + // 1. Create tree pipe but capture table data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='true')" + + " with sink (" + + "'node-urls'='%s')", + "p1", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(0, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 2. Create table pipe but capture tree data + try (final Connection connection = senderEnv.getConnection(BaseEnv.TABLE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='true'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p2", receiverDataNode.getIpAndPortString())); + } catch (final SQLException e) { + e.printStackTrace(); + fail(e.getMessage()); + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + } } diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java index ce33dc3e6a53..7485bc69773d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/TableModelUtils.java @@ -37,6 +37,8 @@ import java.nio.charset.StandardCharsets; import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.text.SimpleDateFormat; import java.time.LocalDate; @@ -426,4 +428,19 @@ public static Tablet generateTablet( return tablet; } + + public static int showPipesCount(final BaseEnv baseEnv, final String sqlDialect) { + try (final Connection connection = baseEnv.getConnection(sqlDialect); + final Statement statement = connection.createStatement()) { + final ResultSet resultSet = statement.executeQuery("show pipes"); + int count = 0; + while (resultSet.next()) { + count++; + } + return count; + } catch (final SQLException e) { + fail(e.getMessage()); + } + return 0; + } } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java index 14174dcda475..20cd76e4a41d 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/response/pipe/task/PipeTableResp.java @@ -97,7 +97,7 @@ public PipeTableResp filter(final Boolean whereClause, final String pipeName) { public PipeTableResp filter( final Boolean whereClause, final String pipeName, final boolean isTableModel) { final PipeTableResp resp = filter(whereClause, pipeName); - resp.allPipeMeta.removeIf(meta -> !meta.matchSqlDialect(isTableModel)); + resp.allPipeMeta.removeIf(meta -> !meta.visibleUnder(isTableModel)); return resp; } diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java index 6e6d10ce979f..f2f03c71c2e0 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/pipe/PipeTaskInfo.java @@ -37,7 +37,6 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeConnectorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.PipeProcessorConstant; -import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; import org.apache.iotdb.commons.snapshot.SnapshotProcessor; import org.apache.iotdb.confignode.consensus.request.ConfigPhysicalPlan; import org.apache.iotdb.confignode.consensus.request.write.pipe.runtime.PipeHandleLeaderChangePlan; @@ -347,8 +346,7 @@ public boolean isPipeExisted(final String pipeName) { public boolean isPipeExisted(final String pipeName, final boolean isTableModel) { acquireReadLock(); try { - return pipeMetaKeeper.containsPipeMeta( - pipeName, SystemConstant.getSqlDialectValue(isTableModel)); + return pipeMetaKeeper.containsPipeMeta(pipeName, isTableModel); } finally { releaseReadLock(); } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java index 1badb0b0e7bd..033fe6505824 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMeta.java @@ -21,6 +21,7 @@ import org.apache.iotdb.commons.pipe.config.constant.PipeExtractorConstant; import org.apache.iotdb.commons.pipe.config.constant.SystemConstant; +import org.apache.iotdb.pipe.api.customizer.parameter.PipeParameters; import org.apache.tsfile.utils.PublicBAOS; @@ -65,47 +66,39 @@ public PipeTemporaryMeta getTemporaryMeta() { return temporaryMeta; } - public boolean matchSqlDialect(final boolean isTableModel) { - return matchSqlDialect(SystemConstant.getSqlDialectValue(isTableModel)); - } + public boolean visibleUnder(final boolean isTableModel) { + final PipeParameters extractorParameters = getStaticMeta().getExtractorParameters(); - public boolean matchSqlDialect(final String sqlDialect) { - if (Objects.isNull(sqlDialect)) { + // visible under all model when 'mode.double-living' is set to true + final boolean isDoubleLiving = + extractorParameters.getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, + PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), + PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); + if (isDoubleLiving) { return true; } - return matchBoth() - || sqlDialect.equalsIgnoreCase( - getStaticMeta() - .getExtractorParameters() - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE)); - } - - private boolean matchBoth() { - return // 1. 'mode.double-living' is set to true - getStaticMeta() - .getExtractorParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, - PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), - PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE) - || - // 2. 'capture.tree' and 'capture.table' are set to true - (getStaticMeta() - .getExtractorParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - false) - && getStaticMeta() - .getExtractorParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - false)); + + final boolean isTreeDialect = + extractorParameters + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); + final Boolean _isCaptureTree = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); + final boolean isCaptureTree = Objects.nonNull(_isCaptureTree) ? _isCaptureTree : isTreeDialect; + final Boolean _isCaptureTable = + extractorParameters.getBooleanByKeys( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); + final boolean isCaptureTable = + Objects.nonNull(_isCaptureTable) ? _isCaptureTable : !isTreeDialect; + + // visible under specific tree or table model <-> actually capture tree or table data + return isTableModel ? isCaptureTable : isCaptureTree; } public ByteBuffer serialize() throws IOException { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java index 2074cbaa404e..4009288dfc70 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/agent/task/meta/PipeMetaKeeper.java @@ -85,12 +85,12 @@ public boolean containsPipeMeta(String pipeName) { return pipeNameToPipeMetaMap.containsKey(pipeName); } - public boolean containsPipeMeta(String pipeName, String sqlDialect) { + public boolean containsPipeMeta(String pipeName, boolean isTableModel) { final PipeMeta pipeMeta = pipeNameToPipeMetaMap.get(pipeName); if (Objects.isNull(pipeMeta)) { return false; } - return pipeMeta.matchSqlDialect(sqlDialect); + return pipeMeta.visibleUnder(isTableModel); } public Iterable getPipeMetaList() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java index 3fa32552b372..105390eab9ca 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/config/constant/SystemConstant.java @@ -52,12 +52,6 @@ public static PipeParameters addSystemKeysIfNecessary(final PipeParameters given return new PipeParameters(attributes); } - public static String getSqlDialectValue(final boolean isTableModel) { - return isTableModel - ? SystemConstant.SQL_DIALECT_TABLE_VALUE - : SystemConstant.SQL_DIALECT_TREE_VALUE; - } - /////////////////////////////////// Private Constructor /////////////////////////////////// private SystemConstant() { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java index 6c0b81d3b938..cecee61c7fae 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/extractor/IoTDBExtractor.java @@ -104,23 +104,21 @@ private void validateDoubleLiving(final PipeParameters parameters) { } // check 'capture.tree' - final Boolean isTreeModelDataAllowedToBeCaptured = + final Boolean isCaptureTree = parameters.getBooleanByKeys( PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY); - if (Objects.nonNull(isTreeModelDataAllowedToBeCaptured) - && !isTreeModelDataAllowedToBeCaptured) { + if (Objects.nonNull(isCaptureTree) && !isCaptureTree) { throw new PipeParameterNotValidException( "capture.tree can not be specified to false when double living is enabled"); } // check 'capture.table' - final Boolean isTableModelDataAllowedToBeCaptured = + final Boolean isCaptureTable = parameters.getBooleanByKeys( PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY); - if (Objects.nonNull(isTableModelDataAllowedToBeCaptured) - && !isTableModelDataAllowedToBeCaptured) { + if (Objects.nonNull(isCaptureTable) && !isCaptureTable) { throw new PipeParameterNotValidException( "capture.table can not be specified to false when double living is enabled"); } From b27d6a355de8427a4a011f5a2caae0ee1d057b0a Mon Sep 17 00:00:00 2001 From: VGalaxies Date: Thu, 2 Jan 2025 21:06:41 +0800 Subject: [PATCH 13/13] capture.tree and capture.table can not both be specified as false --- .../it/tablemodel/IoTDBPipeIsolationIT.java | 22 +++++++ .../dataregion/IoTDBDataRegionExtractor.java | 57 ++++++++++--------- 2 files changed, 53 insertions(+), 26 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java index ed2644debc0b..57a3be38e217 100644 --- a/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/pipe/it/tablemodel/IoTDBPipeIsolationIT.java @@ -351,5 +351,27 @@ public void testCaptureCornerCases() { // Show table pipe by table session Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); + + // 3. Create pipe with capture.tree and capture.table set to false + try (final Connection connection = senderEnv.getConnection(BaseEnv.TREE_SQL_DIALECT); + final Statement statement = connection.createStatement()) { + statement.execute( + String.format( + "create pipe %s" + + " with source (" + + "'capture.tree'='false'," + + "'capture.table'='false')" + + " with sink (" + + "'node-urls'='%s')", + "p3", receiverDataNode.getIpAndPortString())); + fail(); + } catch (final SQLException ignored) { + } + + // Show tree pipe by tree session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TREE_SQL_DIALECT)); + + // Show table pipe by table session + Assert.assertEquals(1, TableModelUtils.showPipesCount(senderEnv, BaseEnv.TABLE_SQL_DIALECT)); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java index fcc3bcb73a30..0a033aff297b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/dataregion/IoTDBDataRegionExtractor.java @@ -48,6 +48,7 @@ import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent; import org.apache.iotdb.pipe.api.event.dml.insertion.TsFileInsertionEvent; import org.apache.iotdb.pipe.api.exception.PipeException; +import org.apache.iotdb.pipe.api.exception.PipeParameterNotValidException; import org.apache.tsfile.utils.Pair; import org.slf4j.Logger; @@ -127,6 +128,34 @@ public class IoTDBDataRegionExtractor extends IoTDBExtractor { public void validate(final PipeParameterValidator validator) throws Exception { super.validate(validator); + // Validate whether the pipe needs to extract table model data or tree model data + final boolean isTreeDialect = + validator + .getParameters() + .getStringOrDefault( + SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) + .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); + final boolean isCaptureTree = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), + isTreeDialect); + final boolean isCaptureTable = + validator + .getParameters() + .getBooleanOrDefault( + Arrays.asList( + PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, + PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), + !isTreeDialect); + if (!isCaptureTree && !isCaptureTable) { + throw new PipeParameterNotValidException( + "capture.tree and capture.table can not both be specified as false"); + } + final boolean isDoubleLiving = validator .getParameters() @@ -135,32 +164,8 @@ public void validate(final PipeParameterValidator validator) throws Exception { PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_KEY, PipeExtractorConstant.SOURCE_MODE_DOUBLE_LIVING_KEY), PipeExtractorConstant.EXTRACTOR_MODE_DOUBLE_LIVING_DEFAULT_VALUE); - - // Validate whether the pipe needs to extract table model data or tree model data - final boolean isTreeDialect = - validator - .getParameters() - .getStringOrDefault( - SystemConstant.SQL_DIALECT_KEY, SystemConstant.SQL_DIALECT_TREE_VALUE) - .equals(SystemConstant.SQL_DIALECT_TREE_VALUE); - final boolean isTreeModelDataAllowedToBeCaptured = - isDoubleLiving - || validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TREE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TREE_KEY), - isTreeDialect); - final boolean isTableModelDataAllowedToBeCaptured = - isDoubleLiving - || validator - .getParameters() - .getBooleanOrDefault( - Arrays.asList( - PipeExtractorConstant.EXTRACTOR_CAPTURE_TABLE_KEY, - PipeExtractorConstant.SOURCE_CAPTURE_TABLE_KEY), - !isTreeDialect); + final boolean isTreeModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTree; + final boolean isTableModelDataAllowedToBeCaptured = isDoubleLiving || isCaptureTable; if (!isTreeModelDataAllowedToBeCaptured && validator .getParameters()