From c0e3287275ffd0df92162b6c8311c37f2aec3247 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Tue, 6 Feb 2024 19:49:25 +0800 Subject: [PATCH] [improve](routine-load) reduce routine load job transaction info storage in memory #30903 --- .../cloud/transaction/CloudGlobalTransactionMgr.java | 6 ------ .../org/apache/doris/service/FrontendServiceImpl.java | 3 --- .../doris/transaction/DatabaseTransactionMgr.java | 11 ----------- .../doris/transaction/GlobalTransactionMgr.java | 6 ------ .../doris/transaction/GlobalTransactionMgrIface.java | 3 --- 5 files changed, 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java index c006906c0a2fcf..134c4f50565af5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/transaction/CloudGlobalTransactionMgr.java @@ -594,12 +594,6 @@ public void updateMultiTableRunningTransactionTableIds(Long dbId, Long transacti throw new UserException(NOT_SUPPORTED_MSG); } - @Override - public void putTransactionTableNames(Long dbId, Long transactionId, List tableIds) - throws Exception { - throw new Exception(NOT_SUPPORTED_MSG); - } - @Override public TWaitingTxnStatusResult getWaitingTxnStatus(TWaitingTxnStatusRequest request) throws AnalysisException, TimeoutException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 798d3900ce8e71..f80ed0f54ace8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -1931,7 +1931,6 @@ public TStreamLoadMultiTablePutResult streamLoadMultiTablePut(TStreamLoadPutRequ } long timeoutMs = request.isSetThriftRpcTimeoutMs() ? request.getThriftRpcTimeoutMs() : 5000; List planFragmentParamsList = new ArrayList<>(tableNames.size()); - List tableIds = olapTables.stream().map(OlapTable::getId).collect(Collectors.toList()); // todo: if is multi table, we need consider the lock time and the timeout boolean enablePipelineLoad = Config.enable_pipeline_load; try { @@ -1949,8 +1948,6 @@ public TStreamLoadMultiTablePutResult streamLoadMultiTablePut(TStreamLoadPutRequ } multiTableFragmentInstanceIdIndexMap.put(request.getTxnId(), ++index); } - Env.getCurrentGlobalTransactionMgr() - .putTransactionTableNames(db.getId(), request.getTxnId(), tableIds); LOG.debug("receive stream load multi table put request result: {}", result); } catch (Throwable e) { LOG.warn("catch unknown result.", e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java index e772b28ade4792..fd48b974544193 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java @@ -2246,17 +2246,6 @@ private static String getStackTrace(Thread t) { return msgBuilder.toString(); } - protected void putTransactionTableNames(long transactionId, List tableIds) { - if (CollectionUtils.isEmpty(tableIds)) { - return; - } - if (multiTableRunningTransactionTableIdMaps.contains(transactionId)) { - multiTableRunningTransactionTableIdMaps.get(transactionId).addAll(tableIds); - return; - } - multiTableRunningTransactionTableIdMaps.put(transactionId, tableIds); - } - /** * Update transaction table ids by transaction id. * it's used for multi table transaction. diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java index 84125950a4e7b2..f755d148c9203b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgr.java @@ -455,12 +455,6 @@ public void updateMultiTableRunningTransactionTableIds(Long dbId, Long transacti getDatabaseTransactionMgr(dbId).updateMultiTableRunningTransactionTableIds(transactionId, tableIds); } - @Override - public void putTransactionTableNames(Long dbId, Long transactionId, List tableIds) - throws Exception { - getDatabaseTransactionMgr(dbId).putTransactionTableNames(transactionId, tableIds); - } - @Override public TWaitingTxnStatusResult getWaitingTxnStatus(TWaitingTxnStatusRequest request) throws AnalysisException, TimeoutException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java index 6890222f1824b6..489b4240d0d054 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/GlobalTransactionMgrIface.java @@ -122,9 +122,6 @@ public boolean isPreviousTransactionsFinished(long endTransactionId, long dbId, public void updateMultiTableRunningTransactionTableIds(Long dbId, Long transactionId, List tableIds) throws UserException; - public void putTransactionTableNames(Long dbId, Long transactionId, List tableIds) - throws Exception; - public TWaitingTxnStatusResult getWaitingTxnStatus(TWaitingTxnStatusRequest request) throws AnalysisException, TimeoutException;