From 2f8a935e0838071cdc003de600adc179d851a0ec Mon Sep 17 00:00:00 2001 From: BePPPower Date: Mon, 6 Jan 2025 17:07:49 +0800 Subject: [PATCH 1/5] fix --- .../doris/datasource/hive/HMSDlaTable.java | 71 +++++++++ .../datasource/hive/HMSExternalTable.java | 80 ++--------- .../datasource/hive/HiveExternalTable.java | 135 ++++++++++++++++++ 3 files changed, 220 insertions(+), 66 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java new file mode 100644 index 00000000000000..879ad694753a03 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.mtmv.MTMVBaseTableIf; +import org.apache.doris.mtmv.MTMVRefreshContext; +import org.apache.doris.mtmv.MTMVSnapshotIf; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public abstract class HMSDlaTable implements MTMVBaseTableIf { + HMSExternalTable hmsTable; + + public HMSDlaTable(HMSExternalTable table) { + this.hmsTable = table; + } + + + abstract Map getAndCopyPartitionItems(Optional snapshot) + throws AnalysisException; + + abstract PartitionType getPartitionType(Optional snapshot); + + abstract Set getPartitionColumnNames(Optional snapshot) throws DdlException; + + abstract List getPartitionColumns(Optional snapshot); + + abstract MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, + Optional snapshot) throws AnalysisException; + + abstract MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) + throws AnalysisException; + + public boolean needAutoRefresh() { + return true; + } + + abstract boolean isPartitionColumnAllowNull(); + + @Override + public void beforeMTMVRefresh(MTMV mtmv) throws DdlException { + Env.getCurrentEnv().getRefreshManager() + .refreshTable(hmsTable.getCatalog().getName(), hmsTable.getDbName(), hmsTable.getName(), true); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index b554f508103992..13882264554529 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -40,11 +40,9 @@ import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.mtmv.MTMVBaseTableIf; -import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot; import org.apache.doris.mtmv.MTMVRefreshContext; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVSnapshotIf; -import org.apache.doris.mtmv.MTMVTimestampSnapshot; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.qe.GlobalVariable; @@ -159,6 +157,8 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private DLAType dlaType = DLAType.UNKNOWN; + private HMSDlaTable dlaTable; + // record the event update time when enable hms event listener protected volatile long eventUpdateTime; @@ -200,6 +200,7 @@ protected synchronized void makeSureInitialized() { dlaType = DLAType.HUDI; } else if (supportedHiveTable()) { dlaType = DLAType.HIVE; + dlaTable = new HiveExternalTable(this); } else { // Should not reach here. Because `supportedHiveTable` will throw exception if not return true. throw new NotSupportedException("Unsupported dlaType for table: " + getNameWithFullQualifiers()); @@ -306,7 +307,7 @@ public List getPartitionColumns() { @Override public List getPartitionColumns(Optional snapshot) { - return getPartitionColumns(); + return dlaTable.getPartitionColumns(snapshot); } @Override @@ -800,7 +801,7 @@ public Set getDistributionColumnNames() { @Override public PartitionType getPartitionType(Optional snapshot) { - return getPartitionType(); + return dlaTable.getPartitionType(snapshot); } public PartitionType getPartitionType() { @@ -808,8 +809,8 @@ public PartitionType getPartitionType() { } @Override - public Set getPartitionColumnNames(Optional snapshot) { - return getPartitionColumnNames(); + public Set getPartitionColumnNames(Optional snapshot) throws DdlException { + return dlaTable.getPartitionColumnNames(snapshot); } public Set getPartitionColumnNames() { @@ -818,78 +819,26 @@ public Set getPartitionColumnNames() { } @Override - public Map getAndCopyPartitionItems(Optional snapshot) { - return getNameToPartitionItems(); + public Map getAndCopyPartitionItems(Optional snapshot) + throws AnalysisException { + return dlaTable.getAndCopyPartitionItems(snapshot); } @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, Optional snapshot) throws AnalysisException { - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - getDbName(), getName(), getPartitionColumnTypes()); - Long partitionId = getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues); - HivePartition hivePartition = getHivePartitionByIdOrAnalysisException(partitionId, - hivePartitionValues, cache); - return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime()); + return dlaTable.getPartitionSnapshot(partitionName, context, snapshot); } @Override public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) throws AnalysisException { - if (getPartitionType() == PartitionType.UNPARTITIONED) { - return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime()); - } - HivePartition maxPartition = null; - long maxVersionTime = 0L; - long visibleVersionTime; - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) getCatalog()); - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - getDbName(), getName(), getPartitionColumnTypes()); - List partitionList = cache.getAllPartitionsWithCache(getDbName(), getName(), - Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values())); - if (CollectionUtils.isEmpty(partitionList)) { - throw new AnalysisException("partitionList is empty, table name: " + getName()); - } - for (HivePartition hivePartition : partitionList) { - visibleVersionTime = hivePartition.getLastModifiedTime(); - if (visibleVersionTime > maxVersionTime) { - maxVersionTime = visibleVersionTime; - maxPartition = hivePartition; - } - } - return new MTMVMaxTimestampSnapshot(maxPartition.getPartitionName(getPartitionColumns()), maxVersionTime); - } - - private Long getPartitionIdByNameOrAnalysisException(String partitionName, - HiveMetaStoreCache.HivePartitionValues hivePartitionValues) - throws AnalysisException { - Long partitionId = hivePartitionValues.getPartitionNameToIdMap().get(partitionName); - if (partitionId == null) { - throw new AnalysisException("can not find partition: " + partitionName); - } - return partitionId; - } - - private HivePartition getHivePartitionByIdOrAnalysisException(Long partitionId, - HiveMetaStoreCache.HivePartitionValues hivePartitionValues, - HiveMetaStoreCache cache) throws AnalysisException { - List partitionValues = hivePartitionValues.getPartitionValuesMap().get(partitionId); - if (CollectionUtils.isEmpty(partitionValues)) { - throw new AnalysisException("can not find partitionValues: " + partitionId); - } - HivePartition partition = cache.getHivePartition(getDbName(), getName(), partitionValues); - if (partition == null) { - throw new AnalysisException("can not find partition: " + partitionId); - } - return partition; + return dlaTable.getTableSnapshot(context, snapshot); } @Override public boolean isPartitionColumnAllowNull() { - return true; + return dlaTable.isPartitionColumnAllowNull(); } /** @@ -1024,8 +973,7 @@ public boolean isPartitionedTable() { @Override public void beforeMTMVRefresh(MTMV mtmv) throws DdlException { - Env.getCurrentEnv().getRefreshManager() - .refreshTable(getCatalog().getName(), getDbName(), getName(), true); + dlaTable.beforeMTMVRefresh(mtmv); } public HoodieTableMetaClient getHudiClient() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java new file mode 100644 index 00000000000000..80ed43b76df061 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java @@ -0,0 +1,135 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.mtmv.MTMVMaxTimestampSnapshot; +import org.apache.doris.mtmv.MTMVRefreshContext; +import org.apache.doris.mtmv.MTMVSnapshotIf; +import org.apache.doris.mtmv.MTMVTimestampSnapshot; + +import com.google.common.collect.Lists; +import org.apache.commons.collections.CollectionUtils; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class HiveExternalTable extends HMSDlaTable { + + public HiveExternalTable(HMSExternalTable table) { + super(table); + } + + @Override + public PartitionType getPartitionType(Optional snapshot) { + return hmsTable.getPartitionType(); + } + + @Override + public Set getPartitionColumnNames(Optional snapshot) { + return hmsTable.getPartitionColumnNames(); + } + + @Override + public List getPartitionColumns(Optional snapshot) { + return hmsTable.getPartitionColumns(); + } + + @Override + public Map getAndCopyPartitionItems(Optional snapshot) { + return hmsTable.getNameToPartitionItems(); + } + + @Override + public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, + Optional snapshot) throws AnalysisException { + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + hmsTable.getDbName(), hmsTable.getName(), hmsTable.getPartitionColumnTypes()); + Long partitionId = getPartitionIdByNameOrAnalysisException(partitionName, hivePartitionValues); + HivePartition hivePartition = getHivePartitionByIdOrAnalysisException(partitionId, + hivePartitionValues, cache); + return new MTMVTimestampSnapshot(hivePartition.getLastModifiedTime()); + } + + @Override + public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) + throws AnalysisException { + if (hmsTable.getPartitionType() == PartitionType.UNPARTITIONED) { + return new MTMVMaxTimestampSnapshot(hmsTable.getName(), hmsTable.getLastDdlTime()); + } + HivePartition maxPartition = null; + long maxVersionTime = 0L; + long visibleVersionTime; + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + hmsTable.getDbName(), hmsTable.getName(), hmsTable.getPartitionColumnTypes()); + List partitionList = cache.getAllPartitionsWithCache(hmsTable.getDbName(), hmsTable.getName(), + Lists.newArrayList(hivePartitionValues.getPartitionValuesMap().values())); + if (CollectionUtils.isEmpty(partitionList)) { + throw new AnalysisException("partitionList is empty, table name: " + hmsTable.getName()); + } + for (HivePartition hivePartition : partitionList) { + visibleVersionTime = hivePartition.getLastModifiedTime(); + if (visibleVersionTime > maxVersionTime) { + maxVersionTime = visibleVersionTime; + maxPartition = hivePartition; + } + } + return new MTMVMaxTimestampSnapshot(maxPartition.getPartitionName( + hmsTable.getPartitionColumns()), maxVersionTime); + } + + private Long getPartitionIdByNameOrAnalysisException(String partitionName, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues) + throws AnalysisException { + Long partitionId = hivePartitionValues.getPartitionNameToIdMap().get(partitionName); + if (partitionId == null) { + throw new AnalysisException("can not find partition: " + partitionName); + } + return partitionId; + } + + private HivePartition getHivePartitionByIdOrAnalysisException(Long partitionId, + HiveMetaStoreCache.HivePartitionValues hivePartitionValues, + HiveMetaStoreCache cache) throws AnalysisException { + List partitionValues = hivePartitionValues.getPartitionValuesMap().get(partitionId); + if (CollectionUtils.isEmpty(partitionValues)) { + throw new AnalysisException("can not find partitionValues: " + partitionId); + } + HivePartition partition = cache.getHivePartition(hmsTable.getDbName(), hmsTable.getName(), partitionValues); + if (partition == null) { + throw new AnalysisException("can not find partition: " + partitionId); + } + return partition; + } + + @Override + public boolean isPartitionColumnAllowNull() { + return true; + } +} From 3d82fd601c677c16e771c5e59f20b53dfe86b51d Mon Sep 17 00:00:00 2001 From: TieweiFang Date: Mon, 6 Jan 2025 20:49:13 +0800 Subject: [PATCH 2/5] fix 2 --- .../java/org/apache/doris/datasource/hive/HMSExternalTable.java | 1 + 1 file changed, 1 insertion(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 13882264554529..3d37ec162fa2e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -157,6 +157,7 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private DLAType dlaType = DLAType.UNKNOWN; + @SerializedName(value = "dlaTable") private HMSDlaTable dlaTable; // record the event update time when enable hms event listener From 976bf436a3f2d97a30ad8cfc301a85e4a7761759 Mon Sep 17 00:00:00 2001 From: BePPPower Date: Tue, 7 Jan 2025 14:36:38 +0800 Subject: [PATCH 3/5] fix 3 --- .../doris/datasource/hive/HMSExternalTable.java | 11 +++++++++-- .../{HiveExternalTable.java => HiveDlaTable.java} | 4 ++-- 2 files changed, 11 insertions(+), 4 deletions(-) rename fe/fe-core/src/main/java/org/apache/doris/datasource/hive/{HiveExternalTable.java => HiveDlaTable.java} (98%) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 3d37ec162fa2e1..956f4e7cacccc6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -157,7 +157,6 @@ public class HMSExternalTable extends ExternalTable implements MTMVRelatedTableI private DLAType dlaType = DLAType.UNKNOWN; - @SerializedName(value = "dlaTable") private HMSDlaTable dlaTable; // record the event update time when enable hms event listener @@ -201,7 +200,7 @@ protected synchronized void makeSureInitialized() { dlaType = DLAType.HUDI; } else if (supportedHiveTable()) { dlaType = DLAType.HIVE; - dlaTable = new HiveExternalTable(this); + dlaTable = new HiveDlaTable(this); } else { // Should not reach here. Because `supportedHiveTable` will throw exception if not return true. throw new NotSupportedException("Unsupported dlaType for table: " + getNameWithFullQualifiers()); @@ -308,6 +307,7 @@ public List getPartitionColumns() { @Override public List getPartitionColumns(Optional snapshot) { + makeSureInitialized(); return dlaTable.getPartitionColumns(snapshot); } @@ -802,6 +802,7 @@ public Set getDistributionColumnNames() { @Override public PartitionType getPartitionType(Optional snapshot) { + makeSureInitialized(); return dlaTable.getPartitionType(snapshot); } @@ -811,6 +812,7 @@ public PartitionType getPartitionType() { @Override public Set getPartitionColumnNames(Optional snapshot) throws DdlException { + makeSureInitialized(); return dlaTable.getPartitionColumnNames(snapshot); } @@ -822,23 +824,27 @@ public Set getPartitionColumnNames() { @Override public Map getAndCopyPartitionItems(Optional snapshot) throws AnalysisException { + makeSureInitialized(); return dlaTable.getAndCopyPartitionItems(snapshot); } @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, Optional snapshot) throws AnalysisException { + makeSureInitialized(); return dlaTable.getPartitionSnapshot(partitionName, context, snapshot); } @Override public MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) throws AnalysisException { + makeSureInitialized(); return dlaTable.getTableSnapshot(context, snapshot); } @Override public boolean isPartitionColumnAllowNull() { + makeSureInitialized(); return dlaTable.isPartitionColumnAllowNull(); } @@ -974,6 +980,7 @@ public boolean isPartitionedTable() { @Override public void beforeMTMVRefresh(MTMV mtmv) throws DdlException { + makeSureInitialized(); dlaTable.beforeMTMVRefresh(mtmv); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java similarity index 98% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java rename to fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java index 80ed43b76df061..7636d211336f7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveDlaTable.java @@ -36,9 +36,9 @@ import java.util.Optional; import java.util.Set; -public class HiveExternalTable extends HMSDlaTable { +public class HiveDlaTable extends HMSDlaTable { - public HiveExternalTable(HMSExternalTable table) { + public HiveDlaTable(HMSExternalTable table) { super(table); } From 2aab2fccb49481ea4bcc546d575d59da5f3e18ba Mon Sep 17 00:00:00 2001 From: BePPPower Date: Tue, 7 Jan 2025 15:55:26 +0800 Subject: [PATCH 4/5] fix 4 --- .../doris/datasource/hive/HMSDlaTable.java | 18 +++++++++++++----- .../nereids/rules/analysis/BindRelation.java | 3 +++ .../org/apache/doris/qe/HmsQueryCacheTest.java | 5 +++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java index 879ad694753a03..dfc1eb65b8c111 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSDlaTable.java @@ -34,6 +34,19 @@ import java.util.Optional; import java.util.Set; +/** + * This abstract class represents a Hive Metastore (HMS) Dla Table and provides a blueprint for + * various operations related to metastore tables in Doris. + * + * Purpose: + * - To encapsulate common functionalities that HMS Dla tables should have for implementing other interfaces + * + * Why needed: + * - To provide a unified way to manage and interact with different kinds of Dla Table + * - To facilitate the implementation of multi-table materialized views (MTMV) by providing necessary + * methods for snapshot and partition management. + * - To abstract out the specific details of HMS table operations, making the code more modular and maintainable. + */ public abstract class HMSDlaTable implements MTMVBaseTableIf { HMSExternalTable hmsTable; @@ -41,7 +54,6 @@ public HMSDlaTable(HMSExternalTable table) { this.hmsTable = table; } - abstract Map getAndCopyPartitionItems(Optional snapshot) throws AnalysisException; @@ -57,10 +69,6 @@ abstract MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshCo abstract MTMVSnapshotIf getTableSnapshot(MTMVRefreshContext context, Optional snapshot) throws AnalysisException; - public boolean needAutoRefresh() { - return true; - } - abstract boolean isPartitionColumnAllowNull(); @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index d494f90c9cb804..a0e24e4a53889e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -431,6 +431,9 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio default: throw new AnalysisException("Unsupported tableType " + table.getType()); } + } catch (Exception e) { + System.out.println(e); + return null; } finally { if (!isView) { Optional sqlCacheContext = cascadesContext.getStatementContext().getSqlCacheContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 0a981dab8a9dcc..b1ab65c412ef88 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.hive.HiveDlaTable; import org.apache.doris.datasource.hive.source.HiveScanNode; import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; import org.apache.doris.planner.OlapScanNode; @@ -123,6 +124,7 @@ private void init(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(tbl, "catalog", hmsCatalog); Deencapsulation.setField(tbl, "dbName", "hms_db"); Deencapsulation.setField(tbl, "name", "hms_tbl"); + Deencapsulation.setField(tbl, "dlaTable", new HiveDlaTable(tbl)); new Expectations(tbl) { { tbl.getId(); @@ -173,6 +175,7 @@ private void init(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(tbl2, "catalog", hmsCatalog); Deencapsulation.setField(tbl2, "dbName", "hms_db"); Deencapsulation.setField(tbl2, "name", "hms_tbl2"); + Deencapsulation.setField(tbl2, "dlaTable", new HiveDlaTable(tbl2)); new Expectations(tbl2) { { tbl2.getId(); @@ -218,6 +221,7 @@ private void init(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view1, "objectCreated", true); + Deencapsulation.setField(view1, "dlaTable", new HiveDlaTable(view1)); new Expectations(view1) { { @@ -272,6 +276,7 @@ private void init(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view2, "objectCreated", true); + Deencapsulation.setField(view2, "dlaTable", new HiveDlaTable(view2)); new Expectations(view2) { { view2.getId(); From 47dc4c7c02614b9f89cc0256d1c7008815b9404f Mon Sep 17 00:00:00 2001 From: BePPPower Date: Tue, 7 Jan 2025 19:50:58 +0800 Subject: [PATCH 5/5] fix 5 --- .../org/apache/doris/nereids/rules/analysis/BindRelation.java | 3 --- .../java/org/apache/doris/external/hms/HmsCatalogTest.java | 2 ++ .../src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java | 2 -- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index a0e24e4a53889e..d494f90c9cb804 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -431,9 +431,6 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio default: throw new AnalysisException("Unsupported tableType " + table.getType()); } - } catch (Exception e) { - System.out.println(e); - return null; } finally { if (!isView) { Optional sqlCacheContext = cascadesContext.getStatementContext().getSqlCacheContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index 30a233dd1a999e..83d1de56f5b12c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -34,6 +34,7 @@ import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; +import org.apache.doris.datasource.hive.HiveDlaTable; import org.apache.doris.nereids.datasets.tpch.AnalyzeCheckTestBase; import org.apache.doris.qe.SessionVariable; @@ -107,6 +108,7 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(tbl, "catalog", hmsCatalog); Deencapsulation.setField(tbl, "dbName", "hms_db"); Deencapsulation.setField(tbl, "name", "hms_tbl"); + Deencapsulation.setField(tbl, "dlaTable", new HiveDlaTable(tbl)); new Expectations(tbl) { { tbl.getId(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index b1ab65c412ef88..35ac0367d4681c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -221,7 +221,6 @@ private void init(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view1, "objectCreated", true); - Deencapsulation.setField(view1, "dlaTable", new HiveDlaTable(view1)); new Expectations(view1) { { @@ -276,7 +275,6 @@ private void init(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view2, "objectCreated", true); - Deencapsulation.setField(view2, "dlaTable", new HiveDlaTable(view2)); new Expectations(view2) { { view2.getId();