diff --git a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql index 5cc0a0ea685e37..7722d09463686e 100644 --- a/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql +++ b/docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/paimon/run01.sql @@ -55,4 +55,16 @@ insert into null_partition values(1,'bj'); insert into null_partition values(2,null); insert into null_partition values(3,NULL); insert into null_partition values(4,'null'); -insert into null_partition values(5,'NULL'); \ No newline at end of file +insert into null_partition values(5,'NULL'); + +drop table if exists date_partition; +CREATE TABLE date_partition ( + id BIGINT, + create_date DATE +) PARTITIONED BY (create_date) TBLPROPERTIES ( + 'primary-key' = 'create_date,id', + 'bucket'=10, + 'file.format'='orc' +); + +insert into date_partition values(1,date '2020-01-01'); \ No newline at end of file diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 7264d83621d8be..347e8df5c8f75e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -165,6 +165,9 @@ public Map getAndCopyPartitionItems(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return PartitionType.UNPARTITIONED; + } return getPartitionColumns(snapshot).size() > 0 ? PartitionType.LIST : PartitionType.UNPARTITIONED; } @@ -176,9 +179,17 @@ public Set getPartitionColumnNames(Optional snapshot) { @Override public List getPartitionColumns(Optional snapshot) { + if (isPartitionInvalid(snapshot)) { + return Collections.emptyList(); + } return getPaimonSchemaCacheValue(snapshot).getPartitionColumns(); } + private boolean isPartitionInvalid(Optional snapshot) { + PaimonSnapshotCacheValue paimonSnapshotCacheValue = getOrFetchSnapshotCacheValue(snapshot); + return paimonSnapshotCacheValue.getPartitionInfo().isPartitionInvalid(); + } + @Override public MTMVSnapshotIf getPartitionSnapshot(String partitionName, MTMVRefreshContext context, Optional snapshot) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 4d3326f8e48376..88515a2510d2c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -45,4 +45,9 @@ public Map getNameToPartitionItem() { public Map getNameToPartition() { return nameToPartition; } + + public boolean isPartitionInvalid() { + // when transfer to partitionItem failed, will not equal + return nameToPartitionItem.size() != nameToPartition.size(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index 1f7576dca51d93..b3df41bc5cef17 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -117,7 +117,7 @@ public static PaimonPartition rowToPartition(InternalRow row) { } public static PaimonPartitionInfo generatePartitionInfo(List partitionColumns, - List paimonPartitions) throws AnalysisException { + List paimonPartitions) { Map nameToPartitionItem = Maps.newHashMap(); Map nameToPartition = Maps.newHashMap(); PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(nameToPartitionItem, nameToPartition); @@ -127,7 +127,14 @@ public static PaimonPartitionInfo generatePartitionInfo(List partitionCo for (PaimonPartition paimonPartition : paimonPartitions) { String partitionName = getPartitionName(partitionColumns, paimonPartition.getPartitionValues()); nameToPartition.put(partitionName, paimonPartition); - nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + try { + // partition values return by paimon api, may have problem, + // to avoid affecting the query, we catch exceptions here + nameToPartitionItem.put(partitionName, toListPartitionItem(partitionName, partitionColumns)); + } catch (Exception e) { + LOG.warn("toListPartitionItem failed, partitionColumns: {}, partitionValues: {}", partitionColumns, + paimonPartition.getPartitionValues(), e); + } } return partitionInfo; } diff --git a/regression-test/data/mtmv_p0/test_paimon_mtmv.out b/regression-test/data/mtmv_p0/test_paimon_mtmv.out index ba6fc06c1d2491..0cfe4bd293cf51 100644 --- a/regression-test/data/mtmv_p0/test_paimon_mtmv.out +++ b/regression-test/data/mtmv_p0/test_paimon_mtmv.out @@ -140,3 +140,9 @@ true 4 null 5 NULL +-- !date_partition_base_table -- +1 2020-01-01 + +-- !date_partition -- +1 2020-01-01 + diff --git a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy index 48d63e03ec3db5..98477ab92d542e 100644 --- a/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_paimon_mtmv.groovy @@ -264,6 +264,36 @@ suite("test_paimon_mtmv", "p0,external,mtmv,external_docker,external_docker_dori order_qt_null_partition "SELECT * FROM ${mvName} " sql """drop materialized view if exists ${mvName};""" + // date type will has problem + order_qt_date_partition_base_table "SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition" + test { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (`create_date`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition; + """ + exception "Unable to find a suitable base table" + } + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS + SELECT * FROM ${catalogName}.`test_paimon_spark`.date_partition; + """ + sql """ + REFRESH MATERIALIZED VIEW ${mvName} auto; + """ + waitingMTMVTaskFinishedByMvName(mvName) + order_qt_date_partition "SELECT * FROM ${mvName} " + + sql """drop materialized view if exists ${mvName};""" sql """drop catalog if exists ${catalogName}""" }