fix PaimonScanNode and be ut

apache · Dec 2, 2024 · c2e27cb · c2e27cb
1 parent 45b4328
commit c2e27cb
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 16 deletions.
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -756,6 +756,21 @@ Status VFileScanner::_get_next_reader() {
         // JNI reader can only push down column value range
         bool push_down_predicates =
                 !_is_load && _params->format_type != TFileFormatType::FORMAT_JNI;
+        // for compatibility, this logic is deprecated in 3.1
+        if (format_type == TFileFormatType::FORMAT_JNI && range.__isset.table_format_params) {
+            if (range.table_format_params.table_format_type == "paimon" &&
+                !range.table_format_params.paimon_params.__isset.paimon_split) {
+                // use native reader
+                auto format = range.table_format_params.paimon_params.file_format;
+                if (format == "orc") {
+                    format_type = TFileFormatType::FORMAT_ORC;
+                } else if (format == "parquet") {
+                    format_type = TFileFormatType::FORMAT_PARQUET;
+                } else {
+                    return Status::InternalError("Not supported paimon file format: {}", format);
+                }
+            }
+        }
         bool need_to_get_parsed_schema = false;
         switch (format_type) {
         case TFileFormatType::FORMAT_JNI: {

diff --git a/be/test/vec/exec/vfile_scanner_exception_test.cpp b/be/test/vec/exec/vfile_scanner_exception_test.cpp
@@ -261,10 +261,10 @@ void VfileScannerExceptionTest::init() {
 
     _range_desc.start_offset = 0;
     _range_desc.size = 1000;
-    _range_desc.format_type = TFileFormatType::FORMAT_JNI;
     _ranges.push_back(_range_desc);
     _scan_range.ranges = _ranges;
     _scan_range.__isset.params = true;
+    _scan_range.params.format_type = TFileFormatType::FORMAT_JNI;
     _kv_cache.reset(new ShardedKVCache(48));
 
     _cluster_info.reset(new ClusterInfo());

diff --git a/be/test/vec/exec/vwal_scanner_test.cpp b/be/test/vec/exec/vwal_scanner_test.cpp
@@ -274,10 +274,10 @@ void VWalScannerTest::init() {
 
     _range_desc.start_offset = 0;
     _range_desc.size = 1000;
-    _range_desc.format_type = TFileFormatType::FORMAT_WAL;
     _ranges.push_back(_range_desc);
     _scan_range.ranges = _ranges;
     _scan_range.__isset.params = true;
+    _scan_range.params.format_type = TFileFormatType::FORMAT_WAL;
     _kv_cache.reset(new ShardedKVCache(48));
 
     _cluster_info.reset(new ClusterInfo());

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -141,19 +141,6 @@ public static <T> String encodeObjectToString(T t) {
     @Override
     protected void setScanParams(TFileRangeDesc rangeDesc, Split split) {
         if (split instanceof PaimonSplit) {
-            PaimonSplit paimonSplit = (PaimonSplit) split;
-            if (rangeDesc.getFormatType() == TFileFormatType.FORMAT_JNI
-                    && paimonSplit.getSplit() == null) {
-                // fall back to native reader
-                String fileFormat = getFileFormat(paimonSplit.getPathString());
-                if (fileFormat.equals("orc")) {
-                    rangeDesc.setFormatType(TFileFormatType.FORMAT_ORC);
-                } else if (fileFormat.equals("parquet")) {
-                    rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
-                } else {
-                    throw new RuntimeException("Unsupported file format: " + fileFormat);
-                }
-            }
             setPaimonParams(rangeDesc, (PaimonSplit) split);
         }
     }
@@ -168,11 +155,24 @@ private void setPaimonParams(TFileRangeDesc rangeDesc, PaimonSplit paimonSplit)
         tableFormatFileDesc.setTableFormatType(paimonSplit.getTableFormatType().value());
         TPaimonFileDesc fileDesc = new TPaimonFileDesc();
         org.apache.paimon.table.source.Split split = paimonSplit.getSplit();
+
+        String fileFormat = getFileFormat(paimonSplit.getPathString());
         if (split != null) {
             // use jni reader
+            rangeDesc.setFormatType(TFileFormatType.FORMAT_JNI);
             fileDesc.setPaimonSplit(encodeObjectToString(split));
+        } else {
+            // use native reader
+            if (fileFormat.equals("orc")) {
+                rangeDesc.setFormatType(TFileFormatType.FORMAT_ORC);
+            } else if (fileFormat.equals("parquet")) {
+                rangeDesc.setFormatType(TFileFormatType.FORMAT_PARQUET);
+            } else {
+                throw new RuntimeException("Unsupported file format: " + fileFormat);
+            }
         }
-        fileDesc.setFileFormat(getFileFormat(paimonSplit.getPathString()));
+
+        fileDesc.setFileFormat(fileFormat);
         fileDesc.setPaimonPredicate(encodeObjectToString(predicates));
         fileDesc.setPaimonColumnNames(source.getDesc().getSlots().stream().map(slot -> slot.getColumn().getName())
                 .collect(Collectors.joining(",")));