Revert "[Enhancement](MaxCompute)Refactoring maxCompute catalog using…

… Storage API.(apache#40225 , apache#40888 ,apache#41386 ) (apache#41610)" This reverts commit 8c0f73c.
hubgeter · Nov 20, 2024 · edab727 · edab727
1 parent 643301f
commit edab727
Show file tree

Hide file tree

Showing 33 changed files with 1,022 additions and 2,972 deletions.
diff --git a/be/src/runtime/descriptors.cpp b/be/src/runtime/descriptors.cpp
@@ -189,21 +189,7 @@ MaxComputeTableDescriptor::MaxComputeTableDescriptor(const TTableDescriptor& tde
           _tunnel_url(tdesc.mcTable.tunnel_url),
           _access_key(tdesc.mcTable.access_key),
           _secret_key(tdesc.mcTable.secret_key),
-          _public_access(tdesc.mcTable.public_access) {
-    if (tdesc.mcTable.__isset.endpoint) {
-        _endpoint = tdesc.mcTable.endpoint;
-    } else {
-        _init_status = Status::InvalidArgument(
-                "fail to init MaxComputeTableDescriptor, missing endpoint.");
-    }
-
-    if (tdesc.mcTable.__isset.quota) {
-        _quota = tdesc.mcTable.quota;
-    } else {
-        _init_status =
-                Status::InvalidArgument("fail to init MaxComputeTableDescriptor, missing quota.");
-    }
-}
+          _public_access(tdesc.mcTable.public_access) {}
 
 MaxComputeTableDescriptor::~MaxComputeTableDescriptor() = default;
 

diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
@@ -214,22 +214,16 @@ class MaxComputeTableDescriptor : public TableDescriptor {
     std::string access_key() const { return _access_key; }
     std::string secret_key() const { return _secret_key; }
     std::string public_access() const { return _public_access; }
-    std::string endpoint() const { return _endpoint; }
-    std::string quota() const { return _quota; }
-    Status init_status() const { return _init_status; }
 
 private:
-    std::string _region; //deprecated
+    std::string _region;
     std::string _project;
     std::string _table;
-    std::string _odps_url;   //deprecated
-    std::string _tunnel_url; //deprecated
+    std::string _odps_url;
+    std::string _tunnel_url;
     std::string _access_key;
     std::string _secret_key;
-    std::string _public_access; //deprecated
-    std::string _endpoint;
-    std::string _quota;
-    Status _init_status = Status::OK();
+    std::string _public_access;
 };
 
 class EsTableDescriptor : public TableDescriptor {

diff --git a/be/src/vec/exec/format/table/max_compute_jni_reader.cpp b/be/src/vec/exec/format/table/max_compute_jni_reader.cpp
@@ -63,21 +63,19 @@ MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_des
         }
         index++;
     }
-    std::map<String, String> params = {
-            {"access_key", _table_desc->access_key()},
-            {"secret_key", _table_desc->secret_key()},
-            {"endpoint", _table_desc->endpoint()},
-            {"quota", _table_desc->quota()},
-            {"project", _table_desc->project()},
-            {"table", _table_desc->table()},
-
-            {"session_id", _max_compute_params.session_id},
-            {"scan_serializer", _max_compute_params.table_batch_read_session},
-
-            {"start_offset", std::to_string(_range.start_offset)},
-            {"split_size", std::to_string(_range.size)},
-            {"required_fields", required_fields.str()},
-            {"columns_types", columns_types.str()}};
+    std::map<String, String> params = {{"region", _table_desc->region()},
+                                       {"odps_url", _table_desc->odps_url()},
+                                       {"tunnel_url", _table_desc->tunnel_url()},
+                                       {"access_key", _table_desc->access_key()},
+                                       {"secret_key", _table_desc->secret_key()},
+                                       {"project", _table_desc->project()},
+                                       {"partition_spec", _max_compute_params.partition_spec},
+                                       {"table", _table_desc->table()},
+                                       {"public_access", _table_desc->public_access()},
+                                       {"start_offset", std::to_string(_range.start_offset)},
+                                       {"split_size", std::to_string(_range.size)},
+                                       {"required_fields", required_fields.str()},
+                                       {"columns_types", columns_types.str()}};
     _jni_connector = std::make_unique<JniConnector>(
             "org/apache/doris/maxcompute/MaxComputeJniScanner", params, column_names);
 }

diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp
@@ -81,7 +81,6 @@ Status JniConnector::open(RuntimeState* state, RuntimeProfile* profile) {
     }
     RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env));
     SCOPED_TIMER(_open_scanner_time);
-    _scanner_params.emplace("time_zone", _state->timezone_obj().name());
     RETURN_IF_ERROR(_init_jni_scanner(env, batch_size));
     // Call org.apache.doris.common.jni.JniScanner#open
     env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_open);

diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -802,9 +802,6 @@ Status VFileScanner::_get_next_reader() {
                 range.table_format_params.table_format_type == "max_compute") {
                 const auto* mc_desc = static_cast<const MaxComputeTableDescriptor*>(
                         _real_tuple_desc->table_desc());
-                if (!mc_desc->init_status()) {
-                    return mc_desc->init_status();
-                }
                 std::unique_ptr<MaxComputeJniReader> mc_reader = MaxComputeJniReader::create_unique(
                         mc_desc, range.table_format_params.max_compute_params, _file_slot_descs,
                         range, _state, _profile);

diff --git a/conf/be.conf b/conf/be.conf
@@ -24,10 +24,10 @@ LOG_DIR="${DORIS_HOME}/log/"
 JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xloggc:$DORIS_HOME/log/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
 
 # For jdk 9+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_9="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.nio=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_9="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$DORIS_HOME/log/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives"
 
 # For jdk 17+, this JAVA_OPTS will be used as default JVM options
-JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED"
+JAVA_OPTS_FOR_JDK_17="-Dfile.encoding=UTF-8 -Xmx2048m -DlogPath=$LOG_DIR/jni.log -Xlog:gc:$LOG_DIR/be.gc.log.$CUR_DATE -Djavax.security.auth.useSubjectCredsOnly=false -Dsun.security.krb5.debug=true -Dsun.java.command=DorisBE -XX:-CriticalJNINatives --add-opens=java.base/java.net=ALL-UNNAMED"
 
 # since 1.2, the JAVA_HOME need to be set to run BE process.
 # JAVA_HOME=/path/to/jdk/

diff --git a/...ava-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java b/...ava-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java
@@ -59,18 +59,6 @@ public interface ColumnValue {
 
     LocalDate getDate();
 
-    default String getChar() {
-        return getString();
-    }
-
-    default byte[] getCharAsBytes() {
-        return getStringAsBytes();
-    }
-
-    default boolean canGetCharAsBytes() {
-        return canGetStringAsBytes();
-    }
-
     LocalDateTime getDateTime();
 
     byte[] getBytes();

diff --git a/...va-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java b/...va-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java
@@ -1527,12 +1527,6 @@ public void appendValue(ColumnValue o) {
                 appendDateTime(o.getDateTime());
                 break;
             case CHAR:
-                if (o.canGetCharAsBytes()) {
-                    appendBytesAndOffset(o.getCharAsBytes());
-                } else {
-                    appendStringAndOffset(o.getChar());
-                }
-                break;
             case VARCHAR:
             case STRING:
                 if (o.canGetStringAsBytes()) {

diff --git a/fe/be-java-extensions/max-compute-scanner/pom.xml b/fe/be-java-extensions/max-compute-scanner/pom.xml
@@ -43,7 +43,6 @@ under the License.
         <dependency>
             <groupId>com.aliyun.odps</groupId>
             <artifactId>odps-sdk-core</artifactId>
-            <version>${maxcompute.version}</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.codehaus.jackson</groupId>
@@ -55,11 +54,6 @@ under the License.
                 </exclusion>
             </exclusions>
         </dependency>
-        <dependency>
-            <groupId>com.aliyun.odps</groupId>
-            <artifactId>odps-sdk-table-api</artifactId>
-            <version>${maxcompute.version}</version>
-        </dependency>
         <dependency>
             <groupId>org.apache.arrow</groupId>
             <artifactId>arrow-vector</artifactId>