Merge branch 'main' into feat/service_add_dbfs_api_2.0

.github/workflows/bindings_java.yml

@@ -81,15 +81,3 @@ jobs:
         run: |
           ./mvnw clean install -DskipTests
           ./mvnw verify artifact:compare
-      - name: Behavior tests
-        working-directory: bindings/java
-        shell: bash
-        run: |
-          export OPENDAL_TEST=memory
-          export OPENDAL_MEMORY_ROOT=/opendal
-          ./mvnw test -Dtest="behavior.*Test"
-
-          export OPENDAL_TEST=fs
-          export OPENDAL_FS_ROOT=/tmp
-          ./mvnw test -Dtest="behavior.*Test"
-

bindings/java/src/blocking_operator.rs

@@ -20,12 +20,15 @@ use jni::objects::JClass;
 use jni::objects::JObject;
 use jni::objects::JString;
 use jni::sys::jobject;
+use jni::sys::jobjectArray;
+use jni::sys::jsize;
 use jni::sys::{jbyteArray, jlong};
 use jni::JNIEnv;
 
 use opendal::BlockingOperator;
 
 use crate::jstring_to_string;
+use crate::make_entry;
 use crate::make_metadata;
 use crate::Result;
 
@@ -221,3 +224,58 @@ fn intern_rename(
 
     Ok(op.rename(&source_path, &target_path)?)
 }
+
+/// # Safety
+///
+/// This function should not be called before the Operator are ready.
+#[no_mangle]
+pub unsafe extern "system" fn Java_org_apache_opendal_BlockingOperator_removeAll(
+    mut env: JNIEnv,
+    _: JClass,
+    op: *mut BlockingOperator,
+    path: JString,
+) {
+    intern_remove_all(&mut env, &mut *op, path).unwrap_or_else(|e| {
+        e.throw(&mut env);
+    })
+}
+
+fn intern_remove_all(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<()> {
+    let path = jstring_to_string(env, &path)?;
+
+    Ok(op.remove_all(&path)?)
+}
+
+/// # Safety
+///
+/// This function should not be called before the Operator are ready.
+#[no_mangle]
+pub unsafe extern "system" fn Java_org_apache_opendal_BlockingOperator_list(
+    mut env: JNIEnv,
+    _: JClass,
+    op: *mut BlockingOperator,
+    path: JString,
+) -> jobjectArray {
+    intern_list(&mut env, &mut *op, path).unwrap_or_else(|e| {
+        e.throw(&mut env);
+        JObject::default().into_raw()
+    })
+}
+
+fn intern_list(env: &mut JNIEnv, op: &mut BlockingOperator, path: JString) -> Result<jobjectArray> {
+    let path = jstring_to_string(env, &path)?;
+    let obs = op.list(&path)?;
+
+    let jarray = env.new_object_array(
+        obs.len() as jsize,
+        "org/apache/opendal/Entry",
+        JObject::null(),
+    )?;
+
+    for (idx, entry) in obs.iter().enumerate() {
+        let entry = make_entry(env, entry.to_owned())?;
+        env.set_object_array_element(&jarray, idx as jsize, entry)?;
+    }
+
+    Ok(jarray.into_raw())
+}

bindings/java/src/lib.rs

@@ -32,8 +32,10 @@ use jni::JavaVM;
 use once_cell::sync::OnceCell;
 use opendal::raw::PresignedRequest;
 use opendal::Capability;
+use opendal::Entry;
 use opendal::EntryMode;
 use opendal::Metadata;
+use opendal::Metakey;
 use opendal::OperatorInfo;
 use tokio::runtime::Builder;
 use tokio::runtime::Runtime;
@@ -233,31 +235,68 @@ fn make_metadata<'a>(env: &mut JNIEnv<'a>, metadata: Metadata) -> Result<JObject
         EntryMode::Unknown => 2,
     };
 
-    let last_modified = metadata.last_modified().map_or_else(
-        || Ok::<JObject<'_>, Error>(JObject::null()),
-        |v| {
-            Ok(env.new_object(
-                "java/util/Date",
-                "(J)V",
-                &[JValue::Long(v.timestamp_millis())],
-            )?)
-        },
-    )?;
+    let metakey = metadata.metakey();
+
+    let contains_metakey = |k| metakey.contains(k) || metakey.contains(Metakey::Complete);
+
+    let last_modified = if contains_metakey(Metakey::LastModified) {
+        metadata.last_modified().map_or_else(
+            || Ok::<JObject<'_>, Error>(JObject::null()),
+            |v| {
+                Ok(env.new_object(
+                    "java/util/Date",
+                    "(J)V",
+                    &[JValue::Long(v.timestamp_millis())],
+                )?)
+            },
+        )?
+    } else {
+        JObject::null()
+    };
 
-    let cache_control = string_to_jstring(env, metadata.cache_control())?;
-    let content_disposition = string_to_jstring(env, metadata.content_disposition())?;
-    let content_md5 = string_to_jstring(env, metadata.content_md5())?;
-    let content_type = string_to_jstring(env, metadata.content_type())?;
-    let etag = string_to_jstring(env, metadata.etag())?;
-    let version = string_to_jstring(env, metadata.version())?;
+    let cache_control = if contains_metakey(Metakey::CacheControl) {
+        string_to_jstring(env, metadata.cache_control())?
+    } else {
+        JObject::null()
+    };
+    let content_disposition = if contains_metakey(Metakey::ContentDisposition) {
+        string_to_jstring(env, metadata.content_disposition())?
+    } else {
+        JObject::null()
+    };
+    let content_md5 = if contains_metakey(Metakey::ContentMd5) {
+        string_to_jstring(env, metadata.content_md5())?
+    } else {
+        JObject::null()
+    };
+    let content_type = if contains_metakey(Metakey::ContentType) {
+        string_to_jstring(env, metadata.content_type())?
+    } else {
+        JObject::null()
+    };
+    let etag = if contains_metakey(Metakey::Etag) {
+        string_to_jstring(env, metadata.etag())?
+    } else {
+        JObject::null()
+    };
+    let version = if contains_metakey(Metakey::Version) {
+        string_to_jstring(env, metadata.version())?
+    } else {
+        JObject::null()
+    };
+    let content_length = if contains_metakey(Metakey::ContentLength) {
+        metadata.content_length() as jlong
+    } else {
+        -1
+    };
 
     let result = env
         .new_object(
             "org/apache/opendal/Metadata",
             "(IJLjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/util/Date;Ljava/lang/String;)V",
             &[
                 JValue::Int(mode as jint),
-                JValue::Long(metadata.content_length() as jlong),
+                JValue::Long(content_length),
                 JValue::Object(&content_disposition),
                 JValue::Object(&content_md5),
                 JValue::Object(&content_type),
@@ -270,6 +309,17 @@ fn make_metadata<'a>(env: &mut JNIEnv<'a>, metadata: Metadata) -> Result<JObject
     Ok(result)
 }
 
+fn make_entry<'a>(env: &mut JNIEnv<'a>, entry: Entry) -> Result<JObject<'a>> {
+    let path = env.new_string(entry.path())?;
+    let metadata = make_metadata(env, entry.metadata().to_owned())?;
+
+    Ok(env.new_object(
+        "org/apache/opendal/Entry",
+        "(Ljava/lang/String;Lorg/apache/opendal/Metadata;)V",
+        &[JValue::Object(&path), JValue::Object(&metadata)],
+    )?)
+}
+
 fn string_to_jstring<'a>(env: &mut JNIEnv<'a>, s: Option<&str>) -> Result<JObject<'a>> {
     s.map_or_else(
         || Ok(JObject::null()),

bindings/java/src/main/java/org/apache/opendal/BlockingOperator.java

@@ -20,6 +20,8 @@
 package org.apache.opendal;
 
 import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
 
 /**
@@ -92,6 +94,14 @@ public void rename(String sourcePath, String targetPath) {
         rename(nativeHandle, sourcePath, targetPath);
     }
 
+    public void removeAll(String path) {
+        removeAll(nativeHandle, path);
+    }
+
+    public List<Entry> list(String path) {
+        return Arrays.asList(list(nativeHandle, path));
+    }
+
     @Override
     protected native void disposeInternal(long handle);
 
@@ -110,4 +120,8 @@ public void rename(String sourcePath, String targetPath) {
     private static native long copy(long nativeHandle, String sourcePath, String targetPath);
 
     private static native long rename(long nativeHandle, String sourcePath, String targetPath);
+
+    private static native void removeAll(long nativeHandle, String path);
+
+    private static native Entry[] list(long nativeHandle, String path);
 }

bindings/java/src/main/java/org/apache/opendal/Entry.java

@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.opendal;
+
+import lombok.Data;
+
+@Data
+public class Entry {
+    public final String path;
+    public final Metadata metadata;
+
+    public Entry(String path, Metadata metadata) {
+        this.path = path;
+        this.metadata = metadata;
+    }
+}

bindings/java/src/main/java/org/apache/opendal/Metadata.java

@@ -27,14 +27,60 @@
  */
 @Data
 public class Metadata {
+    /**
+     * Mode of the entry.
+     */
     public final EntryMode mode;
+    /**
+     * Content Length of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be -1.
+     */
     public final long contentLength;
+    /**
+     * Content-Disposition of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String contentDisposition;
+    /**
+     * Content MD5 of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String contentMd5;
+    /**
+     * Content Type of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String contentType;
+    /**
+     * Cache Control of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String cacheControl;
+    /**
+     * Etag of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String etag;
+    /**
+     * Last Modified of the entry.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final Date lastModified;
+    /**
+     * Version of the entry.
+     * Version is a string that can be used to identify the version of this entry.
+     * This field may come out from the version control system, like object
+     * versioning in AWS S3.
+     *
+     * Note: For now, this value is only available when calling on result of `stat`, otherwise it will be null.
+     */
     public final String version;
 
     public Metadata(
@@ -67,11 +113,17 @@ public boolean isDir() {
     }
 
     public enum EntryMode {
-        /// FILE means the path has data to read.
+        /**
+         * FILE means the path has data to read.
+         */
         FILE,
-        /// DIR means the path can be listed.
+        /**
+         * DIR means the path can be listed.
+         */
         DIR,
-        /// Unknown means we don't know what we can do on this path.
+        /**
+         * Unknown means we don't know what we can do on this path.
+         */
         UNKNOWN;
 
         public static EntryMode of(int mode) {

bindings/java/src/main/java/org/apache/opendal/Operator.java

@@ -21,7 +21,10 @@
 
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
@@ -205,6 +208,17 @@ public CompletableFuture<Void> rename(String sourcePath, String targetPath) {
         return AsyncRegistry.take(requestId);
     }
 
+    public CompletableFuture<Void> removeAll(String path) {
+        final long requestId = removeAll(nativeHandle, path);
+        return AsyncRegistry.take(requestId);
+    }
+
+    public CompletableFuture<List<Entry>> list(String path) {
+        final long requestid = list(nativeHandle, path);
+        final CompletableFuture<Entry[]> result = AsyncRegistry.take(requestid);
+        return Objects.requireNonNull(result).thenApplyAsync(Arrays::asList);
+    }
+
     @Override
     protected native void disposeInternal(long handle);
 
@@ -237,4 +251,8 @@ public CompletableFuture<Void> rename(String sourcePath, String targetPath) {
     private static native long copy(long nativeHandle, String sourcePath, String targetPath);
 
     private static native long rename(long nativeHandle, String sourcePath, String targetPath);
+
+    private static native long removeAll(long nativeHandle, String path);
+
+    private static native long list(long nativeHandle, String path);
 }