feat(services/oss): Add allow anonymous support (#3321)

Signed-off-by: Xuanwo <[email protected]>

.github/workflows/service_test_s3.yml

@@ -218,9 +218,6 @@ jobs:
           # This is the R2's limitation
           # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
           OPENDAL_S3_BATCH_MAX_OPERATIONS: 700
-          # This is the R2's limitation
-          # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information
-          OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true
 
   java:
     runs-on: ubuntu-latest

core/src/docs/upgrade.md

@@ -6,6 +6,15 @@
 
 OpenDAL bumps it's MSRV to 1.67.0.
 
+### S3 Service Configuration
+
+- The `enable_exact_buf_write` option has been deprecated and is superseded by `BufferedWriter`, introduced in version 0.40.
+
+### Oss Service Configuration
+
+- The `write_min_size` option has been deprecated and replaced by `BufferedWriter`, also introduced in version 0.40.
+- A new setting, `allow_anonymous`, has been added. Since v0.41, OSS will now return an error if credential loading fails. Enabling `allow_anonymous` to fallback to request without credentials.
+
 # Upgrade to v0.41
 
 There is no public API and raw API changes.

core/src/services/oss/backend.rs

@@ -50,17 +50,16 @@ pub struct OssBuilder {
     presign_endpoint: Option<String>,
     bucket: String,
 
-    // sse options
+    // OSS features
     server_side_encryption: Option<String>,
     server_side_encryption_key_id: Option<String>,
+    allow_anonymous: bool,
 
     // authenticate options
     access_key_id: Option<String>,
     access_key_secret: Option<String>,
 
     http_client: Option<HttpClient>,
-    /// the size of each part, and the range is 5MB ~ 5 GB.
-    write_min_size: Option<usize>,
     /// batch_max_operations
     batch_max_operations: Option<usize>,
 }
@@ -70,7 +69,8 @@ impl Debug for OssBuilder {
         let mut d = f.debug_struct("Builder");
         d.field("root", &self.root)
             .field("bucket", &self.bucket)
-            .field("endpoint", &self.endpoint);
+            .field("endpoint", &self.endpoint)
+            .field("allow_anonymous", &self.allow_anonymous);
 
         d.finish_non_exhaustive()
     }
@@ -233,20 +233,19 @@ impl OssBuilder {
         self
     }
 
-    /// set the minimum size of unsized write, it should be greater than 5 MB.
-    /// Reference: [OSS Multipart upload](https://www.alibabacloud.com/help/en/object-storage-service/latest/multipart-upload-6)
-    pub fn write_min_size(&mut self, write_min_size: usize) -> &mut Self {
-        self.write_min_size = Some(write_min_size);
-
-        self
-    }
-
     /// Set maximum batch operations of this backend.
     pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self {
         self.batch_max_operations = Some(batch_max_operations);
 
         self
     }
+
+    /// Allow anonymous will allow opendal to send request without signing
+    /// when credential is not loaded.
+    pub fn allow_anonymous(&mut self) -> &mut Self {
+        self.allow_anonymous = true;
+        self
+    }
 }
 
 impl Builder for OssBuilder {
@@ -268,10 +267,12 @@ impl Builder for OssBuilder {
             .map(|v| builder.server_side_encryption(v));
         map.get("server_side_encryption_key_id")
             .map(|v| builder.server_side_encryption_key_id(v));
-        map.get("write_min_size")
-            .map(|v| builder.write_min_size(v.parse::<usize>().unwrap()));
         map.get("batch_max_operations")
             .map(|v| builder.batch_max_operations(v.parse::<usize>().unwrap()));
+        map.get("allow_anonymous")
+            .filter(|v| *v == "on" || *v == "true")
+            .map(|_| builder.allow_anonymous());
+
         builder
     }
 
@@ -355,6 +356,7 @@ impl Builder for OssBuilder {
                 endpoint,
                 host,
                 presign_endpoint,
+                allow_anonymous: self.allow_anonymous,
                 signer,
                 loader,
                 client,

core/src/services/oss/core.rs

@@ -57,6 +57,7 @@ pub struct OssCore {
     pub host: String,
     pub endpoint: String,
     pub presign_endpoint: String,
+    pub allow_anonymous: bool,
 
     pub server_side_encryption: Option<HeaderValue>,
     pub server_side_encryption_key_id: Option<HeaderValue>,
@@ -88,8 +89,16 @@ impl OssCore {
 
         if let Some(cred) = cred {
             Ok(Some(cred))
-        } else {
+        } else if self.allow_anonymous {
+            // If allow_anonymous has been set, we will not sign the request.
             Ok(None)
+        } else {
+            // Mark this error as temporary since it could be caused by Aliyun STS.
+            Err(Error::new(
+                ErrorKind::PermissionDenied,
+                "no valid credential found, please check configuration or try again",
+            )
+            .set_temporary())
         }
     }
 

core/src/services/s3/backend.rs

@@ -91,7 +91,6 @@ pub struct S3Builder {
     default_storage_class: Option<String>,
     enable_virtual_host_style: bool,
     batch_max_operations: Option<usize>,
-    enable_exact_buf_write: bool,
 
     http_client: Option<HttpClient>,
 }
@@ -517,16 +516,6 @@ impl S3Builder {
         self
     }
 
-    /// Enable exact buf write so that opendal will write data with exact size.
-    ///
-    /// This option is used for services like R2 which requires all parts must be the same size
-    /// except the last part.
-    pub fn enable_exact_buf_write(&mut self) -> &mut Self {
-        self.enable_exact_buf_write = true;
-
-        self
-    }
-
     /// Detect region of S3 bucket.
     ///
     /// # Args
@@ -686,9 +675,6 @@ impl Builder for S3Builder {
             .map(|v: &String| builder.default_storage_class(v));
         map.get("batch_max_operations")
             .map(|v| builder.batch_max_operations(v.parse().expect("input must be a number")));
-        map.get("enable_exact_buf_write")
-            .filter(|v| *v == "on" || *v == "true")
-            .map(|_| builder.enable_exact_buf_write());
 
         builder
     }
@@ -868,7 +854,6 @@ impl Builder for S3Builder {
                 server_side_encryption_customer_key_md5,
                 default_storage_class,
                 allow_anonymous: self.allow_anonymous,
-                enable_exact_buf_write: self.enable_exact_buf_write,
                 signer,
                 loader,
                 client,

core/src/services/s3/core.rs

@@ -79,7 +79,6 @@ pub struct S3Core {
     pub server_side_encryption_customer_key_md5: Option<HeaderValue>,
     pub default_storage_class: Option<HeaderValue>,
     pub allow_anonymous: bool,
-    pub enable_exact_buf_write: bool,
 
     pub signer: AwsV4Signer,
     pub loader: Box<dyn AwsCredentialLoad>,