diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml index 423efab0a2b5..a0a6f943322d 100644 --- a/.github/workflows/service_test_s3.yml +++ b/.github/workflows/service_test_s3.yml @@ -218,9 +218,6 @@ jobs: # This is the R2's limitation # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information OPENDAL_S3_BATCH_MAX_OPERATIONS: 700 - # This is the R2's limitation - # Refer to https://opendal.apache.org/docs/services/s3#compatible-services for more information - OPENDAL_S3_ENABLE_EXACT_BUF_WRITE: true java: runs-on: ubuntu-latest diff --git a/core/src/docs/upgrade.md b/core/src/docs/upgrade.md index 14552a4e46af..e061fdc2532b 100644 --- a/core/src/docs/upgrade.md +++ b/core/src/docs/upgrade.md @@ -6,6 +6,15 @@ OpenDAL bumps it's MSRV to 1.67.0. +### S3 Service Configuration + +- The `enable_exact_buf_write` option has been deprecated and is superseded by `BufferedWriter`, introduced in version 0.40. + +### Oss Service Configuration + +- The `write_min_size` option has been deprecated and replaced by `BufferedWriter`, also introduced in version 0.40. +- A new setting, `allow_anonymous`, has been added. Since v0.41, OSS will now return an error if credential loading fails. Enabling `allow_anonymous` to fallback to request without credentials. + # Upgrade to v0.41 There is no public API and raw API changes. diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 4f90ea9579a2..8ce0d65d899b 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -50,17 +50,16 @@ pub struct OssBuilder { presign_endpoint: Option, bucket: String, - // sse options + // OSS features server_side_encryption: Option, server_side_encryption_key_id: Option, + allow_anonymous: bool, // authenticate options access_key_id: Option, access_key_secret: Option, http_client: Option, - /// the size of each part, and the range is 5MB ~ 5 GB. - write_min_size: Option, /// batch_max_operations batch_max_operations: Option, } @@ -70,7 +69,8 @@ impl Debug for OssBuilder { let mut d = f.debug_struct("Builder"); d.field("root", &self.root) .field("bucket", &self.bucket) - .field("endpoint", &self.endpoint); + .field("endpoint", &self.endpoint) + .field("allow_anonymous", &self.allow_anonymous); d.finish_non_exhaustive() } @@ -233,20 +233,19 @@ impl OssBuilder { self } - /// set the minimum size of unsized write, it should be greater than 5 MB. - /// Reference: [OSS Multipart upload](https://www.alibabacloud.com/help/en/object-storage-service/latest/multipart-upload-6) - pub fn write_min_size(&mut self, write_min_size: usize) -> &mut Self { - self.write_min_size = Some(write_min_size); - - self - } - /// Set maximum batch operations of this backend. pub fn batch_max_operations(&mut self, batch_max_operations: usize) -> &mut Self { self.batch_max_operations = Some(batch_max_operations); self } + + /// Allow anonymous will allow opendal to send request without signing + /// when credential is not loaded. + pub fn allow_anonymous(&mut self) -> &mut Self { + self.allow_anonymous = true; + self + } } impl Builder for OssBuilder { @@ -268,10 +267,12 @@ impl Builder for OssBuilder { .map(|v| builder.server_side_encryption(v)); map.get("server_side_encryption_key_id") .map(|v| builder.server_side_encryption_key_id(v)); - map.get("write_min_size") - .map(|v| builder.write_min_size(v.parse::().unwrap())); map.get("batch_max_operations") .map(|v| builder.batch_max_operations(v.parse::().unwrap())); + map.get("allow_anonymous") + .filter(|v| *v == "on" || *v == "true") + .map(|_| builder.allow_anonymous()); + builder } @@ -355,6 +356,7 @@ impl Builder for OssBuilder { endpoint, host, presign_endpoint, + allow_anonymous: self.allow_anonymous, signer, loader, client, diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs index 25e870f99851..28a4936a5b6e 100644 --- a/core/src/services/oss/core.rs +++ b/core/src/services/oss/core.rs @@ -57,6 +57,7 @@ pub struct OssCore { pub host: String, pub endpoint: String, pub presign_endpoint: String, + pub allow_anonymous: bool, pub server_side_encryption: Option, pub server_side_encryption_key_id: Option, @@ -88,8 +89,16 @@ impl OssCore { if let Some(cred) = cred { Ok(Some(cred)) - } else { + } else if self.allow_anonymous { + // If allow_anonymous has been set, we will not sign the request. Ok(None) + } else { + // Mark this error as temporary since it could be caused by Aliyun STS. + Err(Error::new( + ErrorKind::PermissionDenied, + "no valid credential found, please check configuration or try again", + ) + .set_temporary()) } } diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 06010685100d..0216b19361be 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -91,7 +91,6 @@ pub struct S3Builder { default_storage_class: Option, enable_virtual_host_style: bool, batch_max_operations: Option, - enable_exact_buf_write: bool, http_client: Option, } @@ -517,16 +516,6 @@ impl S3Builder { self } - /// Enable exact buf write so that opendal will write data with exact size. - /// - /// This option is used for services like R2 which requires all parts must be the same size - /// except the last part. - pub fn enable_exact_buf_write(&mut self) -> &mut Self { - self.enable_exact_buf_write = true; - - self - } - /// Detect region of S3 bucket. /// /// # Args @@ -686,9 +675,6 @@ impl Builder for S3Builder { .map(|v: &String| builder.default_storage_class(v)); map.get("batch_max_operations") .map(|v| builder.batch_max_operations(v.parse().expect("input must be a number"))); - map.get("enable_exact_buf_write") - .filter(|v| *v == "on" || *v == "true") - .map(|_| builder.enable_exact_buf_write()); builder } @@ -868,7 +854,6 @@ impl Builder for S3Builder { server_side_encryption_customer_key_md5, default_storage_class, allow_anonymous: self.allow_anonymous, - enable_exact_buf_write: self.enable_exact_buf_write, signer, loader, client, diff --git a/core/src/services/s3/core.rs b/core/src/services/s3/core.rs index 3c809246c048..1021399e416e 100644 --- a/core/src/services/s3/core.rs +++ b/core/src/services/s3/core.rs @@ -79,7 +79,6 @@ pub struct S3Core { pub server_side_encryption_customer_key_md5: Option, pub default_storage_class: Option, pub allow_anonymous: bool, - pub enable_exact_buf_write: bool, pub signer: AwsV4Signer, pub loader: Box,