diff --git a/.github/services/azfile/azfile/action.yml b/.github/services/azfile/azfile/action.yml new file mode 100644 index 000000000000..d2dc23cad5d9 --- /dev/null +++ b/.github/services/azfile/azfile/action.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: azfile +description: 'Behavior test for Azfile' + +runs: + using: "composite" + steps: + - name: Setup + uses: 1password/load-secrets-action@v1 + with: + export-env: true + env: + OPENDAL_AZFILE_ACCOUNT_NAME: op://services/azfile/account_name + OPENDAL_AZFILE_ENDPOINT: op://services/azfile/endpoint + OPENDAL_AZFILE_ACCOUNT_KEY: op://services/azfile/account_key + OPENDAL_AZFILE_SHARE_NAME: op://services/azfile/share_name + + - name: Add extra settings + shell: bash + run: | + cat << EOF >> $GITHUB_ENV + OPENDAL_AZFILE_ROOT=test/ + EOF diff --git a/bindings/java/Cargo.toml b/bindings/java/Cargo.toml index df1ca7e66dd2..c5e7707b96bf 100644 --- a/bindings/java/Cargo.toml +++ b/bindings/java/Cargo.toml @@ -82,6 +82,7 @@ services-all = [ "services-wasabi", "services-mongodb", "services-sqlite", + "services-azfile", ] # Default services provided by opendal. @@ -129,6 +130,7 @@ services-wasabi = ["opendal/services-wasabi"] services-mysql = ["opendal/services-mysql"] services-mongodb = ["opendal/services-mongodb"] services-sqlite = ["opendal/services-sqlite"] +services-azfile = ["opendal/services-azfile"] [dependencies] anyhow = "1.0.71" diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index c7d8ad41cb91..75e475699e97 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -195,6 +195,7 @@ impl Builder for AzfileBuilder { let config_loader = AzureStorageConfig { account_name: Some(account_name), + account_key: self.account_key.clone(), sas_token: self.sas_token.clone(), ..Default::default() }; @@ -289,6 +290,11 @@ impl Accessor for AzfileBackend { Ok(RpCreateDir::default()) } _ => { + // we cannot just check status code because 409 Conflict has two meaning: + // 1. If a directory by the same name is being deleted when Create Directory is called, the server returns status code 409 (Conflict) + // 2. If a directory or file with the same name already exists, the operation fails with status code 409 (Conflict). + // but we just need case 2 (already exists) + // ref: https://learn.microsoft.com/en-us/rest/api/storageservices/create-directory if resp .headers() .get("x-ms-error-code") @@ -380,7 +386,7 @@ impl Accessor for AzfileBackend { let status = resp.status(); match status { - StatusCode::ACCEPTED => { + StatusCode::ACCEPTED | StatusCode::NOT_FOUND => { resp.into_body().consume().await?; Ok(RpDelete::default()) } diff --git a/core/src/services/azfile/core.rs b/core/src/services/azfile/core.rs index ce9717ac5536..84fdc8a9faf3 100644 --- a/core/src/services/azfile/core.rs +++ b/core/src/services/azfile/core.rs @@ -38,9 +38,10 @@ use crate::*; const X_MS_VERSION: &str = "x-ms-version"; const X_MS_WRITE: &str = "x-ms-write"; -const X_MS_RENAME_SOURCE: &str = "x-ms-rename-source"; +const X_MS_FILE_RENAME_SOURCE: &str = "x-ms-file-rename-source"; const X_MS_CONTENT_LENGTH: &str = "x-ms-content-length"; const X_MS_TYPE: &str = "x-ms-type"; +const X_MS_FILE_RENAME_REPLACE_IF_EXISTS: &str = "x-ms-file-rename-replace-if-exists"; pub struct AzfileCore { pub root: String, @@ -111,7 +112,19 @@ impl AzfileCore { let mut req = Request::get(&url); - req = req.header(RANGE, range.to_header()); + if !range.is_full() { + // azfile doesn't support read with suffix range. + // + // ref: https://learn.microsoft.com/en-us/rest/api/storageservices/specifying-the-range-header-for-file-service-operations + if range.offset().is_none() && range.size().is_some() { + return Err(Error::new( + ErrorKind::Unsupported, + "azblob doesn't support read with suffix range", + )); + } + + req = req.header(RANGE, range.to_header()); + } let mut req = req .body(AsyncBody::Empty) @@ -262,14 +275,14 @@ impl AzfileCore { "{}/{}/{}?restype=directory&comp=rename", self.endpoint, self.share_name, - percent_encode_path(&p) + percent_encode_path(&new_p) ) } else { format!( "{}/{}/{}?comp=rename", self.endpoint, self.share_name, - percent_encode_path(&p) + percent_encode_path(&new_p) ) }; @@ -277,7 +290,21 @@ impl AzfileCore { req = req.header(CONTENT_LENGTH, 0); - req = req.header(X_MS_RENAME_SOURCE, percent_encode_path(&new_p)); + // x-ms-file-rename-source specifies the file or directory to be renamed. + // the value must be a URL style path + // the official document does not mention the URL style path + // find the solution from the community FAQ and implementation of the Java-SDK + // ref: https://learn.microsoft.com/en-us/answers/questions/799611/azure-file-service-rest-api(rename)?page=1 + let source_url = format!( + "{}/{}/{}", + self.endpoint, + self.share_name, + percent_encode_path(&p) + ); + + req = req.header(X_MS_FILE_RENAME_SOURCE, &source_url); + + req = req.header(X_MS_FILE_RENAME_REPLACE_IF_EXISTS, "true"); let mut req = req .body(AsyncBody::Empty) diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index c7976e2c7b58..211444064581 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -159,6 +159,8 @@ impl Operator { Scheme::Azblob => Self::from_map::(map)?.finish(), #[cfg(feature = "services-azdls")] Scheme::Azdls => Self::from_map::(map)?.finish(), + #[cfg(feature = "services-azfile")] + Scheme::Azfile => Self::from_map::(map)?.finish(), #[cfg(feature = "services-cacache")] Scheme::Cacache => Self::from_map::(map)?.finish(), #[cfg(feature = "services-cos")] diff --git a/core/src/types/scheme.rs b/core/src/types/scheme.rs index 879dc6b9e19a..32b84c04c5a9 100644 --- a/core/src/types/scheme.rs +++ b/core/src/types/scheme.rs @@ -165,6 +165,8 @@ impl Scheme { Scheme::Azblob, #[cfg(feature = "services-azdls")] Scheme::Azdls, + #[cfg(feature = "services-azfile")] + Scheme::Azfile, #[cfg(feature = "services-cacache")] Scheme::Cacache, #[cfg(feature = "services-cos")]