diff --git a/core/src/raw/http_util/header.rs b/core/src/raw/http_util/header.rs index 1cb9e1f4c3f9..87748c96f4db 100644 --- a/core/src/raw/http_util/header.rs +++ b/core/src/raw/http_util/header.rs @@ -32,7 +32,6 @@ use http::HeaderMap; use http::HeaderName; use http::HeaderValue; use md5::Digest; -use std::collections::HashMap; use crate::raw::*; use crate::EntryMode; @@ -190,45 +189,6 @@ pub fn parse_into_metadata(path: &str, headers: &HeaderMap) -> Result Ok(m) } -/// parse_metadata will parse http headers(including standards http headers -/// and user defined metadata header) into Metadata. -/// -/// # Arguments -/// -/// * `user_metadata_prefix` is the prefix of user defined metadata key -/// -/// # Notes -/// -/// before return the user defined metadata, we'll strip the user_metadata_prefix from the key -pub fn parse_metadata( - path: &str, - user_metadata_prefix: &str, - headers: &HeaderMap, -) -> Result { - let mut m = parse_into_metadata(path, headers)?; - - let data: HashMap = headers - .iter() - .filter_map(|(key, _)| { - if key.as_str().starts_with(user_metadata_prefix) { - if let Ok(Some(value)) = parse_header_to_str(headers, key) { - let key_str = key.to_string(); - let stripped_key = key_str - .strip_prefix(user_metadata_prefix) - .expect("strip prefix must succeed"); - return Some((stripped_key.to_string(), value.to_string())); - } - } - None - }) - .collect(); - if !data.is_empty() { - m.with_user_metadata(data); - } - - Ok(m) -} - /// format content md5 header by given input. pub fn format_content_md5(bs: &[u8]) -> String { let mut hasher = md5::Md5::new(); diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs index 7ce12c737b12..965ea0bd494f 100644 --- a/core/src/raw/http_util/mod.rs +++ b/core/src/raw/http_util/mod.rs @@ -44,7 +44,6 @@ pub use header::parse_header_to_str; pub use header::parse_into_metadata; pub use header::parse_last_modified; pub use header::parse_location; -pub use header::parse_metadata; mod uri; pub use uri::percent_decode_path; diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index a159f5f649be..61e548bf4573 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -449,7 +449,8 @@ impl Access for OssBackend { StatusCode::OK => { let headers = resp.headers(); let mut meta = - parse_metadata(path, constants::USER_METADATA_PREFIX, resp.headers())?; + self.core + .parse_metadata(path, constants::X_OSS_META_PREFIX, resp.headers())?; if let Some(v) = parse_header_to_str(headers, "x-oss-version-id")? { meta.set_version(v); diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs index dcd69f53df12..00c0e1b95598 100644 --- a/core/src/services/oss/core.rs +++ b/core/src/services/oss/core.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +use std::collections::HashMap; use std::fmt::Debug; use std::fmt::Formatter; use std::fmt::Write; @@ -28,10 +29,10 @@ use http::header::CONTENT_TYPE; use http::header::IF_MATCH; use http::header::IF_NONE_MATCH; use http::header::RANGE; -use http::HeaderName; use http::HeaderValue; use http::Request; use http::Response; +use http::{HeaderMap, HeaderName}; use reqsign::AliyunCredential; use reqsign::AliyunLoader; use reqsign::AliyunOssSigner; @@ -49,7 +50,8 @@ pub mod constants { pub const RESPONSE_CONTENT_DISPOSITION: &str = "response-content-disposition"; pub const OSS_QUERY_VERSION_ID: &str = "versionId"; - pub const USER_METADATA_PREFIX: &str = "x-oss-meta-"; + + pub const X_OSS_META_PREFIX: &str = "x-oss-meta-"; } pub struct OssCore { @@ -157,6 +159,88 @@ impl OssCore { } req } + + fn insert_metadata_headers( + &self, + mut req: http::request::Builder, + size: Option, + args: &OpWrite, + ) -> Result { + req = req.header(CONTENT_LENGTH, size.unwrap_or_default()); + + if let Some(mime) = args.content_type() { + req = req.header(CONTENT_TYPE, mime); + } + + if let Some(pos) = args.content_disposition() { + req = req.header(CONTENT_DISPOSITION, pos); + } + + if let Some(cache_control) = args.cache_control() { + req = req.header(CACHE_CONTROL, cache_control); + } + + if let Some(user_metadata) = args.user_metadata() { + for (key, value) in user_metadata { + // before insert user defined metadata header, add prefix to the header name + if !self.check_user_metadata_key(key) { + return Err(Error::new( + ErrorKind::Unsupported, + "the format of the user metadata key is invalid, please refer the document", + )); + } + req = req.header(format!("{}{}", constants::X_OSS_META_PREFIX, key), value) + } + } + + Ok(req) + } + + // According to https://help.aliyun.com/zh/oss/developer-reference/putobject + // there are some limits in user defined metadata key + fn check_user_metadata_key(&self, key: &str) -> bool { + key.chars().all(|c| c.is_ascii_alphanumeric() || c == '-') + } + + /// parse_metadata will parse http headers(including standards http headers + /// and user defined metadata header) into Metadata. + /// + /// # Arguments + /// + /// * `user_metadata_prefix` is the prefix of user defined metadata key + /// + /// # Notes + /// + /// before return the user defined metadata, we'll strip the user_metadata_prefix from the key + pub fn parse_metadata( + &self, + path: &str, + user_metadata_prefix: &str, + headers: &HeaderMap, + ) -> Result { + let mut m = parse_into_metadata(path, headers)?; + + let data: HashMap = headers + .iter() + .filter_map(|(key, _)| { + if key.as_str().starts_with(user_metadata_prefix) { + if let Ok(Some(value)) = parse_header_to_str(headers, key) { + let key_str = key.to_string(); + let stripped_key = key_str + .strip_prefix(user_metadata_prefix) + .expect("strip prefix must succeed"); + return Some((stripped_key.to_string(), value.to_string())); + } + } + None + }) + .collect(); + if !data.is_empty() { + m.with_user_metadata(data); + } + + Ok(m) + } } impl OssCore { @@ -175,7 +259,7 @@ impl OssCore { let mut req = Request::put(&url); - req = self.insert_metadata_headers(req, size, args); + req = self.insert_metadata_headers(req, size, args)?; // set sse headers req = self.insert_sse_headers(req); @@ -203,7 +287,7 @@ impl OssCore { let mut req = Request::post(&url); - req = self.insert_metadata_headers(req, Some(size), args); + req = self.insert_metadata_headers(req, Some(size), args)?; // set sse headers req = self.insert_sse_headers(req); @@ -212,36 +296,6 @@ impl OssCore { Ok(req) } - fn insert_metadata_headers( - &self, - mut req: http::request::Builder, - size: Option, - args: &OpWrite, - ) -> http::request::Builder { - req = req.header(CONTENT_LENGTH, size.unwrap_or_default()); - - if let Some(mime) = args.content_type() { - req = req.header(CONTENT_TYPE, mime); - } - - if let Some(pos) = args.content_disposition() { - req = req.header(CONTENT_DISPOSITION, pos); - } - - if let Some(cache_control) = args.cache_control() { - req = req.header(CACHE_CONTROL, cache_control); - } - - if let Some(user_metadata) = args.user_metadata() { - for (key, value) in user_metadata { - // before insert user defined metadata header, add prefix to the header name - req = req.header(format!("{}{}", constants::USER_METADATA_PREFIX, key), value) - } - } - - req - } - pub fn oss_get_object_request( &self, path: &str, diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 1503fc0fe85b..e83caf2768ca 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -330,8 +330,14 @@ impl>> FutureWrite { /// /// we don't need to include the user defined metadata prefix in the key /// every service will handle it internally - pub fn user_metadata(self, data: HashMap) -> Self { - self.map(|(args, options, bs)| (args.with_user_metadata(data), options, bs)) + pub fn user_metadata(self, data: impl IntoIterator) -> Self { + self.map(|(args, options, bs)| { + ( + args.with_user_metadata(HashMap::from_iter(data)), + options, + bs, + ) + }) } } @@ -402,8 +408,8 @@ impl>> FutureWriter { /// /// we don't need to include the user defined metadata prefix in the key /// every service will handle it internally - pub fn user_metadata(self, data: HashMap) -> Self { - self.map(|(args, options)| (args.with_user_metadata(data), options)) + pub fn user_metadata(self, data: impl IntoIterator) -> Self { + self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options)) } } diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs index baeabc685425..604e9566312b 100644 --- a/core/tests/behavior/async_write.rs +++ b/core/tests/behavior/async_write.rs @@ -215,10 +215,7 @@ pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> { } let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); - let target_user_metadata: HashMap = - [("location".to_string(), "everywhere".to_string())] - .into_iter() - .collect(); + let target_user_metadata = vec![("location".to_string(), "everywhere".to_string())]; op.write_with(&path, content) .user_metadata(target_user_metadata.clone()) .await?; @@ -226,7 +223,10 @@ pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> { let meta = op.stat(&path).await.expect("stat must succeed"); let resp_meta = meta.user_metadata().expect("meta data must exist"); - assert_eq!(*resp_meta, target_user_metadata); + assert_eq!( + *resp_meta, + target_user_metadata.into_iter().collect::>() + ); Ok(()) }