Add user metadata key checks for oss && refeactor some code

core/src/raw/http_util/header.rs

@@ -32,7 +32,6 @@ use http::HeaderMap;
 use http::HeaderName;
 use http::HeaderValue;
 use md5::Digest;
-use std::collections::HashMap;
 
 use crate::raw::*;
 use crate::EntryMode;
@@ -190,45 +189,6 @@ pub fn parse_into_metadata(path: &str, headers: &HeaderMap) -> Result<Metadata>
     Ok(m)
 }
 
-/// parse_metadata will parse http headers(including standards http headers
-/// and user defined metadata header) into Metadata.
-///
-/// # Arguments
-///
-/// * `user_metadata_prefix` is the prefix of user defined metadata key
-///
-/// # Notes
-///
-/// before return the user defined metadata, we'll strip the user_metadata_prefix from the key
-pub fn parse_metadata(
-    path: &str,
-    user_metadata_prefix: &str,
-    headers: &HeaderMap,
-) -> Result<Metadata> {
-    let mut m = parse_into_metadata(path, headers)?;
-
-    let data: HashMap<String, String> = headers
-        .iter()
-        .filter_map(|(key, _)| {
-            if key.as_str().starts_with(user_metadata_prefix) {
-                if let Ok(Some(value)) = parse_header_to_str(headers, key) {
-                    let key_str = key.to_string();
-                    let stripped_key = key_str
-                        .strip_prefix(user_metadata_prefix)
-                        .expect("strip prefix must succeed");
-                    return Some((stripped_key.to_string(), value.to_string()));
-                }
-            }
-            None
-        })
-        .collect();
-    if !data.is_empty() {
-        m.with_user_metadata(data);
-    }
-
-    Ok(m)
-}
-
 /// format content md5 header by given input.
 pub fn format_content_md5(bs: &[u8]) -> String {
     let mut hasher = md5::Md5::new();

core/src/raw/http_util/mod.rs

@@ -44,7 +44,6 @@ pub use header::parse_header_to_str;
 pub use header::parse_into_metadata;
 pub use header::parse_last_modified;
 pub use header::parse_location;
-pub use header::parse_metadata;
 
 mod uri;
 pub use uri::percent_decode_path;

core/src/services/oss/backend.rs

@@ -449,7 +449,8 @@ impl Access for OssBackend {
             StatusCode::OK => {
                 let headers = resp.headers();
                 let mut meta =
-                    parse_metadata(path, constants::USER_METADATA_PREFIX, resp.headers())?;
+                    self.core
+                        .parse_metadata(path, constants::X_OSS_META_PREFIX, resp.headers())?;
 
                 if let Some(v) = parse_header_to_str(headers, "x-oss-version-id")? {
                     meta.set_version(v);

core/src/services/oss/core.rs

@@ -15,6 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
+use std::collections::HashMap;
 use std::fmt::Debug;
 use std::fmt::Formatter;
 use std::fmt::Write;
@@ -28,10 +29,10 @@ use http::header::CONTENT_TYPE;
 use http::header::IF_MATCH;
 use http::header::IF_NONE_MATCH;
 use http::header::RANGE;
-use http::HeaderName;
 use http::HeaderValue;
 use http::Request;
 use http::Response;
+use http::{HeaderMap, HeaderName};
 use reqsign::AliyunCredential;
 use reqsign::AliyunLoader;
 use reqsign::AliyunOssSigner;
@@ -49,7 +50,8 @@ pub mod constants {
     pub const RESPONSE_CONTENT_DISPOSITION: &str = "response-content-disposition";
 
     pub const OSS_QUERY_VERSION_ID: &str = "versionId";
-    pub const USER_METADATA_PREFIX: &str = "x-oss-meta-";
+
+    pub const X_OSS_META_PREFIX: &str = "x-oss-meta-";
 }
 
 pub struct OssCore {
@@ -157,6 +159,88 @@ impl OssCore {
         }
         req
     }
+
+    fn insert_metadata_headers(
+        &self,
+        mut req: http::request::Builder,
+        size: Option<u64>,
+        args: &OpWrite,
+    ) -> Result<http::request::Builder> {
+        req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
+
+        if let Some(mime) = args.content_type() {
+            req = req.header(CONTENT_TYPE, mime);
+        }
+
+        if let Some(pos) = args.content_disposition() {
+            req = req.header(CONTENT_DISPOSITION, pos);
+        }
+
+        if let Some(cache_control) = args.cache_control() {
+            req = req.header(CACHE_CONTROL, cache_control);
+        }
+
+        if let Some(user_metadata) = args.user_metadata() {
+            for (key, value) in user_metadata {
+                // before insert user defined metadata header, add prefix to the header name
+                if !self.check_user_metadata_key(key) {
+                    return Err(Error::new(
+                        ErrorKind::Unsupported,
+                        "the format of the user metadata key is invalid, please refer the document",
+                    ));
+                }
+                req = req.header(format!("{}{}", constants::X_OSS_META_PREFIX, key), value)
+            }
+        }
+
+        Ok(req)
+    }
+
+    // According to https://help.aliyun.com/zh/oss/developer-reference/putobject
+    // there are some limits in user defined metadata key
+    fn check_user_metadata_key(&self, key: &str) -> bool {
+        key.chars().all(|c| c.is_ascii_alphanumeric() || c == '-')
+    }
+
+    /// parse_metadata will parse http headers(including standards http headers
+    /// and user defined metadata header) into Metadata.
+    ///
+    /// # Arguments
+    ///
+    /// * `user_metadata_prefix` is the prefix of user defined metadata key
+    ///
+    /// # Notes
+    ///
+    /// before return the user defined metadata, we'll strip the user_metadata_prefix from the key
+    pub fn parse_metadata(
+        &self,
+        path: &str,
+        user_metadata_prefix: &str,
+        headers: &HeaderMap,
+    ) -> Result<Metadata> {
+        let mut m = parse_into_metadata(path, headers)?;
+
+        let data: HashMap<String, String> = headers
+            .iter()
+            .filter_map(|(key, _)| {
+                if key.as_str().starts_with(user_metadata_prefix) {
+                    if let Ok(Some(value)) = parse_header_to_str(headers, key) {
+                        let key_str = key.to_string();
+                        let stripped_key = key_str
+                            .strip_prefix(user_metadata_prefix)
+                            .expect("strip prefix must succeed");
+                        return Some((stripped_key.to_string(), value.to_string()));
+                    }
+                }
+                None
+            })
+            .collect();
+        if !data.is_empty() {
+            m.with_user_metadata(data);
+        }
+
+        Ok(m)
+    }
 }
 
 impl OssCore {
@@ -175,7 +259,7 @@ impl OssCore {
 
         let mut req = Request::put(&url);
 
-        req = self.insert_metadata_headers(req, size, args);
+        req = self.insert_metadata_headers(req, size, args)?;
 
         // set sse headers
         req = self.insert_sse_headers(req);
@@ -203,7 +287,7 @@ impl OssCore {
 
         let mut req = Request::post(&url);
 
-        req = self.insert_metadata_headers(req, Some(size), args);
+        req = self.insert_metadata_headers(req, Some(size), args)?;
 
         // set sse headers
         req = self.insert_sse_headers(req);
@@ -212,36 +296,6 @@ impl OssCore {
         Ok(req)
     }
 
-    fn insert_metadata_headers(
-        &self,
-        mut req: http::request::Builder,
-        size: Option<u64>,
-        args: &OpWrite,
-    ) -> http::request::Builder {
-        req = req.header(CONTENT_LENGTH, size.unwrap_or_default());
-
-        if let Some(mime) = args.content_type() {
-            req = req.header(CONTENT_TYPE, mime);
-        }
-
-        if let Some(pos) = args.content_disposition() {
-            req = req.header(CONTENT_DISPOSITION, pos);
-        }
-
-        if let Some(cache_control) = args.cache_control() {
-            req = req.header(CACHE_CONTROL, cache_control);
-        }
-
-        if let Some(user_metadata) = args.user_metadata() {
-            for (key, value) in user_metadata {
-                // before insert user defined metadata header, add prefix to the header name
-                req = req.header(format!("{}{}", constants::USER_METADATA_PREFIX, key), value)
-            }
-        }
-
-        req
-    }
-
     pub fn oss_get_object_request(
         &self,
         path: &str,

core/src/types/operator/operator_futures.rs

@@ -330,8 +330,14 @@ impl<F: Future<Output = Result<()>>> FutureWrite<F> {
     ///
     /// we don't need to include the user defined metadata prefix in the key
     /// every service will handle it internally
-    pub fn user_metadata(self, data: HashMap<String, String>) -> Self {
-        self.map(|(args, options, bs)| (args.with_user_metadata(data), options, bs))
+    pub fn user_metadata(self, data: impl IntoIterator<Item = (String, String)>) -> Self {
+        self.map(|(args, options, bs)| {
+            (
+                args.with_user_metadata(HashMap::from_iter(data)),
+                options,
+                bs,
+            )
+        })
     }
 }
 
@@ -402,8 +408,8 @@ impl<F: Future<Output = Result<Writer>>> FutureWriter<F> {
     ///
     /// we don't need to include the user defined metadata prefix in the key
     /// every service will handle it internally
-    pub fn user_metadata(self, data: HashMap<String, String>) -> Self {
-        self.map(|(args, options)| (args.with_user_metadata(data), options))
+    pub fn user_metadata(self, data: impl IntoIterator<Item = (String, String)>) -> Self {
+        self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options))
     }
 }
 

core/tests/behavior/async_write.rs

@@ -215,18 +215,18 @@ pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> {
     }
 
     let (path, content, _) = TEST_FIXTURE.new_file(op.clone());
-    let target_user_metadata: HashMap<String, String> =
-        [("location".to_string(), "everywhere".to_string())]
-            .into_iter()
-            .collect();
+    let target_user_metadata = vec![("location".to_string(), "everywhere".to_string())];
     op.write_with(&path, content)
         .user_metadata(target_user_metadata.clone())
         .await?;
 
     let meta = op.stat(&path).await.expect("stat must succeed");
     let resp_meta = meta.user_metadata().expect("meta data must exist");
 
-    assert_eq!(*resp_meta, target_user_metadata);
+    assert_eq!(
+        *resp_meta,
+        target_user_metadata.into_iter().collect::<HashMap<_, _>>()
+    );
 
     Ok(())
 }