From f91ab5c8ac59bc26ff5095ee34ab759e4884ace0 Mon Sep 17 00:00:00 2001 From: Wenbin1002 Date: Wed, 1 Jan 2025 17:38:32 +0800 Subject: [PATCH 1/4] add is_current for metadata (cherry picked from commit 39bb79d34a6699359cfbda3c9ce0a4ac2aa0d539) --- core/src/services/oss/backend.rs | 3 +++ core/src/services/oss/lister.rs | 1 + core/src/services/s3/backend.rs | 3 +++ core/src/services/s3/lister.rs | 3 ++- core/src/types/metadata.rs | 23 +++++++++++++++++++++++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index 0ec56cdc2acd..f3bf93553bf7 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -521,8 +521,11 @@ impl Access for OssBackend { let headers = resp.headers(); let mut meta = self.core.parse_metadata(path, resp.headers())?; + // If version id exists, set the version; otherwise, mark as the current version. if let Some(v) = parse_header_to_str(headers, "x-oss-version-id")? { meta.set_version(v); + } else { + meta.set_is_current(true); } Ok(RpStat::new(meta)) diff --git a/core/src/services/oss/lister.rs b/core/src/services/oss/lister.rs index a130ea201b56..e69096e50db2 100644 --- a/core/src/services/oss/lister.rs +++ b/core/src/services/oss/lister.rs @@ -102,6 +102,7 @@ impl oio::PageList for OssLister { } let mut meta = Metadata::new(EntryMode::from_path(&path)); + meta.set_is_current(true); meta.set_etag(&object.etag); meta.set_content_md5(object.etag.trim_matches('"')); meta.set_content_length(object.size); diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 2d8ba50588e3..a8a5f4c7a87b 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -1016,8 +1016,11 @@ impl Access for S3Backend { meta.with_user_metadata(user_meta); } + // If version id exists, set the version; otherwise, mark as the current version. if let Some(v) = parse_header_to_str(headers, "x-amz-version-id")? { meta.set_version(v); + } else { + meta.set_is_current(true); } Ok(RpStat::new(meta)) diff --git a/core/src/services/s3/lister.rs b/core/src/services/s3/lister.rs index 9063e610448d..fb27f23ee5bc 100644 --- a/core/src/services/s3/lister.rs +++ b/core/src/services/s3/lister.rs @@ -124,7 +124,7 @@ impl oio::PageList for S3Lister { } let mut meta = Metadata::new(EntryMode::from_path(&path)); - + meta.set_is_current(true); if let Some(etag) = &object.etag { meta.set_etag(etag); meta.set_content_md5(etag.trim_matches('"')); @@ -239,6 +239,7 @@ impl oio::PageList for S3ObjectVersionsLister { let mut meta = Metadata::new(EntryMode::from_path(&path)); meta.set_version(&version_object.version_id); + meta.set_is_current(version_object.is_latest); meta.set_content_length(version_object.size); meta.set_last_modified(parse_datetime_from_rfc3339( version_object.last_modified.as_str(), diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs index c1d4155e1971..957f5072a6a3 100644 --- a/core/src/types/metadata.rs +++ b/core/src/types/metadata.rs @@ -43,6 +43,7 @@ pub struct Metadata { etag: Option, last_modified: Option>, version: Option, + is_current: Option, user_metadata: Option>, } @@ -64,6 +65,7 @@ impl Metadata { content_disposition: None, version: None, user_metadata: None, + is_current: None, } } @@ -404,6 +406,27 @@ impl Metadata { self } + /// Is_current of this entry. + /// + /// Is_current is a boolean that can be used to identify + /// if the version of this entry is the latest version. + pub fn is_current(&self) -> Option { + self.is_current + } + + /// Set is_current of this entry. + /// + /// For HeadObject without version_id, we will set it to Some(true). + /// + /// For HeadObject with version_id, we will set it to None. + /// + /// For ListObjects, we will set all keys to Some(ture) + /// + /// For ListObjectVersions, we will decide the value based on IsLatest. + pub fn set_is_current(&mut self, is_current:bool) { + self.is_current = Some(is_current); + } + /// User defined metadata of this entry /// /// The prefix of the user defined metadata key(for example: in oss, it's x-oss-meta-) From 63c9fd86813819fecd2b0adc8e4ff11fc3add297 Mon Sep 17 00:00:00 2001 From: Wenbin1002 Date: Wed, 1 Jan 2025 17:59:53 +0800 Subject: [PATCH 2/4] fmt --- core/src/types/metadata.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs index 957f5072a6a3..73e26f6201e2 100644 --- a/core/src/types/metadata.rs +++ b/core/src/types/metadata.rs @@ -407,23 +407,23 @@ impl Metadata { } /// Is_current of this entry. - /// - /// Is_current is a boolean that can be used to identify + /// + /// Is_current is a boolean that can be used to identify /// if the version of this entry is the latest version. pub fn is_current(&self) -> Option { self.is_current } - + /// Set is_current of this entry. - /// + /// /// For HeadObject without version_id, we will set it to Some(true). - /// + /// /// For HeadObject with version_id, we will set it to None. - /// + /// /// For ListObjects, we will set all keys to Some(ture) - /// + /// /// For ListObjectVersions, we will decide the value based on IsLatest. - pub fn set_is_current(&mut self, is_current:bool) { + pub fn set_is_current(&mut self, is_current: bool) { self.is_current = Some(is_current); } From 56a798e2183d9509326db027d402c6353c02ceac Mon Sep 17 00:00:00 2001 From: Wenbin1002 Date: Wed, 1 Jan 2025 18:00:41 +0800 Subject: [PATCH 3/4] fix typo --- core/src/types/metadata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs index 73e26f6201e2..f36df4930461 100644 --- a/core/src/types/metadata.rs +++ b/core/src/types/metadata.rs @@ -420,7 +420,7 @@ impl Metadata { /// /// For HeadObject with version_id, we will set it to None. /// - /// For ListObjects, we will set all keys to Some(ture) + /// For ListObjects, we will set all keys to Some(true) /// /// For ListObjectVersions, we will decide the value based on IsLatest. pub fn set_is_current(&mut self, is_current: bool) { From b4bd78601d3a67a79f603067390c5db69c0f9357 Mon Sep 17 00:00:00 2001 From: Wenbin1002 Date: Thu, 2 Jan 2025 15:21:51 +0800 Subject: [PATCH 4/4] fix --- core/src/services/oss/backend.rs | 3 --- core/src/services/s3/backend.rs | 3 --- core/src/types/metadata.rs | 30 +++++++++++++++++++----------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index f3bf93553bf7..0ec56cdc2acd 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -521,11 +521,8 @@ impl Access for OssBackend { let headers = resp.headers(); let mut meta = self.core.parse_metadata(path, resp.headers())?; - // If version id exists, set the version; otherwise, mark as the current version. if let Some(v) = parse_header_to_str(headers, "x-oss-version-id")? { meta.set_version(v); - } else { - meta.set_is_current(true); } Ok(RpStat::new(meta)) diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index a8a5f4c7a87b..2d8ba50588e3 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -1016,11 +1016,8 @@ impl Access for S3Backend { meta.with_user_metadata(user_meta); } - // If version id exists, set the version; otherwise, mark as the current version. if let Some(v) = parse_header_to_str(headers, "x-amz-version-id")? { meta.set_version(v); - } else { - meta.set_is_current(true); } Ok(RpStat::new(meta)) diff --git a/core/src/types/metadata.rs b/core/src/types/metadata.rs index f36df4930461..fb9fbe92335d 100644 --- a/core/src/types/metadata.rs +++ b/core/src/types/metadata.rs @@ -406,25 +406,33 @@ impl Metadata { self } - /// Is_current of this entry. + /// Determines if the provided metadata reflects the current status of the path. /// - /// Is_current is a boolean that can be used to identify - /// if the version of this entry is the latest version. + /// - `Ok(true)` indicates it is the latest status. + /// - `Ok(false)` indicates it is an older version of the file. + /// - `None` indicates uncertainty about its status. + /// + /// This API allows users to verify if the version is up-to-date when listing with versions. pub fn is_current(&self) -> Option { self.is_current } - /// Set is_current of this entry. - /// - /// For HeadObject without version_id, we will set it to Some(true). + /// Set the `is_current` status of this entry. /// - /// For HeadObject with version_id, we will set it to None. - /// - /// For ListObjects, we will set all keys to Some(true) + /// By default, this value will be `None`. Please avoid using this API if it's unclear whether the entry is current. + /// Set it to `true` if it is known to be the latest; otherwise, set it to `false`. + pub fn with_is_current(mut self, is_current: Option) -> Self { + self.is_current = is_current; + self + } + + /// Set the `is_current` status of this entry. /// - /// For ListObjectVersions, we will decide the value based on IsLatest. - pub fn set_is_current(&mut self, is_current: bool) { + /// By default, this value will be `None`. Please avoid using this API if it's unclear whether the entry is current. + /// Set it to `true` if it is known to be the latest; otherwise, set it to `false`. + pub fn set_is_current(&mut self, is_current: bool) -> &mut Self { self.is_current = Some(is_current); + self } /// User defined metadata of this entry