[Feature] fix conflict with main

Signed-off-by: jukejian <[email protected]>
StarRocks · May 27, 2024 · 61d5aad · 61d5aad
1 parent 8abc139
commit 61d5aad
Show file tree

Hide file tree

Showing 7 changed files with 30 additions and 113 deletions.
diff --git a/be/src/storage/lake/lake_delvec_loader.cpp b/be/src/storage/lake/lake_delvec_loader.cpp
@@ -18,11 +18,9 @@
 #include "common/status.h"
 #include "storage/lake/location_provider.h"
 
-
 namespace starrocks {
 namespace lake {
 
-
 Status LakeDelvecLoader::load(const TabletSegmentId& tsid, int64_t version, DelVectorPtr* pdelvec) {
     if (_pk_builder != nullptr) {
         // 1. find in meta builder first
@@ -41,8 +39,9 @@ Status LakeDelvecLoader::load_from_file(const TabletSegmentId& tsid, int64_t ver
     (*pdelvec).reset(new DelVector());
     // 2. find in delvec file
     std::string filepath = _lake_io_opts.location_provider->tablet_metadata_location(tsid.tablet_id, version);
-    ASSIGN_OR_RETURN(auto metadata, _tablet_manager->get_tablet_metadata(_lake_io_opts.fs, filepath, false));
-    RETURN_IF_ERROR(lake::get_del_vec(_tablet_manager, *metadata, tsid.segment_id, _lake_io_opts, pdelvec->get()));
+    ASSIGN_OR_RETURN(auto metadata, _tablet_manager->get_tablet_metadata(_lake_io_opts.fs, filepath, _fill_cache));
+    RETURN_IF_ERROR(
+            lake::get_del_vec(_tablet_manager, *metadata, tsid.segment_id, _fill_cache, _lake_io_opts, pdelvec->get()));
     return Status::OK();
 }
 

diff --git a/be/src/storage/lake/lake_delvec_loader.h b/be/src/storage/lake/lake_delvec_loader.h
@@ -26,14 +26,19 @@ namespace lake {
 
 class LakeDelvecLoader : public DelvecLoader {
 public:
-    LakeDelvecLoader(TabletManager* tablet_manager, const MetaFileBuilder* pk_builder, LakeIOOptions lake_io_opts)
-            : _tablet_manager(tablet_manager), _pk_builder(pk_builder), _lake_io_opts(std::move(lake_io_opts)) {}
+    LakeDelvecLoader(TabletManager* tablet_manager, const MetaFileBuilder* pk_builder, bool fill_cache,
+                     LakeIOOptions lake_io_opts)
+            : _tablet_manager(tablet_manager),
+              _pk_builder(pk_builder),
+              _fill_cache(fill_cache),
+              _lake_io_opts(std::move(lake_io_opts)) {}
     Status load(const TabletSegmentId& tsid, int64_t version, DelVectorPtr* pdelvec);
     Status load_from_file(const TabletSegmentId& tsid, int64_t version, DelVectorPtr* pdelvec);
 
 private:
     TabletManager* _tablet_manager;
     const MetaFileBuilder* _pk_builder = nullptr;
+    bool _fill_cache = false;
     LakeIOOptions _lake_io_opts;
     UpdateManager* _update_mgr;
 };

diff --git a/be/src/storage/lake/meta_file.cpp b/be/src/storage/lake/meta_file.cpp
@@ -284,8 +284,8 @@ void MetaFileBuilder::finalize_sstable_meta(const PersistentIndexSstableMetaPB&
     _tablet_meta->mutable_sstable_meta()->CopyFrom(sstable_meta);
 }
 
-Status get_del_vec(TabletManager* tablet_mgr, const TabletMetadata& metadata, uint32_t segment_id,
-bool fill_cache, const LakeIOOptions& lake_io_opts, DelVector* delvec) {
+Status get_del_vec(TabletManager* tablet_mgr, const TabletMetadata& metadata, uint32_t segment_id, bool fill_cache,
+                   const LakeIOOptions& lake_io_opts, DelVector* delvec) {
     // find delvec by segment id
     auto iter = metadata.delvec_meta().delvecs().find(segment_id);
     if (iter != metadata.delvec_meta().delvecs().end()) {
@@ -309,14 +309,15 @@ bool fill_cache, const LakeIOOptions& lake_io_opts, DelVector* delvec) {
         }
         const auto& delvec_name = iter2->second.name();
         RandomAccessFileOptions opts{.skip_fill_local_cache = !fill_cache};
-       if (lake_io_opts.fs && lake_io_opts.location_provider) {
-                   ASSIGN_OR_RETURN(
-                           rf, lake_io_opts.fs->new_random_access_file(
-                                       opts, lake_io_opts.location_provider->delvec_location(metadata.id(), delvec_name)));
-               } else {
-                   ASSIGN_OR_RETURN(rf,
-                                    fs::new_random_access_file(opts, tablet_mgr->delvec_location(metadata.id(), delvec_name)));
-               }
+        std::unique_ptr<RandomAccessFile> rf;
+        if (lake_io_opts.fs && lake_io_opts.location_provider) {
+            ASSIGN_OR_RETURN(
+                    rf, lake_io_opts.fs->new_random_access_file(
+                                opts, lake_io_opts.location_provider->delvec_location(metadata.id(), delvec_name)));
+        } else {
+            ASSIGN_OR_RETURN(rf,
+                             fs::new_random_access_file(opts, tablet_mgr->delvec_location(metadata.id(), delvec_name)));
+        }
         RETURN_IF_ERROR(rf->read_at_fully(iter->second.offset(), buf.data(), iter->second.size()));
         // parse delvec
         RETURN_IF_ERROR(delvec->load(iter->second.version(), buf.data(), iter->second.size()));

diff --git a/be/src/storage/lake/meta_file.h b/be/src/storage/lake/meta_file.h
@@ -80,7 +80,7 @@ class MetaFileBuilder {
 };
 
 Status get_del_vec(TabletManager* tablet_mgr, const TabletMetadata& metadata, uint32_t segment_id, bool fill_cache,
- const LakeIOOptions& lake_io_opts, DelVector* delvec);
+                   const LakeIOOptions& lake_io_opts, DelVector* delvec);
 bool is_primary_key(TabletMetadata* metadata);
 bool is_primary_key(const TabletMetadata& metadata);
 

diff --git a/be/src/storage/lake/update_manager.cpp b/be/src/storage/lake/update_manager.cpp
@@ -70,10 +70,6 @@ inline std::string cache_key(uint32_t tablet_id, int64_t txn_id) {
     return strings::Substitute("$0_$1", tablet_id, txn_id);
 }
 
-Status LakeDelvecLoader::load(const TabletSegmentId& tsid, int64_t version, DelVectorPtr* pdelvec) {
-    return _update_mgr->get_del_vec(tsid, version, _pk_builder, _fill_cache, pdelvec);
-}
-
 PersistentIndexBlockCache::PersistentIndexBlockCache(MemTracker* mem_tracker, int64_t cache_limit)
         : _cache(new_lru_cache(cache_limit)) {
     _mem_tracker = std::make_unique<MemTracker>(cache_limit, "lake_persistent_index_block_cache", mem_tracker);

diff --git a/be/test/fs/fs_s3_test.cpp b/be/test/fs/fs_s3_test.cpp
@@ -122,6 +122,14 @@ TEST_F(S3FileSystemTest, test_cache) {
     ASSERT_TRUE(client1 == client2);
 }
 
+TEST_F(S3FileSystemTest, test_prefetch) {
+    std::unordered_map<std::string, std::string> params = {{"fs.s3a.readahead.range", "100"}};
+    RandomAccessFileOptions opts{.skip_fill_local_cache = true};
+    std::unique_ptr<FSOptions> fs_options = std::make_unique<FSOptions>(params);
+    std::unique_ptr<S3FileSystem> fs = std::make_unique<S3FileSystem>(*fs_options);
+    ASSIGN_OR_ABORT(rf, fs->new_random_access_file(opts, "/tmp/a.file"));
+}
+
 TEST_F(S3FileSystemTest, test_retry) {
     TCloudConfiguration cloud_config;
     S3URI uri = S3URI.parse("s3://bucket1/path1");

diff --git a/docker/dockerfiles/dev-env/emr-dev-env.Dockerfile b/docker/dockerfiles/dev-env/emr-dev-env.Dockerfile