tmp

update

update

udpate

be/CMakeLists.txt

@@ -729,6 +729,11 @@ if (BUILD_META_TOOL)
     add_subdirectory(${SRC_DIR}/tools)
 endif()
 
+option(BUILD_MY_TOOL "Build my tool" ON)
+if (BUILD_MY_TOOL)
+    add_subdirectory(${SRC_DIR}/my-tools)
+endif()
+
 option(BUILD_INDEX_TOOL "Build index tool" OFF)
 if (BUILD_INDEX_TOOL)
     add_subdirectory(${SRC_DIR}/index-tools)

be/src/my-tools/CMakeLists.txt

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# where to put generated libraries
+set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/my-tools")
+
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/my-tools")
+
+add_executable(my_tool
+    my_tool.cpp
+)
+
+# pch_reuse(my_tool)
+
+# This permits libraries loaded by dlopen to link to the symbols in the program.
+set_target_properties(my_tool PROPERTIES ENABLE_EXPORTS 1)
+
+target_link_libraries(my_tool
+    ${DORIS_LINK_LIBS}
+)
+
+install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/)
+install(TARGETS my_tool DESTINATION ${OUTPUT_DIR}/lib/)

be/src/my-tools/my_tool.cpp

@@ -0,0 +1,156 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <fmt/format.h>
+#include <gen_cpp/olap_file.pb.h>
+#include <gflags/gflags.h>
+
+#include <cstdint>
+#include <string>
+
+#include "common/status.h"
+#include "olap/olap_meta.h"
+#include "olap/rowset/rowset_meta.h"
+#include "olap/rowset/rowset_meta_manager.h"
+#include "olap/tablet_meta_manager.h"
+
+DEFINE_string(operation, "", "valid operation: show");
+DEFINE_string(root_path, "", "root path");
+
+std::string get_usage(const std::string& progname) {
+    std::stringstream ss;
+    ss << progname << " is the Doris inverted index file tool.\n";
+    ss << "Usage:\n";
+    ss << "./my_tool";
+    return ss.str();
+}
+
+void print_meta_detail(doris::OlapMeta& meta) {
+    using namespace doris;
+
+    // rowset meta
+    int64_t rowset_meta_size_sum {};
+    Status res;
+    // auto load_rowset_func = [&rowset_meta_size_sum](TabletUid tablet_uid, RowsetId rowset_id,
+    //                                                 const std::string& meta_str) -> bool {
+    //     rowset_meta_size_sum += meta_str.size();
+    //     std::cout << fmt::format("collect rowset meta: rowset_id={}, size={}\n",
+    //                              rowset_id.to_string(), meta_str.size());
+    //     return true;
+    // };
+
+    // Status res = RowsetMetaManager::traverse_rowset_metas(&meta, load_rowset_func);
+    // if (!res.ok()) {
+    //     std::cout << fmt::format("failed to traverse rowset meta.\n");
+    // }
+
+    // std::cout << fmt::format("finish collect rowset meta\n");
+
+    // tablet_meta
+    int64_t tablet_meta_size_sum {};
+    int64_t delete_bitmap_size_sum {};
+    std::map<int64_t, int64_t> tablets_delete_bitmap_size;
+    auto load_tablet_func = [&tablet_meta_size_sum, &delete_bitmap_size_sum,
+                             &tablets_delete_bitmap_size](int64_t tablet_id, int32_t schema_hash,
+                                                          const std::string& value) -> bool {
+        tablet_meta_size_sum += value.size();
+        std::cout << fmt::format("collect tablet meta: tablet_id={}, size={}\n", tablet_id,
+                                 value.size());
+        TabletMetaPB tablet_meta_pb;
+        bool parsed = tablet_meta_pb.ParseFromString(value);
+        if (!parsed) {
+            std::cout << fmt::format("failed to parse tablet mete. tablet_id={}\n", tablet_id);
+        }
+
+        if (tablet_meta_pb.has_delete_bitmap()) {
+            auto sz = tablet_meta_pb.ByteSizeLong();
+            tablets_delete_bitmap_size[tablet_id] = sz;
+            delete_bitmap_size_sum += sz;
+            std::cout << fmt::format("collect delete bitmap: tablet_id={}, delete_bitmap_size={)\n",
+                                     tablet_id, sz);
+        }
+
+        return true;
+    };
+
+    res = TabletMetaManager::traverse_headers(&meta, load_tablet_func);
+    if (!res.ok()) {
+        std::cout << fmt::format("failed to traverse tablet meta.\n");
+    }
+
+    std::cout << fmt::format(
+            " ======================== tablet delete bitmaps size ======================== \n");
+    for (const auto& [tablet_id, size] : tablets_delete_bitmap_size) {
+        std::cout << fmt::format("tablet_id={}, size={}\n", tablet_id, size);
+    }
+
+    // pending delete bitmaps
+    std::vector<DeleteBitmapPB> delete_bitmap_pbs;
+    int64_t pending_delete_bitmap_size_sum {};
+    auto collect_delete_bitmap_func =
+            [&delete_bitmap_pbs, &tablets_delete_bitmap_size, &pending_delete_bitmap_size_sum](
+                    int64_t tablet_id, int64_t version, const std::string& val) {
+                DeleteBitmapPB delete_bitmap_pb;
+                delete_bitmap_pb.ParseFromString(val);
+                delete_bitmap_pbs.emplace_back(std::move(delete_bitmap_pb));
+                tablets_delete_bitmap_size[tablet_id] += val.size();
+                pending_delete_bitmap_size_sum += val.size();
+
+                std::cout << fmt::format(
+                        "collect pending delete bitmap: tablet_id={}, version={} val.size={}\n",
+                        tablet_id, version, val.size());
+
+                return true;
+            };
+
+    res = TabletMetaManager::traverse_delete_bitmap(&meta, collect_delete_bitmap_func);
+    if (!res.ok()) {
+        std::cout << fmt::format("failed to traverse delete bitmap.\n");
+    }
+
+    std::cout << fmt::format(
+            " ======================== Summary: ======================== "
+            "tablet_meta.size={}\nrowset_meta.size={} B\ntablet_meta_size={} "
+            "B\ndelete_bitmap_size={} B\npending_delete_bitmap_size={} B\n",
+            tablets_delete_bitmap_size.size(), rowset_meta_size_sum, tablet_meta_size_sum,
+            delete_bitmap_size_sum, pending_delete_bitmap_size_sum);
+}
+
+int main(int argc, char** argv) {
+    using namespace doris;
+
+    std::string usage = get_usage(argv[0]);
+    gflags::SetUsageMessage(usage);
+    google::ParseCommandLineFlags(&argc, &argv, true);
+
+    if (FLAGS_operation == "show") {
+        std::string root_path {FLAGS_root_path};
+        std::cout << fmt::format("get root_path={}\n", root_path);
+        OlapMeta meta {root_path};
+        std::cout << fmt::format("begin to init OlapMeta, root_path={}\n", root_path);
+        Status res = meta.init();
+        if (!res.ok()) {
+            std::cout << fmt::format("init OlapMeta failed, open rocksdb failed, path={}",
+                                     root_path);
+        }
+        std::cout << fmt::format("successfully open RocksDB, path={}\n", meta.get_root_path());
+
+        print_meta_detail(meta);
+    }
+    gflags::ShutDownCommandLineFlags();
+    return 0;
+}

be/src/olap/tablet_meta_manager.cpp

@@ -268,8 +268,8 @@ Status TabletMetaManager::traverse_delete_bitmap(
         TTabletId tablet_id;
         int64_t version;
         decode_delete_bitmap_key(key, &tablet_id, &version);
-        VLOG_NOTICE << "traverse delete bitmap, tablet_id: " << tablet_id
-                    << ", version: " << version;
+        std::cout << "traverse delete bitmap, tablet_id: " << tablet_id << ", version: " << version
+                  << "\n";
         return func(tablet_id, version, value);
     };
     return meta->iterate(META_COLUMN_FAMILY_INDEX, DELETE_BITMAP, traverse_header_func);

be/src/tools/meta_tool.cpp

@@ -61,7 +61,7 @@ using doris::segment_v2::ColumnIteratorOptions;
 using doris::segment_v2::PageFooterPB;
 using doris::io::FileReaderSPtr;
 
-const std::string HEADER_PREFIX = "tabletmeta_";
+// const std::string HEADER_PREFIX = "tabletmeta_";
 
 DEFINE_string(root_path, "", "storage root path");
 DEFINE_string(operation, "get_meta",

build.sh

@@ -140,6 +140,7 @@ BUILD_FE=0
 BUILD_BE=0
 BUILD_BROKER=0
 BUILD_META_TOOL='OFF'
+BUILD_MY_TOOL='ON'
 BUILD_INDEX_TOOL='OFF'
 BUILD_SPARK_DPP=0
 BUILD_BE_JAVA_EXTENSIONS=0
@@ -535,6 +536,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
         -DWITH_MYSQL="${WITH_MYSQL}" \
         -DUSE_LIBCPP="${USE_LIBCPP}" \
         -DBUILD_META_TOOL="${BUILD_META_TOOL}" \
+        -DBUILD_MY_TOOL="${BUILD_MY_TOOL}" \
         -DBUILD_INDEX_TOOL="${BUILD_INDEX_TOOL}" \
         -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
         -DUSE_DWARF="${USE_DWARF}" \