From 39e0308a2c0073e6285cc73bde03af23439aefd2 Mon Sep 17 00:00:00 2001 From: Ketor Date: Sat, 27 Apr 2024 23:08:04 +0800 Subject: [PATCH] [fix][cmake] Upgrade RocksDB to 9.1.1. Signed-off-by: Ketor --- CMakeLists.txt | 14 +++++++++++++- cmake/braft.cmake | 1 + cmake/brpc.cmake | 1 + cmake/faiss-mkl.cmake | 1 - cmake/glog.cmake | 1 + cmake/leveldb.cmake | 3 +++ cmake/rocksdb.cmake | 3 ++- cmake/snappy.cmake | 1 + contrib/braft | 2 +- contrib/brpc | 2 +- contrib/faiss | 2 +- contrib/hdf5 | 2 +- contrib/libbacktrace | 2 +- contrib/openblas | 2 +- contrib/openssl | 2 +- contrib/rapidjson | 2 +- contrib/rocksdb | 2 +- contrib/zstd | 2 +- src/engine/rocks_raw_engine.cc | 4 ++-- 19 files changed, 34 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b4606a83..418866bc5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -117,6 +117,19 @@ elseif(${VECTORIZATION_INSTRUCTION_SET} STREQUAL "avx512") endif() include(CheckCXXCompilerFlag) +if (NOT CMAKE_CXX_STANDARD) + message(STATUS "CMAKE_CXX_STANDARD is not set, will set to 17") + set(CMAKE_CXX_STANDARD 17) +else() + set(ALLOWED_CXX_STANDARDS 17 20 23) + set_property(CACHE CMAKE_CXX_STANDARD PROPERTY STRINGS ${ALLOWED_CXX_STANDARDS}) + list(FIND ALLOWED_CXX_STANDARDS ${CMAKE_CXX_STANDARD} POSITION) + if (POSITION LESS 0) + message(FATAL_ERROR "Invalid CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD}. " + "Must be one of: ${ALLOWED_CXX_STANDARDS}") + endif() +endif() + if (CMAKE_CXX_STANDARD EQUAL 23) check_cxx_compiler_flag("-std=c++23" COMPILER_SUPPORTS_CXX23) if(COMPILER_SUPPORTS_CXX23) @@ -134,7 +147,6 @@ elseif(CMAKE_CXX_STANDARD EQUAL 20) message(FATAL_ERROR "The compiler ${CMAKE_CXX_COMPILER} has no C++20 support. Please use a different C++ compiler.") endif() else() - message(STATUS "CMAKE_CXX_STANDARD: ${CMAKE_CXX_STANDARD} change to c++17") check_cxx_compiler_flag("-std=c++17" COMPILER_SUPPORTS_CXX17) if(COMPILER_SUPPORTS_CXX17) set(CMAKE_CXX_STANDARD 17) diff --git a/cmake/braft.cmake b/cmake/braft.cmake index 248309ad5..6b5017f32 100644 --- a/cmake/braft.cmake +++ b/cmake/braft.cmake @@ -23,6 +23,7 @@ SET(BRAFT_LIBRARIES "${BRAFT_INSTALL_DIR}/lib/libbraft.a" CACHE FILEPATH "braft set(prefix_path "${THIRD_PARTY_PATH}/install/brpc|${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog|${THIRD_PARTY_PATH}/install/leveldb") +message(STATUS "braft search prefix_path in ${prefix_path}") set(CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS} -DUSE_BTHREAD_MUTEX) diff --git a/cmake/brpc.cmake b/cmake/brpc.cmake index 5a62218d6..a84af4e3a 100644 --- a/cmake/brpc.cmake +++ b/cmake/brpc.cmake @@ -24,6 +24,7 @@ SET(BRPC_LIBRARIES "${BRPC_INSTALL_DIR}/lib/libbrpc.a" CACHE FILEPATH "brpc libr # Reference https://stackoverflow.com/questions/45414507/pass-a-list-of-prefix-paths-to-externalproject-add-in-cmake-args set(prefix_path "${THIRD_PARTY_PATH}/install/gflags|${THIRD_PARTY_PATH}/install/openssl|${THIRD_PARTY_PATH}/install/protobuf|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/glog|${THIRD_PARTY_PATH}/install/leveldb") +message(STATUS "brpc search prefix_path in ${prefix_path}") # If minimal .a is need, you can set WITH_DEBUG_SYMBOLS=OFF ExternalProject_Add( diff --git a/cmake/faiss-mkl.cmake b/cmake/faiss-mkl.cmake index 2935d4aa1..7ecf4ce3a 100644 --- a/cmake/faiss-mkl.cmake +++ b/cmake/faiss-mkl.cmake @@ -27,7 +27,6 @@ elseif(${VECTORIZATION_INSTRUCTION_SET} STREQUAL "avx512") SET(FAISS_OPT_LEVEL avx512) SET(FAISS_LIBRARIES "${FAISS_INSTALL_DIR}/lib/libfaiss_avx512.a" CACHE FILEPATH "faiss library." FORCE) else() - message(STATUS "faiss use generic instruction set") SET(FAISS_OPT_LEVEL generic) SET(FAISS_LIBRARIES "${FAISS_INSTALL_DIR}/lib/libfaiss.a" CACHE FILEPATH "faiss library." FORCE) endif() diff --git a/cmake/glog.cmake b/cmake/glog.cmake index 7936db3d8..00628bfd1 100644 --- a/cmake/glog.cmake +++ b/cmake/glog.cmake @@ -29,6 +29,7 @@ ELSE (WIN32) ENDIF (WIN32) set(prefix_path "${THIRD_PARTY_PATH}/install/gflags") +message(STATUS "glog search prefix_path in ${prefix_path}") SET(gflags_BUILD_STATIC_LIBS ON) diff --git a/cmake/leveldb.cmake b/cmake/leveldb.cmake index f82c87d80..0330669e2 100644 --- a/cmake/leveldb.cmake +++ b/cmake/leveldb.cmake @@ -21,6 +21,9 @@ SET(LEVELDB_INSTALL_DIR ${THIRD_PARTY_PATH}/install/leveldb) SET(LEVELDB_INCLUDE_DIR "${LEVELDB_INSTALL_DIR}/include" CACHE PATH "leveldb include directory." FORCE) SET(LEVELDB_LIBRARIES "${LEVELDB_INSTALL_DIR}/lib/libleveldb.a" CACHE FILEPATH "leveldb library." FORCE) +set(prefix_path "${THIRD_PARTY_PATH}/install/snappy") +message(STATUS "leveldb search prefix_path in ${prefix_path}") + ExternalProject_Add( extern_leveldb ${EXTERNAL_PROJECT_LOG_ARGS} diff --git a/cmake/rocksdb.cmake b/cmake/rocksdb.cmake index 99cf73358..16356d06a 100644 --- a/cmake/rocksdb.cmake +++ b/cmake/rocksdb.cmake @@ -22,10 +22,11 @@ SET(ROCKSDB_INCLUDE_DIR "${ROCKSDB_INSTALL_DIR}/include" CACHE PATH "rocksdb inc SET(ROCKSDB_LIBRARIES "${ROCKSDB_INSTALL_DIR}/lib/librocksdb.a" CACHE FILEPATH "rocksdb library." FORCE) set(prefix_path "${THIRD_PARTY_PATH}/install/snappy|${THIRD_PARTY_PATH}/install/zlib|${THIRD_PARTY_PATH}/install/lz4|${THIRD_PARTY_PATH}/install/zstd|${THIRD_PARTY_PATH}/install/gflags") +message(STATUS "rocksdb search prefix_path in ${prefix_path}") # To avoid rocksdb PROTABLE options on old arch machine when build using docker set(ROCKSDB_PROTABLE_OPTION "0" CACHE STRING "An option for rocksdb PROTABLE, default 0") -message(STATUS "Rocksdb protable option(ROCKSDB_PROTABLE_OPTION): ${ROCKSDB_PROTABLE_OPTION}") +message(STATUS "rocksdb protable option: ${ROCKSDB_PROTABLE_OPTION}") ExternalProject_Add( extern_rocksdb diff --git a/cmake/snappy.cmake b/cmake/snappy.cmake index 491b347fb..89d83c63c 100644 --- a/cmake/snappy.cmake +++ b/cmake/snappy.cmake @@ -22,6 +22,7 @@ SET(SNAPPY_INCLUDE_DIR "${SNAPPY_INSTALL_DIR}/include" CACHE PATH "snappy includ SET(SNAPPY_LIBRARIES "${SNAPPY_INSTALL_DIR}/lib/libsnappy.a" CACHE FILEPATH "snappy library." FORCE) set(prefix_path "${THIRD_PARTY_PATH}/install/gflags") +message(STATUS "snappy search prefix_path in ${prefix_path}") ExternalProject_Add( extern_snappy diff --git a/contrib/braft b/contrib/braft index 7a0811ae4..a9012e293 160000 --- a/contrib/braft +++ b/contrib/braft @@ -1 +1 @@ -Subproject commit 7a0811ae48c20fb147381aba60205fad4a42446a +Subproject commit a9012e29389a14dc65e1d1dfc8ca74df5e6ef995 diff --git a/contrib/brpc b/contrib/brpc index df31bf51f..7daed0be6 160000 --- a/contrib/brpc +++ b/contrib/brpc @@ -1 +1 @@ -Subproject commit df31bf51f08ca6afa59ef3b4f3749bf20ebd1858 +Subproject commit 7daed0be6bf1f4d64b25554c72de7d9319a35dd7 diff --git a/contrib/faiss b/contrib/faiss index 6c79cb2e7..bd63f3a7e 160000 --- a/contrib/faiss +++ b/contrib/faiss @@ -1 +1 @@ -Subproject commit 6c79cb2e74947d294b87cb66da3515406d6f9831 +Subproject commit bd63f3a7eafb852b74d184d7fb6bd182e4d0d4b7 diff --git a/contrib/hdf5 b/contrib/hdf5 index 8b5cac6bc..f0ecc8bc2 160000 --- a/contrib/hdf5 +++ b/contrib/hdf5 @@ -1 +1 @@ -Subproject commit 8b5cac6bc498546efa5639f99bb7dbbc1a2d5d90 +Subproject commit f0ecc8bc26972119fb31b1cd548ea23fff4a3227 diff --git a/contrib/libbacktrace b/contrib/libbacktrace index 7ead8c1ea..ae1e707db 160000 --- a/contrib/libbacktrace +++ b/contrib/libbacktrace @@ -1 +1 @@ -Subproject commit 7ead8c1ea2f4aeafe9c5b9ef8a9461a9ba781aa8 +Subproject commit ae1e707dbacd4a5cc82fcf2d3816f410e9c5fec4 diff --git a/contrib/openblas b/contrib/openblas index 6c77e5e31..ce3f668c9 160000 --- a/contrib/openblas +++ b/contrib/openblas @@ -1 +1 @@ -Subproject commit 6c77e5e314474773a7749357b153caba4ec3817d +Subproject commit ce3f668c992cb3cc80849d5c30ed637f5adbd5b2 diff --git a/contrib/openssl b/contrib/openssl index a7e992847..4cb31128b 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit a7e992847de83aa36be0c399c89db3fb827b0be2 +Subproject commit 4cb31128b5790819dfeea2739fbde265f71a10a2 diff --git a/contrib/rapidjson b/contrib/rapidjson index 5ec44fb92..ab1842a2d 160000 --- a/contrib/rapidjson +++ b/contrib/rapidjson @@ -1 +1 @@ -Subproject commit 5ec44fb9206695e5293f610b0a46d21851d0c966 +Subproject commit ab1842a2dae061284c0a62dca1cc6d5e7e37e346 diff --git a/contrib/rocksdb b/contrib/rocksdb index c2467b141..6f7cabeac 160000 --- a/contrib/rocksdb +++ b/contrib/rocksdb @@ -1 +1 @@ -Subproject commit c2467b141e840fdba5b3a1810763043e56449fb9 +Subproject commit 6f7cabeac80a3a6150be2c8a8369fcecb107bf43 diff --git a/contrib/zstd b/contrib/zstd index 63779c798..794ea1b0a 160000 --- a/contrib/zstd +++ b/contrib/zstd @@ -1 +1 @@ -Subproject commit 63779c798237346c2b245c546c40b72a5a5913fe +Subproject commit 794ea1b0afca0f020f4e57b6732332231fb23c70 diff --git a/src/engine/rocks_raw_engine.cc b/src/engine/rocks_raw_engine.cc index cc17ee6a3..3fcc589a4 100644 --- a/src/engine/rocks_raw_engine.cc +++ b/src/engine/rocks_raw_engine.cc @@ -213,7 +213,7 @@ butil::Status Checkpoint::Create(const std::string& dirpath, const std::vectorCreateCheckpoint(dirpath); if (!status.ok()) { - GetDB()->EnableFileDeletions(false); + GetDB()->EnableFileDeletions(); DINGO_LOG(ERROR) << fmt::format("[rocksdb] export column family checkpoint failed, error: {}.", status.ToString()); delete checkpoint; return butil::Status(status.code(), status.ToString()); @@ -228,7 +228,7 @@ butil::Status Checkpoint::Create(const std::string& dirpath, const std::vectorEnableFileDeletions(false); + status = GetDB()->EnableFileDeletions(); if (!status.ok()) { DINGO_LOG(ERROR) << fmt::format("[rocksdb] enable file deletion failed, error: {}.", status.ToString()); return butil::Status(status.code(), status.ToString());