cmake: upgrade to Conan 2 (#2318)

.circleci/config.yml

@@ -85,7 +85,7 @@ commands:
       - run:
           name: "Install Conan"
           command: |
-            pip3 install --user --no-warn-script-location conan==1.64.1
+            pip3 install --user --no-warn-script-location conan==2.10.2
             conan_path="$(python3 -m site --user-base)/bin"
             echo "export \"PATH=$conan_path:\$PATH\"" >> "$BASH_ENV"
             "$conan_path/conan" --version
@@ -128,7 +128,7 @@ commands:
           name: "Save Conan cache"
           key: *conan-cache-key
           paths:
-            - ~/.conan
+            - ~/.conan2
 
   build_fuzzer:
     steps:

.github/actions/fuzzer-common-steps/action.yml

@@ -23,17 +23,12 @@ runs:
 
     - name: Temporary step - conan cache cleanup - to be executed only once per runner
       shell: bash
-      run: conan remove "*" --force
-
-    - name: Preinstall Conan packages
-      shell: bash
-      working-directory: ${{runner.workspace}}/silkworm
-      run: CC=clang-16 CXX=clang++-16 conan install --install-folder=build/conan --build=missing --profile=cmake/profiles/experimental/linux_x64_clang_16_release .
+      run: /opt/conan2/bin/conan remove --confirm "*"
 
     - name: Configure CMake
       shell: bash
       working-directory: ${{runner.workspace}}/silkworm/build
-      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=experimental/linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
+      run: CC=clang-16 CXX=clang++-16 cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCONAN_PROFILE=linux_x64_clang_16_release -DCMAKE_TOOLCHAIN_FILE=../cmake/toolchain/clang_libcxx.cmake -DSILKWORM_FUZZER=ON -DSILKWORM_FUZZER_SANITIZERS=${{inputs.fuzzer_sanitizers}}
 
     - name: Build Silkworm Fuzzer Test
       shell: bash

.github/workflows/macOS.yml

@@ -47,7 +47,7 @@ jobs:
 
     - name: Install Prerequisites
       run: |
-        pip3 install --user --break-system-packages --no-warn-script-location conan==1.64.1 chardet
+        pip3 install --user --break-system-packages --no-warn-script-location conan==2.10.2 chardet
         conan_path="$(python3 -m site --user-base)/bin"
         echo "$conan_path" >> $GITHUB_PATH
         "$conan_path/conan" --version

.github/workflows/windows.yml

@@ -47,7 +47,7 @@ jobs:
       id: conan
       uses: turtlebrowser/get-conan@main
       with:
-        version: 1.64.1
+        version: 2.10.2
 
     - name: Create Build Environment
       # Some projects don't allow in-source building, so create a separate build directory

CMakeLists.txt

@@ -14,25 +14,38 @@
    limitations under the License.
 ]]
 
-cmake_minimum_required(VERSION 3.19.0)
+cmake_minimum_required(VERSION 3.24.0)
 
 if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/third_party/evmone/evmone/evmc/.git)
   message(FATAL_ERROR "Git submodules not initialized, execute:\n  git submodule update --init --recursive")
 endif()
 
 get_directory_property(SILKWORM_HAS_PARENT PARENT_DIRECTORY)
 if(NOT SILKWORM_HAS_PARENT)
+  # reduce the log verbosity of evmone/cmake/cable
+  if(NOT CMAKE_MESSAGE_LOG_LEVEL)
+    set(CMAKE_MESSAGE_LOG_LEVEL_EMPTY YES)
+    set(CMAKE_MESSAGE_LOG_LEVEL NOTICE)
+  endif()
+
   include(third_party/evmone/evmone/cmake/cable/bootstrap.cmake)
   include(CableBuildType)
   cable_set_build_type(DEFAULT Release CONFIGURATION_TYPES Release Debug)
 
+  # restore the log verbosity
+  if(CMAKE_MESSAGE_LOG_LEVEL_EMPTY)
+    unset(CMAKE_MESSAGE_LOG_LEVEL)
+  endif()
+
   if(NOT CMAKE_TOOLCHAIN_FILE)
     set(CMAKE_TOOLCHAIN_FILE
         ${CMAKE_CURRENT_SOURCE_DIR}/cmake/toolchain/cxx20.cmake
         CACHE FILEPATH "" FORCE
     )
     include("${CMAKE_TOOLCHAIN_FILE}")
   endif()
+
+  include(cmake/conan.cmake)
 endif()
 
 project(silkworm)
@@ -70,10 +83,6 @@ option(SILKWORM_FUZZER_SANITIZERS "CLang sanitizer options for fuzzers" OFF)
 option(SILKWORM_USE_MIMALLOC "Enable using mimalloc for dynamic memory management" ON)
 option(SILKWORM_ALLOW_UNUSED_VAR_WARNINGS "Turn unused variable errors into warnings" OFF)
 
-if(NOT SILKWORM_HAS_PARENT)
-  include(cmake/conan.cmake)
-endif()
-
 set_property(
   DIRECTORY
   APPEND
@@ -125,5 +134,5 @@ if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
   # avoid fatal error C1002: compiler is out of heap space
   list(REMOVE_ITEM UNIT_TEST_TARGETS silkworm_rpcdaemon_test)
 endif()
-message(STATUS "UNIT_TEST_TARGETS: ${UNIT_TEST_TARGETS}")
+message(VERBOSE "UNIT_TEST_TARGETS: ${UNIT_TEST_TARGETS}")
 add_custom_target(all_unit_tests DEPENDS ${UNIT_TEST_TARGETS})

README.md

@@ -98,7 +98,7 @@ or AppleClang ([Xcode](https://developer.apple.com/xcode/) >= 15)
 
 Conan requires Python, and can be installed using:
 
-    pip3 install --user conan==1.64.1 chardet
+    pip3 install --user conan==2.10.2 chardet
 
 On Linux the conan binary gets installed into `$HOME/.local/bin` which is typically in PATH already.
 On macOS need to add the binary to PATH manually:
@@ -122,9 +122,7 @@ will use "debug" configuration builds of dependencies.
 
 See available profiles in [cmake/profiles](cmake/profiles).
 
-The conan packages could also be pre-installed using [conan install](https://docs.conan.io/1/reference/commands/consumer/install.html):
-
-    conan install --install-folder=build/conan --build=missing --profile=cmake/profiles/macos_arm64_clang_13_debug .
+During the cmake configuration step `conan_provider.cmake` runs a [conan install](https://docs.conan.io/2/reference/commands/install.html) command that downloads packages and builds some of them from source if needed. The exact arguments to this command are printed in the build log.
 
 
 Then run the build itself

cmake/compiler_settings.cmake

@@ -89,16 +89,6 @@ elseif("${CMAKE_CXX_COMPILER_ID}" MATCHES ".*Clang$")
     link_libraries(c++abi)
   endif()
 
-  # cmake-format: off
-  # abseil in conan is prebuilt with clang 13 (see profiles),
-  # linking absl::log with clang 18+ produces an error due to an ABI change:
-  # https://github.com/abseil/abseil-cpp/issues/1747
-  # https://github.com/llvm/llvm-project/issues/102443
-  # cmake-format: on
-  if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 18)
-    add_compile_options(-fclang-abi-compat=17)
-  endif()
-
 else()
   message(WARNING "${CMAKE_CXX_COMPILER_ID} is not a supported compiler. Use at your own risk.")
 endif()

cmake/conan.cmake

@@ -15,8 +15,18 @@
 ]]
 
 include(${CMAKE_CURRENT_LIST_DIR}/compiler_settings_sanitize.cmake)
+include(${CMAKE_CURRENT_LIST_DIR}/conan_quiet.cmake)
 
 function(guess_conan_profile)
+  if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "" AND CMAKE_HOST_UNIX)
+    execute_process(
+      COMMAND uname -m
+      OUTPUT_VARIABLE CMAKE_HOST_SYSTEM_PROCESSOR
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      COMMAND_ERROR_IS_FATAL ANY
+    )
+  endif()
+
   if("${CMAKE_HOST_SYSTEM_PROCESSOR}" STREQUAL "")
     set(ARCH_NAME "")
   elseif(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL x86_64)
@@ -38,7 +48,7 @@ function(guess_conan_profile)
   elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin" AND ARCH_NAME)
     set(PROFILE macos_${ARCH_NAME}_clang_13_release)
   elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
-    set(PROFILE windows_msvc_16_release)
+    set(PROFILE windows_msvc_193_release)
   else()
     message(FATAL_ERROR "CONAN_PROFILE is not defined for ${CMAKE_HOST_SYSTEM_NAME} on ${CMAKE_HOST_SYSTEM_PROCESSOR}")
   endif()
@@ -49,70 +59,131 @@ function(guess_conan_profile)
   )
 endfunction()
 
-set(CONAN_BINARY_DIR "${CMAKE_BINARY_DIR}/conan")
-list(APPEND CMAKE_MODULE_PATH ${CONAN_BINARY_DIR})
-list(APPEND CMAKE_PREFIX_PATH ${CONAN_BINARY_DIR})
+function(get_conan_build_type profile_path var)
+  file(READ "${profile_path}" CONTENTS)
+  string(REGEX MATCH "build_type=[A-Za-z0-9]+" VALUE "${CONTENTS}")
+  string(SUBSTRING "${VALUE}" 11 -1 VALUE)
+  set(${var}
+      "${VALUE}"
+      PARENT_SCOPE
+  )
+endfunction()
 
-# disable verbose logging from FindXXX.cmake files
-set(CONAN_CMAKE_SILENT_OUTPUT ON)
+macro(format_list_as_json_array list_var var)
+  list(JOIN ${list_var} "\",\"" ${var})
+  set(${var} "[\"${${var}}\"]")
+endmacro()
 
-include("${CMAKE_SOURCE_DIR}/third_party/cmake-conan/conan.cmake")
+# unset(CONAN_COMMAND CACHE)
+find_program(
+  CONAN_COMMAND "conan"
+  PATHS /opt/conan2/bin
+  NO_DEFAULT_PATH
+)
+if(NOT CONAN_COMMAND)
+  find_program(CONAN_COMMAND "conan" PATHS ~/.local/bin REQUIRED)
+endif()
 
-# provide a static conanfile.py instead of generating it with conan_cmake_configure()
-file(COPY "${CMAKE_SOURCE_DIR}/conanfile.py" DESTINATION "${CONAN_BINARY_DIR}")
+# use "verbose" for more detailed conan install logs
+set(CONAN_VERBOSITY "error")
+set(CONAN_BINARY_DIR "${CMAKE_BINARY_DIR}/conan2")
 
 if(NOT DEFINED CONAN_PROFILE)
   guess_conan_profile()
 endif()
-message(STATUS "CONAN_PROFILE: ${CONAN_PROFILE}")
+message(VERBOSE "CONAN_PROFILE: ${CONAN_PROFILE}")
+set(CONAN_PROFILE_PATH "${CMAKE_SOURCE_DIR}/cmake/profiles/${CONAN_PROFILE}")
+set(CONAN_HOST_PROFILE "${CONAN_PROFILE_PATH}")
+set(CONAN_BUILD_PROFILE "${CONAN_PROFILE_PATH}")
+get_conan_build_type("${CONAN_PROFILE_PATH}" CONAN_BUILD_TYPE)
 
 set(CONAN_BUILD "missing")
-set(CONAN_CXXFLAGS_ARG)
-set(CONAN_OPTIONS)
+set(CONAN_SETTINGS "")
+set(CONAN_OPTIONS "")
+set(CONAN_CONF "")
+
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
+  set(OS_VERSION_MIN_CXXFLAG "-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}")
+endif()
+
+if(OS_VERSION_MIN_CXXFLAG AND NOT SILKWORM_SANITIZE_COMPILER_OPTIONS)
+  list(APPEND CONAN_CONF "libtorrent/*:tools.build:cxxflags=[\"${OS_VERSION_MIN_CXXFLAG}\"]")
+endif()
 
 if(SILKWORM_SANITIZE_COMPILER_OPTIONS)
   set(CONAN_CXXFLAGS ${SILKWORM_SANITIZE_COMPILER_OPTIONS})
 
-  if(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
-    list(APPEND CONAN_CXXFLAGS "-mmacosx-version-min=${CMAKE_OSX_DEPLOYMENT_TARGET}")
+  if(OS_VERSION_MIN_CXXFLAG)
+    list(APPEND CONAN_CXXFLAGS ${OS_VERSION_MIN_CXXFLAG})
   endif()
 
-  list(JOIN CONAN_CXXFLAGS "\", \"" CONAN_CXXFLAGS_STR)
-  set(CONAN_CXXFLAGS_STR "[\"${CONAN_CXXFLAGS_STR}\"]")
-  set(CONAN_CXXFLAGS_ARG "tools.build:cxxflags=${CONAN_CXXFLAGS_STR}")
-
-  list(APPEND CONAN_OPTIONS "boost:zlib=False")
+  list(APPEND CONAN_OPTIONS "boost/*:zlib=False")
 
   # libraries that needs to be rebuilt with sanitize flags
   # cmake-format: off
-  set(CONAN_BUILD
+  list(APPEND CONAN_BUILD
       abseil
       boost
       grpc
       libtorrent
       protobuf
   )
   # cmake-format: on
+
+  format_list_as_json_array(CONAN_CXXFLAGS CONAN_CXXFLAGS_STR)
+  foreach(PACKAGE IN LISTS CONAN_BUILD)
+    if(NOT PACKAGE STREQUAL "missing")
+      list(APPEND CONAN_CONF "${PACKAGE}/*:tools.build:cxxflags=${CONAN_CXXFLAGS_STR}")
+    endif()
+  endforeach()
+endif()
+
+if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
+  set(CONAN_VERBOSITY "verbose")
+  set(CONAN_BUILD "missing:libtorrent/*")
+  # most Windows packages on ConanCenter are built for cppstd=14,
+  # but some require at least cppstd=17 (otherwise report "Invalid" status)
+  list(APPEND CONAN_SETTINGS "asio-grpc/*:compiler.cppstd=17")
+  list(APPEND CONAN_SETTINGS "magic_enum/*:compiler.cppstd=17")
+  list(APPEND CONAN_SETTINGS "tomlplusplus/*:compiler.cppstd=17")
 endif()
 
 if(SILKWORM_USE_MIMALLOC)
   # mimalloc override option causes a crash on macOS at startup in rpcdaemon, so we enable it just on Linux. mimalloc
   # should not be used in sanitizer builds or at least its override option must be disabled
   # (https://github.com/microsoft/mimalloc/issues/317#issuecomment-708506405)
   if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux" AND NOT SILKWORM_SANITIZE)
-    list(APPEND CONAN_OPTIONS "mimalloc:override=True")
+    list(APPEND CONAN_OPTIONS "mimalloc/*:override=True")
   endif()
 endif()
 
 if(SILKWORM_CORE_ONLY)
-  set(CONAN_CXXFLAGS_ARG "catch2/*:tools.build:cxxflags=[\"-fno-exceptions\"]")
+  list(APPEND CONAN_CONF "catch2/*:tools.build:cxxflags=[\"-fno-exceptions\"]")
 endif()
 
-conan_cmake_install(
-  PATH_OR_REFERENCE "${CONAN_BINARY_DIR}"
-  INSTALL_FOLDER "${CONAN_BINARY_DIR}"
-  BUILD ${CONAN_BUILD}
-  OPTIONS ${CONAN_OPTIONS}
-  PROFILE "${CMAKE_SOURCE_DIR}/cmake/profiles/${CONAN_PROFILE}"
-  CONF "${CONAN_CXXFLAGS_ARG}"
+# cmake-format: off
+set(CONAN_INSTALL_ARGS
+    -v ${CONAN_VERBOSITY}
+    --output-folder "${CONAN_BINARY_DIR}"
+    # https://github.com/conan-io/cmake-conan/issues/607
+    --settings:all "&:build_type=${CMAKE_BUILD_TYPE}"
 )
+# cmake-format: on
+
+foreach(VALUE IN LISTS CONAN_BUILD)
+  list(APPEND CONAN_INSTALL_ARGS --build=${VALUE})
+endforeach()
+
+foreach(VALUE IN LISTS CONAN_SETTINGS)
+  list(APPEND CONAN_INSTALL_ARGS --settings:all=${VALUE})
+endforeach()
+
+foreach(VALUE IN LISTS CONAN_OPTIONS)
+  list(APPEND CONAN_INSTALL_ARGS --options:all=${VALUE})
+endforeach()
+
+foreach(VALUE IN LISTS CONAN_CONF)
+  list(APPEND CONAN_INSTALL_ARGS --conf:all=${VALUE})
+endforeach()
+
+set(CMAKE_PROJECT_TOP_LEVEL_INCLUDES "${CMAKE_SOURCE_DIR}/third_party/cmake-conan/conan_provider.cmake")

cmake/conan_quiet.cmake

@@ -0,0 +1,38 @@
+# Reduce verbosity of CMakeDeps conan generator
+# do not edit, regenerate with conan_quiet.sh
+
+set(ZLIB_FIND_QUIETLY YES)
+set(Catch2_FIND_QUIETLY YES)
+set(jwt-cpp_FIND_QUIETLY YES)
+set(GTest_FIND_QUIETLY YES)
+set(LibtorrentRasterbar_FIND_QUIETLY YES)
+set(Snappy_FIND_QUIETLY YES)
+set(Microsoft.GSL_FIND_QUIETLY YES)
+set(OpenSSL_FIND_QUIETLY YES)
+set(fmt_FIND_QUIETLY YES)
+set(roaring_FIND_QUIETLY YES)
+set(BZip2_FIND_QUIETLY YES)
+set(c-ares_FIND_QUIETLY YES)
+set(magic_enum_FIND_QUIETLY YES)
+set(absl_FIND_QUIETLY YES)
+set(OpenSSL_FIND_QUIETLY YES)
+set(tomlplusplus_FIND_QUIETLY YES)
+set(spdlog_FIND_QUIETLY YES)
+set(SQLite3_FIND_QUIETLY YES)
+set(CLI11_FIND_QUIETLY YES)
+set(tl-expected_FIND_QUIETLY YES)
+set(asio-grpc_FIND_QUIETLY YES)
+set(benchmark_FIND_QUIETLY YES)
+set(gmp_FIND_QUIETLY YES)
+set(Boost_FIND_QUIETLY YES)
+set(SQLite3_FIND_QUIETLY YES)
+set(mimalloc_FIND_QUIETLY YES)
+set(re2_FIND_QUIETLY YES)
+set(gRPC_FIND_QUIETLY YES)
+set(protobuf_FIND_QUIETLY YES)
+set(ZLIB_FIND_QUIETLY YES)
+set(Protobuf_FIND_QUIETLY YES)
+set(BZip2_FIND_QUIETLY YES)
+set(SQLiteCpp_FIND_QUIETLY YES)
+set(GTest_FIND_QUIETLY YES)
+set(nlohmann_json_FIND_QUIETLY YES)

cmake/conan_quiet.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+project_dir="$script_dir/.."
+build_dir="$1"
+
+if [[ -z "$build_dir" ]]
+then
+	build_dir="$project_dir/build"
+fi
+
+cat << EOF > "$script_dir/conan_quiet.cmake"
+# Reduce verbosity of CMakeDeps conan generator
+# do not edit, regenerate with conan_quiet.sh
+
+EOF
+
+grep -R FIND_QUIETLY "$build_dir/conan2" | sed -E 's/.+\((.+)\)/set(\1 YES)/' >> "$script_dir/conan_quiet.cmake"