Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@SiarheiFedartsou
SiarheiFedartsou committed Oct 29, 2024
1 parent 7c1f0bf commit 12163b0
Show file tree
Hide file tree
Showing 4 changed files with 130 additions and 177 deletions.
257 changes: 127 additions & 130 deletions .github/workflows/osrm-backend.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ concurrency:

jobs:
windows-release-node:
#needs: format-taginfo-docs
needs: format-taginfo-docs
runs-on: windows-2022
continue-on-error: false
env:
Expand All @@ -51,11 +51,8 @@ jobs:
cd build
python3 -m venv .venv
source .venv/Scripts/Activate
python3 -m pip install conan==2.7.1
conan profile detect --force
cmake -DCMAKE_BUILD_TYPE=Release -DENABLE_CONAN=ON -DENABLE_NODE_BINDINGS=ON ..
Expand Down Expand Up @@ -657,130 +654,130 @@ jobs:
ccache -p
ccache -s
# benchmarks:
# if: github.event_name == 'pull_request'
# needs: [format-taginfo-docs]
# runs-on: self-hosted
# env:
# CCOMPILER: clang-16
# CXXCOMPILER: clang++-16
# CC: clang-16
# CXX: clang++-16
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# PR_NUMBER: ${{ github.event.pull_request.number }}
# GITHUB_REPOSITORY: ${{ github.repository }}
# RUN_BIG_BENCHMARK: ${{ contains(github.event.pull_request.labels.*.name, 'Performance') }}
# steps:
# - name: Checkout PR Branch
# uses: actions/checkout@v4
# with:
# ref: ${{ github.head_ref }}
# path: pr
# - name: Activate virtualenv
# run: |
# python3 -m venv .venv
# source .venv/bin/activate
# echo PATH=$PATH >> $GITHUB_ENV
# pip install "conan==2.7.1" "requests==2.31.0" "numpy==1.26.4"
# - name: Prepare data
# run: |
# if [ "$RUN_BIG_BENCHMARK" = "true" ]; then
# rm -rf ~/data.osm.pbf
# wget http://download.geofabrik.de/europe/poland-latest.osm.pbf -O ~/data.osm.pbf --quiet
# gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
# else
# if [ ! -f "~/data.osm.pbf" ]; then
# wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
# else
# echo "Using cached data.osm.pbf"
# fi
# gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
# fi
# - name: Prepare environment
# run: |
# echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
# mkdir -p $HOME/.ccache
# ccache --zero-stats
# ccache --max-size=256M
# - name: Checkout Base Branch
# uses: actions/checkout@v4
# with:
# ref: ${{ github.event.pull_request.base.ref }}
# path: base
# - name: Build Base Branch
# run: |
# cd base
# npm ci --ignore-scripts
# cd ..
# mkdir base/build
# cd base/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
# make -j$(nproc)
# make -j$(nproc) benchmarks
# cd ..
# make -C test/data
# - name: Build PR Branch
# run: |
# cd pr
# npm ci --ignore-scripts
# cd ..
# mkdir -p pr/build
# cd pr/build
# cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
# make -j$(nproc)
# make -j$(nproc) benchmarks
# cd ..
# make -C test/data
# # we run benchmarks in tmpfs to avoid impact of disk IO
# - name: Create folder for tmpfs
# run: |
# # if by any chance it was mounted before(e.g. due to previous job failed), unmount it
# sudo umount ~/benchmarks | true
# rm -rf ~/benchmarks
# mkdir -p ~/benchmarks
# # see https://llvm.org/docs/Benchmarking.html
# - name: Run PR Benchmarks
# run: |
# sudo cset shield -c 2-3 -k on
# sudo mount -t tmpfs -o size=4g none ~/benchmarks
# cp -rf pr/build ~/benchmarks/build
# cp -rf pr/lib ~/benchmarks/lib
# mkdir -p ~/benchmarks/test
# cp -rf pr/test/data ~/benchmarks/test/data
# cp -rf pr/profiles ~/benchmarks/profiles

# sudo cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
# sudo umount ~/benchmarks
# sudo cset shield --reset
# - name: Run Base Benchmarks
# run: |
# sudo cset shield -c 2-3 -k on
# sudo mount -t tmpfs -o size=4g none ~/benchmarks
# cp -rf base/build ~/benchmarks/build
# cp -rf base/lib ~/benchmarks/lib
# mkdir -p ~/benchmarks/test
# cp -rf base/test/data ~/benchmarks/test/data
# cp -rf base/profiles ~/benchmarks/profiles

# # TODO: remove it when base branch will have this file at needed location
# if [ ! -f ~/benchmarks/test/data/portugal_to_korea.json ]; then
# cp base/src/benchmarks/portugal_to_korea.json ~/benchmarks/test/data/portugal_to_korea.json
# fi
# # we intentionally use scripts from PR branch to be able to update them and see results in the same PR
# sudo cset shield --exec -- cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
# sudo umount ~/benchmarks
# sudo cset shield --reset
# - name: Post Benchmark Results
# run: |
# python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
# - name: Show CCache statistics
# run: |
# ccache -p
# ccache -s

# ci-complete:
# runs-on: ubuntu-22.04
# needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks]
# steps:
# - run: echo "CI complete"
benchmarks:
if: github.event_name == 'pull_request'
needs: [format-taginfo-docs]
runs-on: self-hosted
env:
CCOMPILER: clang-16
CXXCOMPILER: clang++-16
CC: clang-16
CXX: clang++-16
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ github.event.pull_request.number }}
GITHUB_REPOSITORY: ${{ github.repository }}
RUN_BIG_BENCHMARK: ${{ contains(github.event.pull_request.labels.*.name, 'Performance') }}
steps:
- name: Checkout PR Branch
uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
path: pr
- name: Activate virtualenv
run: |
python3 -m venv .venv
source .venv/bin/activate
echo PATH=$PATH >> $GITHUB_ENV
pip install "conan==2.7.1" "requests==2.31.0" "numpy==1.26.4"
- name: Prepare data
run: |
if [ "$RUN_BIG_BENCHMARK" = "true" ]; then
rm -rf ~/data.osm.pbf
wget http://download.geofabrik.de/europe/poland-latest.osm.pbf -O ~/data.osm.pbf --quiet
gunzip -c ./pr/test/data/poland_gps_traces.csv.gz > ~/gps_traces.csv
else
if [ ! -f "~/data.osm.pbf" ]; then
wget http://download.geofabrik.de/europe/germany/berlin-latest.osm.pbf -O ~/data.osm.pbf
else
echo "Using cached data.osm.pbf"
fi
gunzip -c ./pr/test/data/berlin_gps_traces.csv.gz > ~/gps_traces.csv
fi
- name: Prepare environment
run: |
echo "CCACHE_DIR=$HOME/.ccache" >> $GITHUB_ENV
mkdir -p $HOME/.ccache
ccache --zero-stats
ccache --max-size=256M
- name: Checkout Base Branch
uses: actions/checkout@v4
with:
ref: ${{ github.event.pull_request.base.ref }}
path: base
- name: Build Base Branch
run: |
cd base
npm ci --ignore-scripts
cd ..
mkdir base/build
cd base/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
- name: Build PR Branch
run: |
cd pr
npm ci --ignore-scripts
cd ..
mkdir -p pr/build
cd pr/build
cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON ..
make -j$(nproc)
make -j$(nproc) benchmarks
cd ..
make -C test/data
# we run benchmarks in tmpfs to avoid impact of disk IO
- name: Create folder for tmpfs
run: |
# if by any chance it was mounted before(e.g. due to previous job failed), unmount it
sudo umount ~/benchmarks | true
rm -rf ~/benchmarks
mkdir -p ~/benchmarks
# see https://llvm.org/docs/Benchmarking.html
- name: Run PR Benchmarks
run: |
sudo cset shield -c 2-3 -k on
sudo mount -t tmpfs -o size=4g none ~/benchmarks
cp -rf pr/build ~/benchmarks/build
cp -rf pr/lib ~/benchmarks/lib
mkdir -p ~/benchmarks/test
cp -rf pr/test/data ~/benchmarks/test/data
cp -rf pr/profiles ~/benchmarks/profiles
sudo cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/pr_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount ~/benchmarks
sudo cset shield --reset
- name: Run Base Benchmarks
run: |
sudo cset shield -c 2-3 -k on
sudo mount -t tmpfs -o size=4g none ~/benchmarks
cp -rf base/build ~/benchmarks/build
cp -rf base/lib ~/benchmarks/lib
mkdir -p ~/benchmarks/test
cp -rf base/test/data ~/benchmarks/test/data
cp -rf base/profiles ~/benchmarks/profiles
# TODO: remove it when base branch will have this file at needed location
if [ ! -f ~/benchmarks/test/data/portugal_to_korea.json ]; then
cp base/src/benchmarks/portugal_to_korea.json ~/benchmarks/test/data/portugal_to_korea.json
fi
# we intentionally use scripts from PR branch to be able to update them and see results in the same PR
sudo cset shield --exec -- cset shield --exec -- ./pr/scripts/ci/run_benchmarks.sh -f ~/benchmarks -r $(pwd)/base_results -s $(pwd)/pr -b ~/benchmarks/build -o ~/data.osm.pbf -g ~/gps_traces.csv
sudo umount ~/benchmarks
sudo cset shield --reset
- name: Post Benchmark Results
run: |
python3 pr/scripts/ci/post_benchmark_results.py base_results pr_results
- name: Show CCache statistics
run: |
ccache -p
ccache -s
ci-complete:
runs-on: ubuntu-22.04
needs: [build-test-publish, docker-image-matrix, windows-release-node, benchmarks]
steps:
- run: echo "CI complete"

6 changes: 2 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ if (NOT WIN32 AND NOT DEFINED ENV{OSRM_BUILD_DIR})
set(ENV{OSRM_BUILD_DIR} ${CMAKE_CURRENT_BINARY_DIR})
endif()

option(ENABLE_CONAN "Use conan for dependencies" OFF)
option(ENABLE_CCACHE "Speed up incremental rebuilds via ccache" ON)
option(BUILD_TOOLS "Build OSRM tools" OFF)
option(BUILD_PACKAGE "Build OSRM package" OFF)
Expand All @@ -45,19 +46,16 @@ if (ENABLE_CONAN)
else()
set(CMAKE_TOOLCHAIN_FILE "./conan_toolchain.cmake")

# message(STATUS "Conan install stdout: ${CONAN_EXECUTABLE} install .. --output-folder=. --build=missing --settings compiler.cppstd=20 --settings build_type=${CMAKE_BUILD_TYPE}")
execute_process(
COMMAND ${CONAN_EXECUTABLE} install .. --output-folder=. --build=missing --settings compiler.cppstd=20 --settings build_type=${CMAKE_BUILD_TYPE}
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
OUTPUT_VARIABLE conan_stdout
ERROR_VARIABLE conan_stderr
RESULT_VARIABLE conan_install_result
)
message(STATUS "Conan install stdout: ${conan_stdout} ${CMAKE_BUILD_TYPE}")
message(STATUS "Conan install stderr: ${conan_stderr}")

if (NOT conan_install_result EQUAL 0)
message(FATAL_ERROR "Conan install failed.")
message(FATAL_ERROR "Conan install failed: ${conan_install_result}. Stderr: ${conan_stderr}. Stdout: ${conan_stdout}")
endif()
endif()
endif()
Expand Down
42 changes: 0 additions & 42 deletions benchmark.js
View file

This file was deleted.

2 changes: 1 addition & 1 deletion unit_tests/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ add_executable(util-tests
${UtilTestsSources}
$<TARGET_OBJECTS:UTIL> $<TARGET_OBJECTS:MICROTAR>)

if(NOT WIN32 AND NOT Boost_USE_STATIC_LIBS AND NOT ENABLE_CONAN)
if(NOT WIN32 AND NOT Boost_USE_STATIC_LIBS)
add_definitions(-DBOOST_TEST_DYN_LINK)
endif()

Expand Down

0 comments on commit 12163b0

Please sign in to comment.