Finish thread sanitizer linker flags #109

Workflow file for this run

	name: Flow-IPC pipeline

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main
	# To run a workflow manually, the workflow must be configured to run on the workflow_dispatch event.
	workflow_dispatch:

	jobs:
	doc:
	strategy:
	fail-fast: false
	matrix:
	# Just pick one that's pre-installed and reasonably new.
	compiler:
	- id: clang-15
	name: clang
	version: 15
	c-path: /usr/bin/clang-15
	cpp-path: /usr/bin/clang++-15
	build-type:
	- id: release
	conan-build-type: Release
	conan-preset: release

	runs-on: ubuntu-latest

	name: \|
	doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}

	steps:
	- name: Update available software list for apt-get
	run: sudo apt-get update

	- name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`)
	uses: actions/checkout@v4
	with:
	submodules: true

	- name: Install Flow-IPC dependencies (inc. Graphviz) with apt-get
	run: \|
	sudo apt-get install -y graphviz

	- name: Install the latest version of Conan which is less than 2
	run: \|
	pip install "conan<2"

	- name: Create Conan profile
	run: \|
	cat <<EOF > conan_profile
	[settings]
	compiler = ${{ matrix.compiler.name }}
	compiler.version = ${{ matrix.compiler.version }}
	compiler.cppstd = 17
	compiler.libcxx = libstdc++11
	arch = x86_64
	os = Linux
	build_type = ${{ matrix.build-type.conan-build-type }}

	[conf]
	tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"}
	tools.env.virtualenv:auto_use = True

	[buildenv]
	CC = ${{ matrix.compiler.c-path }}
	CXX = ${{ matrix.compiler.cpp-path }}

	[options]
	ipc:build = False
	ipc:doc = True
	EOF

	- name: Install Flow-IPC dependencies (inc. Doxygen) with Conan using the profile
	run: \|
	conan install \
	. \
	--profile:build conan_profile \
	--profile:host conan_profile \
	--build missing

	- name: Prepare Makefile using CMake
	run: \|
	cmake \
	--preset ${{ matrix.build-type.conan-preset }} \
	-DCFG_ENABLE_DOC_GEN=ON \
	-DCFG_SKIP_CODE_GEN=ON

	- name: Get the number of processor cores for parallelized work
	uses: SimenB/github-actions-cpu-cores@v1
	id: cpu-cores

	# Note that `flow_doc_public` and `flow_doc_full` targets are also available; and a user locally
	# may well want to use them. We, however, have chosen to dedicate a Flow-only workflow
	# (in `flow` repo) to Flow; it will generate the Flow docs. Meanwhile ipc_doc_* targets create
	# monolithic documentation from the ipc_* submodules and `ipc` itself -- but not from
	# `flow` submodule.
	- name: Doxygen-generate documentation sets using Makefile
	run: \|
	VERBOSE=1 make \
	ipc_doc_public \
	ipc_doc_full \
	--directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
	-j${{ steps.cpu-cores.outputs.count }}

	- name: Package doc tarball [Doxygen documentation sets (full, API-only), landing page]
	run: \|
	cd $GITHUB_WORKSPACE/doc/ipc_doc
	$GITHUB_WORKSPACE/tools/doc/stage_generated_docs.sh $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }}
	rm -rf generated # Save runner space.

	- name: Upload documentation tarball
	uses: actions/upload-artifact@v3
	with:
	name: ipc-doc-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}
	path: \|
	${{ github.workspace }}/doc/ipc_doc.tgz

	build:
	strategy:
	fail-fast: false
	matrix:
	compiler:
	- id: gcc-9
	name: gcc
	version: 9
	c-path: /usr/bin/gcc-9
	cpp-path: /usr/bin/g++-9
	- id: gcc-10
	name: gcc
	version: 10
	c-path: /usr/bin/gcc-10
	cpp-path: /usr/bin/g++-10
	- id: gcc-11
	name: gcc
	version: 11
	c-path: /usr/bin/gcc-11
	cpp-path: /usr/bin/g++-11
	- id: gcc-13
	name: gcc
	version: 13
	c-path: /usr/bin/gcc-13
	cpp-path: /usr/bin/g++-13
	- id: clang-13
	name: clang
	version: 13
	c-path: /usr/bin/clang-13
	cpp-path: /usr/bin/clang++-13
	- id: clang-15
	name: clang
	version: 15
	c-path: /usr/bin/clang-15
	cpp-path: /usr/bin/clang++-15
	- id: clang-16
	name: clang
	version: 16
	c-path: /usr/bin/clang-16
	cpp-path: /usr/bin/clang++-16
	install: True
	- id: clang-17
	name: clang
	version: 17
	c-path: /usr/bin/clang-17
	cpp-path: /usr/bin/clang++-17
	install: True
	build-type:
	- id: debug
	conan-build-type: Debug
	conan-jemalloc-build-type: Debug
	conan-preset: debug
	- id: release
	conan-build-type: Release
	conan-jemalloc-build-type: Release
	conan-preset: release
	- id: relwithdebinfo-tsan
	conan-build-type: RelWithDebInfo
	conan-jemalloc-build-type: Release
	conan-preset: relwithdebinfo
	conan-custom-settings: \|
	data['compiler']['gcc']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined-behavior']
	data['compiler']['clang']['sanitizer'] = ['None', 'address', 'thread', 'memory', 'undefined-behavior']
	conan-profile-custom-conf: \|
	tools.build:cflags = [ "-fsanitize=thread" ]
	tools.build:cxxflags = [ "-fsanitize=thread" ]
	tools.build:sharedlinkflags = [ "-fsanitize=thread" ]
	tools.build:exelinkflags = [ "-fsanitize=thread" ]
	conan-profile-custom-settings: \|
	compiler.sanitizer = thread
	before-each-test: \|
	export TSAN_OPTIONS="suppressions=$GITHUB_WORKSPACE/tsan_suppressions.cfg second_deadlock_stack=1"
	- id: relwithdebinfo
	conan-build-type: RelWithDebInfo
	conan-jemalloc-build-type: Release
	conan-preset: relwithdebinfo
	- id: minsizerel
	conan-build-type: MinSizeRel
	conan-jemalloc-build-type: Release
	conan-preset: minsizerel

	exclude:
	- compiler: { id: gcc-9 }
	build-type: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-10 }
	build-type: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-11 }
	build-type: { id: relwithdebinfo-tsan }
	- compiler: { id: gcc-13 }
	build-type: { id: relwithdebinfo-tsan }

	runs-on: ubuntu-latest

	name: \|
	build-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}

	steps:
	- name: Update available software list for apt-get
	run: sudo apt-get update

	- name: Checkout `ipc` repository and submodules (`flow`, `ipc_*`)
	uses: actions/checkout@v4
	with:
	submodules: true

	# Many compilers are pre-installed; if not then fetch it.
	- name: Install clang compiler
	if: \|
	matrix.compiler.install && matrix.compiler.name == 'clang'
	run: \|
	wget https://apt.llvm.org/llvm.sh
	chmod u+x llvm.sh
	sudo ./llvm.sh ${{ matrix.compiler.version }}

	- name: Install gcc compiler
	if: \|
	matrix.compiler.install && matrix.compiler.name == 'gcc'
	run: \|
	sudo apt-get install -y software-properties-common
	sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
	sudo apt-get update
	sudo apt-get install -y gcc-${{ matrix.compiler.version }} g++-${{ matrix.compiler.version }}

	- name: Install the latest version of Conan which is less than 2
	run: \|
	pip install "conan<2"

	- name: Add custom settings for Conan packages
	if: \|
	matrix.build-type.conan-custom-settings
	run: \|
	conan config init

	pip install PyYAML

	CONAN_SETTINGS_PATH=$(conan config home)/settings.yml

	python -c "
	import yaml

	with open('$CONAN_SETTINGS_PATH', 'r') as file:
	data = yaml.safe_load(file)

	${{ matrix.build-type.conan-custom-settings }}

	with open('$CONAN_SETTINGS_PATH', 'w') as file:
	yaml.dump(data, file)
	"

	- name: Create Conan profile
	run: \|
	cat <<EOF > conan_profile
	[settings]
	compiler = ${{ matrix.compiler.name }}
	compiler.version = ${{ matrix.compiler.version }}
	compiler.cppstd = 17
	compiler.libcxx = libstdc++11
	arch = x86_64
	os = Linux
	build_type = ${{ matrix.build-type.conan-build-type }}
	jemalloc:build_type = ${{ matrix.build-type.conan-jemalloc-build-type }}
	${{ matrix.build-type.conan-profile-custom-settings }}

	[conf]
	tools.build:compiler_executables = {"c": "${{ matrix.compiler.c-path }}", "cpp": "${{ matrix.compiler.cpp-path }}"}
	tools.env.virtualenv:auto_use = True
	${{ matrix.build-type.conan-profile-custom-conf }}

	[buildenv]
	CC = ${{ matrix.compiler.c-path }}
	CXX = ${{ matrix.compiler.cpp-path }}

	[options]
	jemalloc:enable_cxx = False
	jemalloc:prefix = je_
	ipc:build = True
	ipc:doc = False
	flow:build = True
	flow:doc = False
	EOF

	- name: Install Flow-IPC dependencies with Conan using the profile
	run: \|
	conan editable add flow flow/1.0
	conan install \
	. \
	--profile:build conan_profile \
	--profile:host conan_profile \
	--build missing

	- name: Get the number of processor cores for parallelized work
	uses: SimenB/github-actions-cpu-cores@v1
	id: cpu-cores

	# TODO: Ideally let our CMake interrogate jemalloc-config to find jemalloc-prefix itself.
	# Less stuff for us to worry about that way here. If it has not been tried, try it.
	# If it does not work, apply a small effort to see if it can be easily made to work.
	- name: Prepare Makefile using CMake
	run: \|
	cmake \
	--preset ${{ matrix.build-type.conan-preset }} \
	-DCFG_ENABLE_TEST_SUITE=ON \
	-DJEMALLOC_PREFIX=je_ \
	-DCMAKE_INSTALL_PREFIX=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}

	- name: Build targets (libraries, demos/tests) with Makefile
	run: \|
	VERBOSE=1 make \
	--keep-going \
	--directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
	-j${{ steps.cpu-cores.outputs.count }}

	- name: Install targets with Makefile
	run: \|
	VERBOSE=1 make install \
	--directory $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }} \
	-j${{ steps.cpu-cores.outputs.count }}

	# From now on use always() to try to run any demo/test that exists regardless
	# of preceding failures if any. Same with the log-upload at the end.
	# Worst-case they'll all fail in super-basic ways and immediately; no extra harm done.

	- name: Run link test [`ipc_core` - Flow-IPC Core]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./ipc_core_link_test.exec

	- name: Run link test [`ipc_transport_structured` - Flow-IPC Structured Transport]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./ipc_transport_structured_link_test.exec

	# Server will exit automatically (same below).
	- name: Run link test [`ipc_session` - Flow-IPC Sessions]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./ipc_session_link_test_srv.exec &
	sleep 1
	./ipc_session_link_test_cli.exec

	- name: Run link test [`ipc_shm` - Flow-IPC Shared Memory]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./ipc_shm_link_test_srv.exec &
	sleep 1
	./ipc_shm_link_test_cli.exec

	- name: Run link test [`ipc_shm_arena_lend` - Flow-IPC SHM-jemalloc]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./ipc_shm_arena_lend_link_test_srv.exec &
	sleep 1
	./ipc_shm_arena_lend_link_test_cli.exec

	- name: Run unit tests
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin
	./libipc_unit_test.exec

	# Runner can easily run out of space due to logs, so we tar-up log dir
	# at the end of each step. We also blow away the build dir, as all
	# should be installed in install dir by then.

	# This follows the instructions in bin/transport_test/README.txt.
	- name: Run integration test [transport_test - Scripted mode]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cd $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test
	mkdir -p runs/scripted
	./transport_test.exec scripted runs/scripted/transport_test.srv.log info info \
	< srv-script.txt > runs/scripted/transport_test.srv.console.log 2>&1 &
	SRV_PID=$!
	sleep 1
	./transport_test.exec scripted runs/scripted/transport_test.cli.log info info \
	< cli-script.txt > runs/scripted/transport_test.cli.console.log 2>&1 &
	CLI_PID=$!
	wait $SRV_PID
	echo "Server finished OK (logs will be uploaded as artifacts)."
	wait $CLI_PID
	echo "Client finished OK (logs will be uploaded as artifacts)."
	cd runs
	tar cvzf scripted.tgz scripted
	rm -rf scripted

	# The following [Exercise mode] tests follow the instructions in bin/transport_test/README.txt.
	# Note that the creation of ~/bin/ex_..._run and placement of executables there, plus
	# /tmp/var/run for run-time files (PID files and similar), is a necessary consequence of
	# the ipc::session safety model for estabshing IPC conversations (sessions).

	- name: Prepare IPC-session safety-friendly run-time environment for [transport_test - Exercise mode]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	rm -rf $GITHUB_WORKSPACE/build/${{ matrix.build-type.conan-build-type }}
	mkdir -p ~/bin/ex_srv_run ~/bin/ex_cli_run
	mkdir -p /tmp/var/run
	cp -v $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test/transport_test.exec \
	~/bin/ex_srv.exec
	cp -v ~/bin/ex_srv.exec ~/bin/ex_cli.exec

	- name: Prepare run script for [transport_test - Exercise mode] variations below
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	cat <<'EOF' > $GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh
	# Script created by pipeline during job.
	echo "Log level: [$1]."
	echo "Exercise sub-mode: [$2]."
	echo "Sub-mode snippet (none or 'shm-?'): [$3]."
	OUT_DIR=$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test/runs/exercise/$2
	mkdir -p $OUT_DIR
	cd ~/bin/ex_srv_run
	~/bin/ex_srv.exec exercise-srv$3 $OUT_DIR/transport_test.srv.log info $1 \
	> $OUT_DIR/transport_test.srv.console.log 2>&1 &
	SRV_PID=$!
	sleep 5
	~/bin/ex_cli.exec exercise-cli$3 $OUT_DIR/transport_test.cli.log info $1 \
	> $OUT_DIR/transport_test.cli.console.log 2>&1 &
	CLI_PID=$!
	wait $SRV_PID
	echo "Server finished OK (logs will be uploaded as artifacts)."
	wait $CLI_PID
	echo "Client finished OK (logs will be uploaded as artifacts)."
	cd $OUT_DIR/..
	tar cvzf $2.tgz $2
	rm -rf $2
	EOF

	- name: Run integration test [transport_test - Exercise mode - Heap sub-mode]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	/usr/bin/bash -e \
	$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
	info heap

	- name: Run integration test [transport_test - Exercise mode - SHM-classic sub-mode]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	/usr/bin/bash -e \
	$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
	info shm-c -shm-c

	- name: Run integration test [transport_test - Exercise mode - SHM-jemalloc sub-mode]
	if: \|
	!cancelled() && matrix.build-type.id != 'relwithdebinfo-tsan'
	run: \|
	/usr/bin/bash -e \
	$GITHUB_WORKSPACE/install/${{ matrix.build-type.conan-build-type }}/bin/run_transport_test_ex.sh \
	info shm-j -shm-j

	- name: Upload logs for [transport_test - All modes]
	if: \|
	always() && matrix.build-type.id != 'relwithdebinfo-tsan'
	uses: actions/upload-artifact@v3
	with:
	name: ipc-transport-test-run-${{ matrix.compiler.id }}-${{ matrix.build-type.id }}
	path: \|
	${{ github.workspace }}/install/${{ matrix.build-type.conan-build-type }}/bin/transport_test/runs

	# TODO: Look into the topic of debuggability in case of a crash. Is a core generated? Is it saved?
	# Do we need to manually save it as an artifact? For that matter we would then need the binary and
	# ideally the source. For now we save any logs that are not printed to console, and where possible
	# our demos/tests keep it the console exclusively (but in some cases, such as transport_test, it
	# is not practical due to parallel execution and other aspects). In general it is always better that
	# logs are sufficient; but look into situations where they are not.
	#
	# Possibly this is all handled beautifully automatically; then this should be deleted.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Finish thread sanitizer linker flags #109

Workflow file

Finish thread sanitizer linker flags #109

Jobs

Run details

Workflow file for this run