ref: Upgrade rust-arroyo to 2.16.0 #14983

Workflow file for this run

	name: ci
	on:
	push:
	branches:
	- master
	pull_request:

	jobs:
	files-changed:
	name: detect what files changed
	runs-on: ubuntu-latest
	timeout-minutes: 3
	# Map a step output to a job output
	outputs:
	api_changes: ${{ steps.changes.outputs.api_changes }}
	steps:
	- uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2

	- name: Check for backend file changes
	uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2
	id: changes
	with:
	token: ${{ github.token }}
	filters: .github/file-filters.yml

	linting:
	name: "pre-commit hooks" # (includes Python formatting + linting)
	runs-on: ubuntu-latest
	timeout-minutes: 5
	steps:
	- uses: actions/checkout@v3
	name: Checkout code
	- name: Internal github app token
	id: token
	uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0
	continue-on-error: true
	with:
	app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
	private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
	- uses: actions/setup-python@v4
	with:
	python-version: '3.10'
	- uses: actions/cache@v3
	with:
	path: ~/.cache/pre-commit
	key: cache-epoch-1\|${{ env.pythonLocation }}\|${{ hashFiles('.pre-commit-config.yaml') }}
	- name: Setup pre-commit
	run: make setup-git
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	make install-python-dependencies
	- uses: getsentry/paths-filter@v2
	id: files
	with:
	# Enable listing of files matching each filter.
	# Paths to files will be available in `${FILTER_NAME}_files` output variable.
	# Paths will be escaped and space-delimited.
	# Output is usable as command line argument list in linux shell
	list-files: shell

	# It doesn't make sense to lint deleted files.
	# Therefore we specify we are only interested in added or modified files.
	filters: \|
	all:
	- added\|modified: '*/'
	- name: Run pre-commit checks
	# Run pre-commit to lint and format check files that were changed (but not deleted) compared to master.
	# XXX: there is a very small chance that it'll expand to exceed Linux's limits
	# `getconf ARG_MAX` - max # bytes of args + environ for exec()
	# we skip the `no-commit-to-branch` because in CI we are in fact on master already
	# and we have merged to it
	run: \|
	SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }}

	# If working tree is dirty, commit and update if we have a token
	- name: Apply any pre-commit fixed files
	if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
	uses: getsentry/[email protected]
	with:
	github-token: ${{ steps.token.outputs.token }}

	rust-linting:
	name: "Linting - Rust"
	runs-on: ubuntu-latest
	timeout-minutes: 10
	steps:
	- uses: actions/checkout@v3
	name: Checkout code
	- name: Run linter
	run: \|
	make lint-rust format-rust-ci

	typing:
	name: "mypy typing"
	runs-on: ubuntu-latest
	timeout-minutes: 5
	steps:
	- uses: actions/checkout@v3
	name: Checkout code
	- uses: actions/setup-python@v4
	with:
	python-version: '3.10'
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	make install-python-dependencies
	- name: Run mypy
	run: \|
	make backend-typing

	config-validation:
	name: "Dataset Config Validation"
	runs-on: ubuntu-latest
	timeout-minutes: 2
	steps:
	- uses: actions/checkout@v3
	name: Checkout code
	- uses: actions/setup-python@v4
	with:
	python-version: '3.10'
	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	make install-python-dependencies
	- name: Validate configs
	run: \|
	make validate-configs

	snuba-image:
	name: Build snuba CI image
	runs-on: ubuntu-latest
	timeout-minutes: 20
	outputs:
	branch: ${{ steps.branch.outputs.branch }}
	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Get branch name
	id: branch
	# strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that
	# it can be used as a docker tag
	run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} \| tr / -)" >> "$GITHUB_OUTPUT"

	# We are only using ghcr here for CI as `setup-gcloud` is a bit slow
	# Should revisit this when we move off of google cloud build (we may want to move these to GCR)
	- name: Registry login
	run: \|
	echo ${{ secrets.GITHUB_TOKEN }} \| docker login ghcr.io -u $GITHUB_ACTOR --password-stdin

	- name: Build snuba docker image for CI
	uses: docker/build-push-action@v4
	if: github.repository_owner == 'getsentry'
	with:
	context: .
	# push: true
	load: true
	target: testing
	tags: \|
	ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
	ghcr.io/getsentry/snuba-ci:latest
	cache-from: \|
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
	# cache-to is re-implemented using the next step, to run conditionally for contributors

	- name: Publish images for cache
	if: steps.branch.outputs.branch == 'master' \|\| github.event.pull_request.head.repo.full_name == github.repository
	# outside contributors won't be able to push to the docker registry
	# ignore the failures in this step
	continue-on-error: ${{ github.event_name == 'pull_request' }}
	run: \|
	# Useful to speed up CI
	docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
	docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then
	# The latest tag should only be published on `master`
	docker push ghcr.io/getsentry/snuba-ci:latest
	fi

	tests:
	needs: [linting, snuba-image]
	name: Tests and code coverage
	runs-on: ubuntu-latest
	timeout-minutes: 30
	strategy:
	matrix:
	snuba_settings:
	[
	"test",
	"test_rust",
	"test_distributed",
	"test_distributed_migrations",
	]
	steps:
	- uses: actions/checkout@v2
	name: Checkout code

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Build snuba docker image for CI
	uses: docker/build-push-action@v4
	with:
	context: .
	push: false
	tags: snuba-test
	load: true
	target: testing
	cache-from: \|
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:latest

	- name: Docker set up
	run: \|
	docker network create --attachable cloudbuild

	- name: Docker Snuba Rust tests
	run: \|
	SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test-rust
	if: ${{ matrix.snuba_settings == 'test_rust' }}

	- name: Docker Snuba tests
	run: \|
	SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test
	if: ${{ matrix.snuba_settings == 'test' \|\| matrix.snuba_settings == 'test_distributed' }}

	- name: Docker Snuba Multi-Node Tests
	run: \|
	SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d
	SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
	if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }}

	- name: Docker Snuba Init Tests
	run: \|
	SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test
	if: ${{ matrix.snuba_settings == 'test' }}

	- name: Upload to codecov
	run: \|
	curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

	admin-tests:
	needs: [linting]
	name: Front end tests for snuba admin
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2
	name: Checkout code
	- uses: volta-cli/action@v4
	- name: Set up and run tests through yarn
	run: cd snuba/admin && yarn install && yarn run test --coverage
	- name: Upload to codecov
	run: \|
	curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}

	sentry:
	needs: [snuba-image, files-changed]
	runs-on: ubuntu-latest
	timeout-minutes: 60
	strategy:
	matrix:
	instance: [0, 1, 2, 3]
	env:
	MIGRATIONS_TEST_MIGRATE: 1
	# XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
	MATRIX_INSTANCE_TOTAL: 4
	TEST_GROUP_STRATEGY: ROUND_ROBIN

	steps:
	# Checkout codebase
	- name: Checkout snuba
	uses: actions/checkout@v2

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Build snuba docker image for CI
	uses: docker/build-push-action@v4
	with:
	context: .
	push: false
	tags: snuba-test
	load: true
	target: testing
	cache-from: \|
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:latest

	# Checkout Sentry and run integration tests against latest snuba
	# Make sure this is after `docker build`, otherwise we'll break docker cache
	- name: Checkout sentry
	uses: actions/checkout@v2
	with:
	repository: getsentry/sentry
	path: sentry

	- name: Setup steps
	id: setup
	run: \|
	pip install --upgrade pip wheel
	# We cannot execute actions that are not placed under .github of the main repo
	mkdir -p .github/actions
	cp -r sentry/.github/actions/* .github/actions

	- name: Setup Sentry
	id: setup-sentry
	uses: ./.github/actions/setup-sentry
	with:
	workdir: sentry
	snuba: false
	kafka: true
	clickhouse: true

	- name: Start snuba
	run: \|
	docker run -d --rm \
	-p 127.0.0.1:1218:1218 \
	-e PYTHONUNBUFFERED=1 \
	-e SNUBA_SETTINGS=docker \
	-e DEBUG=1 \
	-e DEFAULT_BROKERS=sentry_kafka:9092 \
	-e CLICKHOUSE_HOST=sentry_clickhouse \
	-e CLICKHOUSE_PORT=9000 \
	-e CLICKHOUSE_HTTP_PORT=8123 \
	-e REDIS_HOST=sentry_redis \
	-e REDIS_PORT=6379 \
	-e REDIS_DB=1 \
	--name sentry_snuba \
	--network sentry \
	snuba-test
	docker exec sentry_snuba snuba migrations migrate --force

	- name: Run snuba tests
	if: needs.files-changed.outputs.api_changes == 'false'
	working-directory: sentry
	run: \|
	pytest -k 'not __In' tests \
	-m snuba_ci \
	-vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"

	- name: Run full tests
	# TODO: Adjust which changed files we care about
	if: needs.files-changed.outputs.api_changes == 'true'
	working-directory: sentry
	run: \|
	pytest -k 'not __In' tests/snuba \
	tests/sentry/eventstream/kafka \
	tests/sentry/post_process_forwarder \
	tests/sentry/snuba \
	tests/sentry/search/events \
	tests/sentry/event_manager \
	tests/sentry/api/endpoints/test_organization_ddm_meta.py \
	tests/sentry/api/endpoints/test_organization_profiling_functions.py \
	-vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"

	- name: Run CI module tests
	if: needs.files-changed.outputs.api_changes == 'true'
	working-directory: sentry
	run: pytest -k 'not __In' tests -vv -m snuba_ci

	clickhouse-versions:
	needs: [linting, snuba-image]
	name: Tests on multiple clickhouse versions
	runs-on: ubuntu-latest
	timeout-minutes: 30
	strategy:
	fail-fast: false
	matrix:
	version:
	[
	"21.8.13.1.altinitystable",
	"22.8.15.25.altinitystable",
	"23.3.13.7.altinitystable",
	"23.8.8.21.altinitystable",
	]

	steps:
	- uses: actions/checkout@v2
	name: Checkout code

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v2

	- name: Build snuba docker image for CI
	uses: docker/build-push-action@v4
	with:
	context: .
	push: false
	tags: snuba-test
	load: true
	target: testing
	cache-from: \|
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
	type=registry,ref=ghcr.io/getsentry/snuba-ci:latest

	- name: Docker set up
	run: \|
	docker network create --attachable cloudbuild

	- name: Docker Snuba Test other ClickHouse versions
	run: \|
	export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}}
	SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test

	- name: Upload to codecov
	run: \|
	curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
	publish-to-dockerhub:
	name: Publish Snuba to DockerHub
	runs-on: ubuntu-20.04
	if: ${{ (github.ref_name == 'master') }}
	steps:
	- uses: actions/checkout@v3 # v3.1.0
	- name: Pull the test image
	id: image_pull
	env:
	IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
	shell: bash
	run: \|
	echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out."
	echo "Polling for $IMAGE_URL"
	timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done'
	- name: Get short SHA for docker tag
	id: short_sha
	shell: bash
	run: \|
	SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
	if [[ -z "$SHORT_SHA" ]]; then
	echo "Short SHA empty? Re-running rev-parse."
	git rev-parse --short "$GITHUB_SHA"
	else
	echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT
	fi
	- name: Push built docker image
	shell: bash
	env:
	SHORT_SHA: ${{ steps.short_sha.outputs.sha }}
	IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
	run: \|
	# only login if the password is set
	if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" \| docker login --username=sentrybuilder --password-stdin; fi
	# We push 3 tags to Dockerhub:
	# first, the full sha of the commit
	docker tag ${IMAGE_URL} getsentry/snuba:${GITHUB_SHA}
	docker push getsentry/snuba:${GITHUB_SHA}
	# second, the short sha of the commit
	docker tag ${IMAGE_URL} getsentry/snuba:${SHORT_SHA}
	docker push getsentry/snuba:${SHORT_SHA}
	# finally, nightly
	docker tag ${IMAGE_URL} getsentry/snuba:nightly
	docker push getsentry/snuba:nightly

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

ref: Upgrade rust-arroyo to 2.16.0 #14983

Workflow file

ref: Upgrade rust-arroyo to 2.16.0 #14983

Jobs

Run details

Workflow file for this run