benchmarking.yml

name: Benchmarking
on:
  schedule:
    - cron: '0 0 * * *' # every night at midnight UTC
  push:
    branches:
      - main

  pull_request:
    paths:
      # test changes to Sentry SDK sources
      - 'Sources/**'

      # test changes to benchmarking implementation
      - 'Samples/iOS-Swift/iOS-Swift/**'
      - 'Samples/iOS-Swift/PerformanceBenchmarks/**'
      - '.github/workflows/benchmarking.yml'
      - '.sauce/benchmarking-config.yml'
      - 'fastlane/**'
      - 'scripts/ci-select-xcode.sh'
      - 'scripts/set-device-tests-environment.patch'

jobs:
  build-benchmark-test-target:
    name: Build app and test runner
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh
      - uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - name: Install SentryCli
        run: brew install getsentry/tools/sentry-cli
      - run: git apply ./scripts/set-device-tests-environment.patch
      - name: Cache iOS-Swift App and dSYM build products
        id: ios-swift-cache
        uses: actions/cache@v3
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
            DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/iOS-Swift/**') }}-${{ hashFiles('Sources/Sentry/**') }}
      - name: Cache iOS-Swift UI Test Runner App build product
        id: ios-swift-benchmark-runner-cache
        uses: actions/cache@v3
        with:
          path: |
            DerivedData/Build/Products/Debug-iphoneos/PerformanceBenchmarks-Runner.app
          key: ios-swift-for-ui-testing-cache-key-${{ hashFiles('Samples/iOS-Swift/PerformanceBenchmarks/**') }}
      - run: bundle exec fastlane build_ios_swift_for_tests
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - run: bundle exec fastlane build_ios_benchmark_test
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Upload dSYMs
        run: |
          sentry-cli --auth-token ${{ secrets.SENTRY_AUTH_TOKEN }} upload-dif --org sentry-sdks --project sentry-cocoa DerivedData/Build/Products/Debug-iphoneos/iOS-Swift.app.dSYM
      - name: Archiving DerivedData
        uses: actions/upload-artifact@v3
        with:
          name: DerivedData-Xcode
          path: |
            **/Debug-iphoneos/iOS-Swift.app
            **/Debug-iphoneos/PerformanceBenchmarks-Runner.app

  run-ui-tests-with-sauce:
    name: Run benchmarks on Sauce Labs
    runs-on: ubuntu-latest
    needs: build-benchmark-test-target
    strategy:
      fail-fast: false
      matrix:
        suite: ['High-end device', 'Mid-range device', 'Low-end device']
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: DerivedData-Xcode
      - run: npm install -g saucectl@0.107.2
      - name: Run Benchmarks in SauceLab
        env:
          SAUCE_USERNAME: ${{ secrets.SAUCE_USERNAME }}
          SAUCE_ACCESS_KEY: ${{ secrets.SAUCE_ACCESS_KEY }}
        run: saucectl run --select-suite "${{matrix.suite}}" --config .sauce/benchmarking-config.yml --tags benchmark

  app-metrics:
    name: Collect app metrics
    runs-on: macos-13
    steps:
      - name: Git checkout
        uses: actions/checkout@v4
      - run: ./scripts/ci-select-xcode.sh
      - uses: ruby/setup-ruby@v1
        with:
          bundler-cache: true
      - uses: actions/cache@v3
        id: app-plain-cache
        with:
          path: Tests/Perf/test-app-plain.ipa
          key: ${{ github.workflow }}-${{ github.job }}-appplain-${{ hashFiles('fastlane/Fastfile', 'Tests/Perf/test-app-plain/**') }}
      - name: Build test app plain
        if: steps.app-plain-cache.outputs['cache-hit'] != 'true'
        run: bundle exec fastlane build_perf_test_app_plain
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Build test app with sentry
        run: bundle exec fastlane build_perf_test_app_sentry
        env:
          APP_STORE_CONNECT_KEY_ID: ${{ secrets.APP_STORE_CONNECT_KEY_ID }}
          APP_STORE_CONNECT_ISSUER_ID: ${{ secrets.APP_STORE_CONNECT_ISSUER_ID }}
          APP_STORE_CONNECT_KEY: ${{ secrets.APP_STORE_CONNECT_KEY }}
          FASTLANE_KEYCHAIN_PASSWORD: ${{ secrets.FASTLANE_KEYCHAIN_PASSWORD }}
          MATCH_GIT_PRIVATE_KEY: ${{ secrets.MATCH_GIT_PRIVATE_KEY }}
          MATCH_PASSWORD: ${{ secrets.MATCH_PASSWORD }}
          MATCH_USERNAME: ${{ secrets.MATCH_USERNAME }}
      - name: Collect app metrics
        uses: getsentry/action-app-sdk-overhead-metrics@v1
        with:
          config: Tests/Perf/metrics-test.yml
          sauce-user: ${{ secrets.SAUCE_USERNAME }}
          sauce-key: ${{ secrets.SAUCE_ACCESS_KEY }}