From 87b3adfd8f8d3c84a23e9ddbf7fd9eba1217ee5a Mon Sep 17 00:00:00 2001 From: Pat Somaru Date: Thu, 26 Sep 2024 09:16:46 -0400 Subject: [PATCH] Setup "debugging" and misc cleanup * Fix a couple of misc errors in build scripts. * Tweak scripts/kconfigs to make bpftrace work. * Update how CI caching works to make builds faster (6 minute turnaround time) * Update CI config to generate per-scheduler debug archives w/ guest dmesg/scheduler stdout, guest stdout, bpftrace script output, veristat output. * Update build scripts to accept the following: ** VNG RW -- write to host filesystem (better caching, logging). * For stress tests in particular (via ini config): ** QEMU Opts -- to facilitate reproducing bugs (i.e. high core count). ** bpftrace scripts -- specify bpftrace scripts to run during stress tests. --- .../actions/install-deps-action/action.yml | 30 +++-- .github/workflows/caching-build.yml | 107 +++++++++++------- .github/workflows/sched-ext.config | 20 ++++ meson-scripts/run_stress_tests | 27 ++++- meson-scripts/test_sched | 13 ++- meson-scripts/veristat | 20 ++-- meson.build | 42 ++++--- meson.options | 6 + scripts/bpftrace_stress_wrapper.sh | 27 +++-- 9 files changed, 209 insertions(+), 83 deletions(-) diff --git a/.github/actions/install-deps-action/action.yml b/.github/actions/install-deps-action/action.yml index 4f768cf4b..0afb2db98 100644 --- a/.github/actions/install-deps-action/action.yml +++ b/.github/actions/install-deps-action/action.yml @@ -4,23 +4,35 @@ runs: using: 'composite' steps: ### OTHER REPOS #### - - # Hard turn-off interactive mode - - run: echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - shell: bash - - # Refresh packages list - - run: sudo apt update + # turn off interactive, refresh pkgs, use apt fast + - run: | + echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections + sudo rm /var/lib/man-db/auto-update + echo "deb [signed-by=/etc/apt/keyrings/apt-fast.gpg] http://ppa.launchpad.net/apt-fast/stable/ubuntu $(source /etc/os-release && echo $UBUNTU_CODENAME) main" | sudo tee /etc/apt/sources.list.d/apt-fast.list + wget -q -O- "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0xBC5934FD3DEBD4DAEA544F791E2824A7F22B44BD" | sudo gpg --dearmor -o /etc/apt/keyrings/apt-fast.gpg + sudo apt-get update -y + sudo apt-get install -y apt-fast aria2 tasksel + echo 'debconf apt-fast/maxdownloads string 100' | sudo debconf-set-selections + echo 'debconf apt-fast/dlflag boolean true' | sudo debconf-set-selections + echo 'debconf apt-fast/aptmanager string apt-get' | sudo debconf-set-selections + sudo tasksel remove ubuntu-desktop shell: bash ### DOWNLOAD AND INSTALL DEPENDENCIES ### # Download dependencies packaged by Ubuntu - - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd libseccomp-dev libcap-ng-dev llvm clang python3-full pipx curl meson + - run: | + sudo apt-fast install -f -y bison busybox-static cmake coreutils \ + cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod \ + libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev \ + linux-headers-generic linux-tools-common linux-tools-generic make \ + ninja-build pahole pkg-config python3-dev python3-pip python3-requests \ + qemu-kvm rsync stress-ng udev zstd libseccomp-dev libcap-ng-dev \ + llvm clang python3-full curl meson bpftrace cargo rustc dwarves shell: bash # virtme-ng - - run: pip3 install virtme-ng --break-system-packages + - run: sudo pip3 install virtme-ng --break-system-packages shell: bash # Setup KVM support diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml index bd7588994..33fb58fc2 100644 --- a/.github/workflows/caching-build.yml +++ b/.github/workflows/caching-build.yml @@ -6,7 +6,7 @@ on: - cron: "0 * * * *" push: pull_request: - + jobs: lint: runs-on: ubuntu-24.04 @@ -32,8 +32,8 @@ jobs: - run: sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar # redundancy to exit fast - run: echo 'debconf debconf/frontend select Noninteractive' | sudo debconf-set-selections - - run: sudo apt update - - run: sudo apt install -y git --no-install-recommends + - run: sudo apt-get update + - run: sudo apt-get install -y git --no-install-recommends # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV @@ -45,8 +45,10 @@ jobs: uses: actions/cache@v4 with: path: | - linux - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} + linux/arch/x86/boot/bzImage + linux/usr/include + linux/**/*.h + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-4 - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-deps-action @@ -62,14 +64,6 @@ jobs: - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' && steps.cache-kernel.outputs.cache-hit != 'true' }} run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ - # cache bzImage alone for rust tests (disk space limit workaround) - - name: Cache bzImage - id: cache-bzImage - uses: actions/cache@v4 - with: - path: | - linux/arch/x86/boot/bzImage - key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} name: Clone Kernel @@ -96,14 +90,18 @@ jobs: integration-test: runs-on: ubuntu-24.04 needs: build-kernel - continue-on-error: true strategy: matrix: scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ] + fail-fast: false steps: # prevent cache permission errors - run: sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.scheduler }} + prefix-key: "4" - uses: ./.github/actions/install-deps-action # cache virtiofsd (goes away w/ 24.04) - name: Cache virtiofsd @@ -125,8 +123,10 @@ jobs: uses: actions/cache@v4 with: path: | - linux - key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} + linux/arch/x86/boot/bzImage + linux/usr/include + linux/**/*.h + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-4 # need to re-run job when kernel head changes between build and test running. - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} @@ -139,7 +139,7 @@ jobs: - run: sudo chmod +x /usr/bin/veristat && sudo chmod 755 /usr/bin/veristat # The actual build: - - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true + - run: meson setup build -Dkernel=../linux/arch/x86/boot/bzImage -Dkernel_headers=../linux -Denable_stress=true -Dvng_rw_mount=true - run: meson compile -C build ${{ matrix.scheduler }} # Print CPU model before running the tests (this can be useful for @@ -148,13 +148,35 @@ jobs: # Test schedulers - run: meson compile -C build test_sched_${{ matrix.scheduler }} + # this is where errors we want logs on start occurring, so always generate debug info and save logs + if: always() # Stress schedulers - uses: cytopia/shell-command-retry-action@v0.1.2 name: stress test + if: always() with: retries: 3 command: meson compile -C build stress_tests_${{ matrix.scheduler }} - run: meson compile -C build veristat_${{ matrix.scheduler }} + if: always() + - run: sudo cat /var/log/dmesg > host-dmesg.ci.log + if: always() + - run: echo "NICE_REF=${{ github.event.pull_request && github.head_ref || github.ref_name }}" >> $GITHUB_ENV + if: always() + - run: mkdir -p ./log_save/ + if: always() + # no symlink following here (to avoid cycles) + - run: sudo find '/home/runner/' -iname '*.ci.log' -exec mv {} ./log_save/ \; + if: always() + - name: upload debug logs, bpftrace, veristat, dmesg, etc. + if: always() + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.scheduler }}_logs_${{ env.NICE_REF }}_${{ github.run_id }}_${{ github.run_attempt }} + path: ./log_save/*.ci.log + # it's all txt files w/ 90 day retention, lets be nice. + compression-level: 9 + rust-test-core: runs-on: ubuntu-24.04 @@ -166,6 +188,10 @@ jobs: # prevent cache permission errors - run: sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.component }} + prefix-key: "4" - uses: ./.github/actions/install-deps-action # cache virtiofsd (goes away w/ 24.04) - name: Cache virtiofsd @@ -180,28 +206,24 @@ jobs: # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - # cache bzImage alone for rust tests - - name: Cache bzImage - id: cache-bzImage + + - name: Cache Kernel + id: cache-kernel uses: actions/cache@v4 with: path: | linux/arch/x86/boot/bzImage - key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} + linux/usr/include + linux/**/*.h + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-4 # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} + - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} name: exit if cache stale run: exit -1 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - key: ${{ matrix.component }} - prefix-key: "1" - run: cargo build --manifest-path rust/${{ matrix.component }}/Cargo.toml - - run: cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml --no-run - - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml + - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml rust-test-schedulers: runs-on: ubuntu-24.04 @@ -213,6 +235,10 @@ jobs: # prevent cache permission errors - run: sudo chown root /usr/bin/tar && sudo chmod u+s /usr/bin/tar - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + key: ${{ matrix.scheduler }} + prefix-key: "4" - uses: ./.github/actions/install-deps-action # cache virtiofsd (goes away w/ 24.04) - name: Cache virtiofsd @@ -227,28 +253,24 @@ jobs: # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV - # cache bzImage alone for rust tests - - name: Cache bzImage - id: cache-bzImage + # Cache Kernel alone for rust tests + - name: Cache Kernel + id: cache-kernel uses: actions/cache@v4 with: path: | linux/arch/x86/boot/bzImage - key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} + linux/usr/include + linux/**/*.h + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}-4 # need to re-run job when kernel head changes between build and test running. - - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} + - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} name: exit if cache stale run: exit -1 - - uses: Swatinem/rust-cache@v2 - with: - workspaces: scheds/rust - key: ${{ matrix.scheduler }} - prefix-key: "1" - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml - - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run - - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml + - run: vng -v --rw --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml pages: runs-on: ubuntu-24.04 @@ -270,8 +292,7 @@ jobs: rustup install nightly export PATH="~/.cargo/bin:$PATH" RUSTDOCFLAGS="--enable-index-page -Zunstable-options" ~/.cargo/bin/cargo +nightly doc --workspace --no-deps --bins --lib --examples --document-private-items --all-features - sudo apt update - sudo apt install build-essential graphviz sphinx-doc python3-sphinx-rtd-theme texlive-latex-recommended python3-yaml -y + sudo apt-fast install build-essential graphviz sphinx-doc python3-sphinx-rtd-theme texlive-latex-recommended python3-yaml -y cargo install htmlq git clone --single-branch -b for-next --depth 1 https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git linux cd linux diff --git a/.github/workflows/sched-ext.config b/.github/workflows/sched-ext.config index efb7bda8b..e4ec1598a 100644 --- a/.github/workflows/sched-ext.config +++ b/.github/workflows/sched-ext.config @@ -32,3 +32,23 @@ CONFIG_PREEMPT_RCU=y # CONFIG_DEBUG_LOCKDEP=y CONFIG_DEBUG_ATOMIC_SLEEP=y + +# Bpftrace headers (for additional debug info) +CONFIG_BPF=y +CONFIG_BPF_SYSCALL=y +CONFIG_BPF_JIT=y +CONFIG_HAVE_EBPF_JIT=y +CONFIG_BPF_EVENTS=y +CONFIG_FTRACE_SYSCALLS=y +CONFIG_FUNCTION_TRACER=y +CONFIG_HAVE_DYNAMIC_FTRACE=y +CONFIG_DYNAMIC_FTRACE=y +CONFIG_HAVE_KPROBES=y +CONFIG_KPROBES=y +CONFIG_KPROBE_EVENTS=y +CONFIG_ARCH_SUPPORTS_UPROBES=y +CONFIG_UPROBES=y +CONFIG_UPROBE_EVENTS=y +CONFIG_DEBUG_FS=y +# more bpftrace to make that work +CONFIG_IKHEADERS=y \ No newline at end of file diff --git a/meson-scripts/run_stress_tests b/meson-scripts/run_stress_tests index 0d5a9605d..d597e8e2c 100755 --- a/meson-scripts/run_stress_tests +++ b/meson-scripts/run_stress_tests @@ -40,6 +40,8 @@ def run_stress_test( vng_path: str, kernel: str, verbose: bool, + rw: bool, + headers: str, ) -> int: scheduler_args = config.get('scheduler_args') stress_cmd = config.get('stress_cmd') @@ -47,14 +49,27 @@ def run_stress_test( sched_cmd = s_path + " " + config.get('sched_args') timeout_sec = int(config.get("timeout_sec")) if vng_path: + cmd = [vng_path, "--user", "root", "-v", "-r", kernel] + if config.get("qemu_opts"): + cmd += ['--qemu-opts'] + cmd += [f"'{config.get("qemu_opts")}'"] vm_input = f"{stress_cmd} & timeout --foreground --preserve-status {timeout_sec} {sched_cmd}" if bpftrace_scripts := config.get('bpftrace_scripts'): vm_input = f"\"{build_dir}/bpftrace_stress_wrapper.sh\" '{stress_cmd}' '{sched_cmd}' '{timeout_sec}' '{bpftrace_scripts}'" - cmd = [vng_path, "--user", "root", "-v", "--", vm_input] + if headers: + vm_input += f" '{headers}'" + if rw and os.getenv('CI'): + print('mounting VNG as RW because CI') + cmd += ["--rw"] + elif rw: + print('not mounting VNG as RW because not CI') + cmd += ["--"] + cmd += [vm_input] err = sys.stderr if output == "-" else open(output, "w") out = sys.stdout if output == "-" else err + print(f"vng cmd is {cmd}") proc = subprocess.Popen( - cmd, env=os.environ, cwd=kernel, shell=False, stdout=out, + cmd, env=os.environ, shell=False, stdout=out, stderr=err, stdin=subprocess.PIPE, text=True) proc.wait() return proc.returncode @@ -85,6 +100,8 @@ def stress_tests(args: Namespace) -> None: vng_path, args.kernel, args.verbose, + args.rw, + args.headers ) for test_name, ret in return_codes.items(): if ret not in (143, 0): @@ -114,6 +131,12 @@ if __name__ == "__main__": parser.add_argument( '--sched', default='', help='Scheduler to test (default: all)' ) + parser.add_argument( + '--rw', default=False, help='Mount VNG Directories as RW (dangerous)' + ) + parser.add_argument( + '--headers', default='', help='Kernel Headers Path' + ) args = parser.parse_args() if args.verbose: diff --git a/meson-scripts/test_sched b/meson-scripts/test_sched index a1884511b..fd4281376 100755 --- a/meson-scripts/test_sched +++ b/meson-scripts/test_sched @@ -17,6 +17,16 @@ GUEST_TIMEOUT=60 # declare -A SCHEDS +VNG_RW='' + +# Enable vng rw for when on ci. +if [ $# -ge 3 ] ; then + if [ "$3" == "VNG_RW=true" ]; then + echo 'setting vng to mount rw' + VNG_RW=' --rw ' + fi +fi + # enable running tests on individual schedulers if [ $# -ge 2 ] ; then SCHEDS[$2]="" @@ -63,7 +73,7 @@ for sched in ${!SCHEDS[@]}; do rm -f /tmp/output timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G -v -r ${kernel} -- \ + vng --user root -m 10G --cpu 8 $VNG_RW -v -r ${kernel} -- \ "timeout --foreground --preserve-status ${TEST_TIMEOUT} ${sched_path} ${args}" \ 2> >(tee /tmp/output) &1 | tee veristat.ci.log exit $? else - sudo veristat ${BPF_PATH} + sudo veristat ${BPF_PATH} 2>&1 | tee veristat.ci.log exit $? fi fi @@ -32,10 +38,10 @@ fi for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do if [ -n "${KERNEL}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G -v --user root -r ${KERNEL} -- \ - sudo veristat ${BPF_PATH} + vng --user root -m 10G --cpu 8 $VNG_RW -v --user root -r ${KERNEL} -- \ + sudo veristat ${BPF_PATH} 2>&1 | tee "$(basename "${BPF_PATH}")-veristat.ci.log" else echo "$BPF_PATH" - sudo veristat ${BPF_PATH} + sudo veristat ${BPF_PATH} 2>&1 | tee "$(basename "${BPF_PATH}")-veristat.ci.log" fi done diff --git a/meson.build b/meson.build index 61877b618..6b872245c 100644 --- a/meson.build +++ b/meson.build @@ -366,17 +366,22 @@ else endif run_target('test_sched', command: [test_sched, kernel]) - -foreach s : rust_scheds - run_target('test_sched_'+s, command: [test_sched, kernel, s]) -endforeach - run_target('veristat', command: [run_veristat, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel')]) -foreach s : rust_scheds - run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(), - get_option('veristat_scheduler'), get_option('kernel'), s]) -endforeach + +if get_option('vng_rw_mount') == true + foreach s : rust_scheds + run_target('test_sched_'+s, command: [test_sched, kernel, s, 'VNG_RW=true']) + run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(), + get_option('veristat_scheduler'), get_option('kernel'), s, 'VNG_RW=true']) + endforeach +else + foreach s : rust_scheds + run_target('test_sched_'+s, command: [test_sched, kernel, s]) + run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(), + get_option('veristat_scheduler'), get_option('kernel'), s]) + endforeach +endif run_target('veristat_diff', command: [run_veristat_diff, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel'), @@ -405,13 +410,22 @@ if enable_stress install : false, build_by_default : true) ] - run_target('stress_tests', command: [run_stress_tests, '-k', kernel, '-b', meson.current_build_dir()], depends: [copys]) - foreach s : rust_scheds - run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b', - meson.current_build_dir(), '--sched', s], depends: [copys]) - endforeach + + if get_option('vng_rw_mount') == true + foreach s : rust_scheds + run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b', + meson.current_build_dir(), '--sched', s, + '--rw', 'true', '--headers', get_option('kernel_headers')], + depends: [copys]) + endforeach + else + foreach s : rust_scheds + run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b', + meson.current_build_dir(), '--sched', s], depends: [copys]) + endforeach + endif endif subdir('scheds') diff --git a/meson.options b/meson.options index cf0c08710..7820ea08e 100644 --- a/meson.options +++ b/meson.options @@ -47,4 +47,10 @@ option( type: 'boolean', value: false, description: 'enable build cmds to work outside of src dir', +) +option( + 'vng_rw_mount', + type: 'boolean', + value: false, + description: 'make vng builds mount rw (for log upload)', ) \ No newline at end of file diff --git a/scripts/bpftrace_stress_wrapper.sh b/scripts/bpftrace_stress_wrapper.sh index 7b5bd24b2..7bfae6c88 100755 --- a/scripts/bpftrace_stress_wrapper.sh +++ b/scripts/bpftrace_stress_wrapper.sh @@ -1,36 +1,49 @@ #!/bin/bash +set +x + STRESS_CMD="$1" SCHED_CMD="$2" TIMEOUT_SEC="$3" BPFTRACE_SCRIPTS="$4" +KERNEL_HEADERS="$5" + +echo $PWD SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" for a in $(echo "$BPFTRACE_SCRIPTS" | tr ',' ' '); do - bpftrace "$SCRIPT_DIR/$a" > "$(basename a).txt" 2>&1 & + BPFTRACE_KERNEL_SOURCE=${KERNEL_HEADERS} bpftrace -o "$a.ci.log" $SCRIPT_DIR/$a & done -$STRESS_CMD > stress_cmd.txt 2>&1 & +$STRESS_CMD > stress_cmd.ci.log 2>&1 & STRESS_PID=$! -timeout --foreground --preserve-status $TIMEOUT_SEC $SCHED_CMD > sched_output.txt 2>&1 & +timeout --foreground --preserve-status $TIMEOUT_SEC $SCHED_CMD > sched_output.ci.log 2>&1 & STATUS_PID=$! -tail --pid=$STATUS_PID -f sched_output.txt +# wait for scheduler to exit +tail --pid=$STATUS_PID -f sched_output.ci.log + +# wait for stress to exit, ignore err +wait $STRESS_PID || true + +sleep 10 + +killall -w 10s $(jobs -p) for a in $(echo "$BPFTRACE_SCRIPTS" | tr ',' ' '); do echo "$a OUTPUT" - cat "$(basename a).txt" + cat "$a.ci.log" echo "$a OUTPUT DONE" done echo "STRESS OUTPUT" -cat stress_cmd.txt +cat stress_cmd.ci.log echo "STRESS OUTPUT DONE" # if anything isn't right, exit non 0 so we know. wait $STRESS_PID -wait $STATUS_PID +wait $STATUS_PID \ No newline at end of file