From 120211d731e347091f0b80acdd357441761e5465 Mon Sep 17 00:00:00 2001 From: patso Date: Sun, 8 Sep 2024 02:54:24 -0400 Subject: [PATCH] split build and test jobs split build and test jobs to reduce ci turnaround time and make it clear what is failing when something fails. also add virtiofsd to deps to make test compilation faster (most test time is compliation) and remove all force 9ps. --- .../actions/install-deps-action/action.yml | 2 +- .github/workflows/caching-build.yml | 117 +++++++++++++++--- meson-scripts/run_stress_tests | 8 +- meson-scripts/test_sched | 30 +++-- meson-scripts/veristat | 4 +- meson-scripts/veristat_diff | 8 +- meson.build | 17 +++ scheds/rust/meson.build | 7 +- 8 files changed, 156 insertions(+), 37 deletions(-) diff --git a/.github/actions/install-deps-action/action.yml b/.github/actions/install-deps-action/action.yml index 47612eb1..d83c2749 100644 --- a/.github/actions/install-deps-action/action.yml +++ b/.github/actions/install-deps-action/action.yml @@ -16,7 +16,7 @@ runs: ### DOWNLOAD AND INSTALL DEPENDENCIES ### # Download dependencies packaged by Ubuntu - - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd + - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd libseccomp-dev libcap-ng-dev shell: bash # clang 17 diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml index d93bab24..f8787c71 100644 --- a/.github/workflows/caching-build.yml +++ b/.github/workflows/caching-build.yml @@ -25,11 +25,22 @@ jobs: with: path: | linux - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-deps-action + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' && steps.cache-kernel.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # cache bzImage alone for rust tests (disk space limit workaround) - name: Cache bzImage id: cache-bzImage @@ -37,7 +48,7 @@ jobs: with: path: | linux/arch/x86/boot/bzImage - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} name: Clone Kernel @@ -64,10 +75,23 @@ jobs: integration-test: runs-on: ubuntu-22.04 needs: build-kernel + strategy: + matrix: + scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/install-deps-action - # cache rust deps, invalidate when kernel commit changes + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV @@ -78,7 +102,7 @@ jobs: with: path: | linux - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} # need to re-run job when kernel head changes between build and test running. - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} @@ -92,25 +116,42 @@ jobs: # The actual build: - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true - - run: meson compile -C build + - run: meson compile -C build ${{ matrix.scheduler }} # Print CPU model before running the tests (this can be useful for # debugging purposes) - run: grep 'model name' /proc/cpuinfo | head -1 # Test schedulers - - run: meson compile -C build test_sched + - run: meson compile -C build test_sched_${{ matrix.scheduler }} # Stress schedulers - - run: meson compile -C build stress_tests - - run: meson compile -C build veristat + - uses: cytopia/shell-command-retry-action@v0.1.2 + name: stress test + with: + retries: 3 + command: meson compile -C build stress_tests_${{ matrix.scheduler }} + - run: meson compile -C build veristat_${{ matrix.scheduler }} - rust-test: + rust-test-core: runs-on: ubuntu-22.04 needs: build-kernel + strategy: + matrix: + component: [scx_loader, scx_rustland_core, scx_stats, scx_utils] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/install-deps-action - # cache rust deps, invalidate when kernel commit changes + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV # cache bzImage alone for rust tests @@ -120,7 +161,7 @@ jobs: with: path: | linux/arch/x86/boot/bzImage - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} # need to re-run job when kernel head changes between build and test running. - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} @@ -129,9 +170,55 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - shared-key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} workspaces: rust + key: ${{ matrix.component }} + prefix-key: "1" + - run: cargo build --manifest-path rust/${{ matrix.component }}/Cargo.toml + - run: cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml --no-run + - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml + + rust-test-schedulers: + runs-on: ubuntu-22.04 + needs: build-kernel + strategy: + matrix: + scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps-action + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + + # get latest head commit of sched_ext for-next + - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV + # cache bzImage alone for rust tests + - name: Cache bzImage + id: cache-bzImage + uses: actions/cache@v4 + with: + path: | + linux/arch/x86/boot/bzImage + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} + + # need to re-run job when kernel head changes between build and test running. + - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} + name: exit if cache stale + run: exit -1 + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: scheds/rust + key: ${{ matrix.scheduler }} prefix-key: "1" - - run: cargo build --manifest-path rust/Cargo.toml - - run: cargo test --manifest-path rust/Cargo.toml --no-run - - run: vng -v --memory 10G --cpu 8 --force-9p -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/Cargo.toml + - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml + - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run + - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml + diff --git a/meson-scripts/run_stress_tests b/meson-scripts/run_stress_tests index 2b0bcf18..8d58ec65 100755 --- a/meson-scripts/run_stress_tests +++ b/meson-scripts/run_stress_tests @@ -48,7 +48,7 @@ def run_stress_test( timeout_sec = int(config.get("timeout_sec")) if vng_path: vm_input = f"{stress_cmd} & timeout --foreground --preserve-status {timeout_sec} {sched_cmd}" - cmd = [vng_path, "--user", "root", "--force-9p", "-v", "--", vm_input] + cmd = [vng_path, "--user", "root", "-v", "--", vm_input] err = sys.stderr if output == "-" else open(output, "w") out = sys.stdout if output == "-" else err proc = subprocess.Popen( @@ -73,6 +73,9 @@ def stress_tests(args: Namespace) -> None: for test_name in configs.sections(): if args.stress_test and test_name != args.stress_test: continue + if args.sched and configs[test_name].get('sched') != args.sched: + continue + print(f"Running stress test: {test_name}") return_codes[test_name] = run_stress_test( configs[test_name], args.build_dir, @@ -106,6 +109,9 @@ if __name__ == "__main__": '-v', '--verbose', action='store_true', help='Verbose output') parser.add_argument( '--vng', action='store_true', default=True, help='Run in vng') + parser.add_argument( + '--sched', default='', help='Scheduler to test (default: all)' + ) args = parser.parse_args() if args.verbose: diff --git a/meson-scripts/test_sched b/meson-scripts/test_sched index 1e39a77a..ab62ef83 100755 --- a/meson-scripts/test_sched +++ b/meson-scripts/test_sched @@ -18,13 +18,27 @@ GUEST_TIMEOUT=60 # declare -A SCHEDS -SCHEDS["scx_simple"]="" -SCHEDS["scx_central"]="" -SCHEDS["scx_nest"]="" -SCHEDS["scx_rusty"]="" -SCHEDS["scx_rustland"]="" -SCHEDS["scx_bpfland"]="" -SCHEDS["scx_layered"]="--run-example" +# enable running tests on individual schedulers +if [ $# -ge 2 ] ; then + SCHEDS[$2]="" +else + SCHEDS["scx_simple"]="" + SCHEDS["scx_central"]="" + SCHEDS["scx_nest"]="" + SCHEDS["scx_rusty"]="" + SCHEDS["scx_rustland"]="" + SCHEDS["scx_bpfland"]="" + SCHEDS["scx_layered"]="--run-example" +fi + +if [[ -v SCHEDS["scx_layered"] ]] ; then + SCHEDS["scx_layered"]="--run-example" +fi + +printf "testing scheds:\n" +for i in "${!SCHEDS[@]}"; do + printf "%s=%s\n" "$i" "${SCHEDS[$i]}"; +done if [ ! -x `which vng` ]; then echo "vng not found, please install virtme-ng to enable testing" @@ -48,7 +62,7 @@ for sched in ${!SCHEDS[@]}; do rm -f /tmp/output timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v -r ${kernel} -- \ + vng -m 2G -v -r ${kernel} -- \ "timeout --foreground --preserve-status ${TEST_TIMEOUT} ${sched_path} ${args}" \ 2> >(tee /tmp/output) "${DIFF_DIR}/${SCHED}_new.csv" \ || echo "failed to verify ${SCHED}: ${BPF_PATH}" if [ -n "${DIFF_DIR}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat -C "${DIFF_DIR}/${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \ -e file,prog,verdict,insns \ || echo "failed to compare ${SCHED}: ${BPF_PATH}" @@ -47,12 +47,12 @@ for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do SCHED=$(echo "${BPF_PATH}" | sed 's/.*\/rust\///g' | sed 's/\/.*//g') if [ -n "${KERNEL}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \ || echo "failed to verify ${SCHED}: ${BPF_PATH}" if [ -n "${DIFF_DIR}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat -C "${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \ -e file,prog,verdict,insns \ || echo "failed to compare ${SCHED}: ${BPF_PATH}" diff --git a/meson.build b/meson.build index 7a8f42b9..6077b4ed 100644 --- a/meson.build +++ b/meson.build @@ -330,8 +330,21 @@ endif run_target('test_sched', command: [test_sched, kernel]) +SCHEDULERS = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo', + 'scx_rusty', + 'scx_layered'] # scx_mitosis temporarily excluded + +foreach s : SCHEDULERS + run_target('test_sched_'+s, command: [test_sched, kernel, s]) +endforeach + run_target('veristat', command: [run_veristat, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel')]) +foreach s : SCHEDULERS + run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(), + get_option('veristat_scheduler'), get_option('kernel'), s]) +endforeach + run_target('veristat_diff', command: [run_veristat_diff, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel'), get_option('veristat_diff_dir')]) @@ -343,6 +356,10 @@ subdir('rust') if enable_stress run_target('stress_tests', command: [run_stress_tests, '-k', kernel, '-b', meson.current_build_dir()]) +foreach s : SCHEDULERS + run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b', + meson.current_build_dir(), '--sched', s]) +endforeach endif systemd = dependency('systemd', required: get_option('systemd')) diff --git a/scheds/rust/meson.build b/scheds/rust/meson.build index 26dde06b..1d96c1c3 100644 --- a/scheds/rust/meson.build +++ b/scheds/rust/meson.build @@ -12,12 +12,7 @@ custom_target('rust_scheds', build_by_default: true, build_always_stale: true) -# per-scheduler targets -rust_scheds = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo', - 'scx_rusty', - 'scx_layered'] # scx_mitosis temporarily excluded - -foreach sched: rust_scheds +foreach sched: SCHEDULERS custom_target(sched, output: sched + '@PLAINNAME@.__PHONY__', input: 'Cargo.toml',