diff --git a/.github/actions/install-deps-action/action.yml b/.github/actions/install-deps-action/action.yml index 47612eb16..d83c27494 100644 --- a/.github/actions/install-deps-action/action.yml +++ b/.github/actions/install-deps-action/action.yml @@ -16,7 +16,7 @@ runs: ### DOWNLOAD AND INSTALL DEPENDENCIES ### # Download dependencies packaged by Ubuntu - - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd + - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd libseccomp-dev libcap-ng-dev shell: bash # clang 17 diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml index d93bab240..f8787c710 100644 --- a/.github/workflows/caching-build.yml +++ b/.github/workflows/caching-build.yml @@ -25,11 +25,22 @@ jobs: with: path: | linux - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} uses: ./.github/actions/install-deps-action + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' && steps.cache-kernel.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # cache bzImage alone for rust tests (disk space limit workaround) - name: Cache bzImage id: cache-bzImage @@ -37,7 +48,7 @@ jobs: with: path: | linux/arch/x86/boot/bzImage - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} name: Clone Kernel @@ -64,10 +75,23 @@ jobs: integration-test: runs-on: ubuntu-22.04 needs: build-kernel + strategy: + matrix: + scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/install-deps-action - # cache rust deps, invalidate when kernel commit changes + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV @@ -78,7 +102,7 @@ jobs: with: path: | linux - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }} # need to re-run job when kernel head changes between build and test running. - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }} @@ -92,25 +116,42 @@ jobs: # The actual build: - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true - - run: meson compile -C build + - run: meson compile -C build ${{ matrix.scheduler }} # Print CPU model before running the tests (this can be useful for # debugging purposes) - run: grep 'model name' /proc/cpuinfo | head -1 # Test schedulers - - run: meson compile -C build test_sched + - run: meson compile -C build test_sched_${{ matrix.scheduler }} # Stress schedulers - - run: meson compile -C build stress_tests - - run: meson compile -C build veristat + - uses: cytopia/shell-command-retry-action@v0.1.2 + name: stress test + with: + retries: 3 + command: meson compile -C build stress_tests_${{ matrix.scheduler }} + - run: meson compile -C build veristat_${{ matrix.scheduler }} - rust-test: + rust-test-core: runs-on: ubuntu-22.04 needs: build-kernel + strategy: + matrix: + component: [scx_loader, scx_rustland_core, scx_stats, scx_utils] steps: - uses: actions/checkout@v4 - uses: ./.github/actions/install-deps-action - # cache rust deps, invalidate when kernel commit changes + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + # get latest head commit of sched_ext for-next - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV # cache bzImage alone for rust tests @@ -120,7 +161,7 @@ jobs: with: path: | linux/arch/x86/boot/bzImage - key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} # need to re-run job when kernel head changes between build and test running. - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} @@ -129,9 +170,55 @@ jobs: - uses: Swatinem/rust-cache@v2 with: - shared-key: ${{ env.SCHED_EXT_KERNEL_COMMIT }} workspaces: rust + key: ${{ matrix.component }} + prefix-key: "1" + - run: cargo build --manifest-path rust/${{ matrix.component }}/Cargo.toml + - run: cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml --no-run + - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml + + rust-test-schedulers: + runs-on: ubuntu-22.04 + needs: build-kernel + strategy: + matrix: + scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ] + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/install-deps-action + # cache virtiofsd (goes away w/ 24.04) + - name: Cache virtiofsd + id: cache-virtiofsd + uses: actions/cache@v4 + with: + path: | + /usr/lib/virtiofsd + key: virtiofsd-binary + - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }} + run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/ + + # get latest head commit of sched_ext for-next + - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV + # cache bzImage alone for rust tests + - name: Cache bzImage + id: cache-bzImage + uses: actions/cache@v4 + with: + path: | + linux/arch/x86/boot/bzImage + key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }} + + # need to re-run job when kernel head changes between build and test running. + - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }} + name: exit if cache stale + run: exit -1 + + - uses: Swatinem/rust-cache@v2 + with: + workspaces: scheds/rust + key: ${{ matrix.scheduler }} prefix-key: "1" - - run: cargo build --manifest-path rust/Cargo.toml - - run: cargo test --manifest-path rust/Cargo.toml --no-run - - run: vng -v --memory 10G --cpu 8 --force-9p -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path rust/Cargo.toml + - run: cargo build --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml + - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run + - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml + diff --git a/meson-scripts/run_stress_tests b/meson-scripts/run_stress_tests index 2b0bcf185..8d58ec65a 100755 --- a/meson-scripts/run_stress_tests +++ b/meson-scripts/run_stress_tests @@ -48,7 +48,7 @@ def run_stress_test( timeout_sec = int(config.get("timeout_sec")) if vng_path: vm_input = f"{stress_cmd} & timeout --foreground --preserve-status {timeout_sec} {sched_cmd}" - cmd = [vng_path, "--user", "root", "--force-9p", "-v", "--", vm_input] + cmd = [vng_path, "--user", "root", "-v", "--", vm_input] err = sys.stderr if output == "-" else open(output, "w") out = sys.stdout if output == "-" else err proc = subprocess.Popen( @@ -73,6 +73,9 @@ def stress_tests(args: Namespace) -> None: for test_name in configs.sections(): if args.stress_test and test_name != args.stress_test: continue + if args.sched and configs[test_name].get('sched') != args.sched: + continue + print(f"Running stress test: {test_name}") return_codes[test_name] = run_stress_test( configs[test_name], args.build_dir, @@ -106,6 +109,9 @@ if __name__ == "__main__": '-v', '--verbose', action='store_true', help='Verbose output') parser.add_argument( '--vng', action='store_true', default=True, help='Run in vng') + parser.add_argument( + '--sched', default='', help='Scheduler to test (default: all)' + ) args = parser.parse_args() if args.verbose: diff --git a/meson-scripts/test_sched b/meson-scripts/test_sched index 1e39a77a9..ab62ef831 100755 --- a/meson-scripts/test_sched +++ b/meson-scripts/test_sched @@ -18,13 +18,27 @@ GUEST_TIMEOUT=60 # declare -A SCHEDS -SCHEDS["scx_simple"]="" -SCHEDS["scx_central"]="" -SCHEDS["scx_nest"]="" -SCHEDS["scx_rusty"]="" -SCHEDS["scx_rustland"]="" -SCHEDS["scx_bpfland"]="" -SCHEDS["scx_layered"]="--run-example" +# enable running tests on individual schedulers +if [ $# -ge 2 ] ; then + SCHEDS[$2]="" +else + SCHEDS["scx_simple"]="" + SCHEDS["scx_central"]="" + SCHEDS["scx_nest"]="" + SCHEDS["scx_rusty"]="" + SCHEDS["scx_rustland"]="" + SCHEDS["scx_bpfland"]="" + SCHEDS["scx_layered"]="--run-example" +fi + +if [[ -v SCHEDS["scx_layered"] ]] ; then + SCHEDS["scx_layered"]="--run-example" +fi + +printf "testing scheds:\n" +for i in "${!SCHEDS[@]}"; do + printf "%s=%s\n" "$i" "${SCHEDS[$i]}"; +done if [ ! -x `which vng` ]; then echo "vng not found, please install virtme-ng to enable testing" @@ -48,7 +62,7 @@ for sched in ${!SCHEDS[@]}; do rm -f /tmp/output timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v -r ${kernel} -- \ + vng -m 2G -v -r ${kernel} -- \ "timeout --foreground --preserve-status ${TEST_TIMEOUT} ${sched_path} ${args}" \ 2> >(tee /tmp/output) "${DIFF_DIR}/${SCHED}_new.csv" \ || echo "failed to verify ${SCHED}: ${BPF_PATH}" if [ -n "${DIFF_DIR}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat -C "${DIFF_DIR}/${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \ -e file,prog,verdict,insns \ || echo "failed to compare ${SCHED}: ${BPF_PATH}" @@ -47,12 +47,12 @@ for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do SCHED=$(echo "${BPF_PATH}" | sed 's/.*\/rust\///g' | sed 's/\/.*//g') if [ -n "${KERNEL}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \ || echo "failed to verify ${SCHED}: ${BPF_PATH}" if [ -n "${DIFF_DIR}" ]; then timeout --preserve-status ${GUEST_TIMEOUT} \ - vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ + vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \ veristat -C "${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \ -e file,prog,verdict,insns \ || echo "failed to compare ${SCHED}: ${BPF_PATH}" diff --git a/meson.build b/meson.build index 7a8f42b9e..6077b4edf 100644 --- a/meson.build +++ b/meson.build @@ -330,8 +330,21 @@ endif run_target('test_sched', command: [test_sched, kernel]) +SCHEDULERS = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo', + 'scx_rusty', + 'scx_layered'] # scx_mitosis temporarily excluded + +foreach s : SCHEDULERS + run_target('test_sched_'+s, command: [test_sched, kernel, s]) +endforeach + run_target('veristat', command: [run_veristat, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel')]) +foreach s : SCHEDULERS + run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(), + get_option('veristat_scheduler'), get_option('kernel'), s]) +endforeach + run_target('veristat_diff', command: [run_veristat_diff, meson.current_build_dir(), get_option('veristat_scheduler'), get_option('kernel'), get_option('veristat_diff_dir')]) @@ -343,6 +356,10 @@ subdir('rust') if enable_stress run_target('stress_tests', command: [run_stress_tests, '-k', kernel, '-b', meson.current_build_dir()]) +foreach s : SCHEDULERS + run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b', + meson.current_build_dir(), '--sched', s]) +endforeach endif systemd = dependency('systemd', required: get_option('systemd')) diff --git a/scheds/rust/meson.build b/scheds/rust/meson.build index 26dde06bc..1d96c1c37 100644 --- a/scheds/rust/meson.build +++ b/scheds/rust/meson.build @@ -12,12 +12,7 @@ custom_target('rust_scheds', build_by_default: true, build_always_stale: true) -# per-scheduler targets -rust_scheds = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo', - 'scx_rusty', - 'scx_layered'] # scx_mitosis temporarily excluded - -foreach sched: rust_scheds +foreach sched: SCHEDULERS custom_target(sched, output: sched + '@PLAINNAME@.__PHONY__', input: 'Cargo.toml',