split build and test jobs

split build and test jobs to reduce ci turnaround time and make it clear what is failing when something fails. also add virtiofsd to deps to make test compilation faster (most test time is compliation) and remove all force 9ps.
sched-ext · Sep 8, 2024 · 120211d · 120211d
1 parent 92b887c
commit 120211d
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 37 deletions.
diff --git a/.github/actions/install-deps-action/action.yml b/.github/actions/install-deps-action/action.yml
@@ -16,7 +16,7 @@ runs:
     ### DOWNLOAD AND INSTALL DEPENDENCIES ###
 
     # Download dependencies packaged by Ubuntu
-    - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd
+    - run: sudo apt -y install bison busybox-static cargo cmake coreutils cpio elfutils file flex gcc gcc-multilib git iproute2 jq kbd kmod libcap-dev libelf-dev libunwind-dev libvirt-clients libzstd-dev linux-headers-generic linux-tools-common linux-tools-generic make ninja-build pahole pkg-config python3-dev python3-pip python3-requests qemu-kvm rsync rustc stress-ng udev zstd libseccomp-dev libcap-ng-dev
       shell: bash
 
     # clang 17

diff --git a/.github/workflows/caching-build.yml b/.github/workflows/caching-build.yml
@@ -25,19 +25,30 @@ jobs:
         with:
           path: |
             linux
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
         uses: ./.github/actions/install-deps-action
 
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' && steps.cache-kernel.outputs.cache-hit != 'true' }}
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # cache bzImage alone for rust tests (disk space limit workaround)
       - name: Cache bzImage
         id: cache-bzImage
         uses: actions/cache@v4
         with:
           path: |
             linux/arch/x86/boot/bzImage
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
         name: Clone Kernel
@@ -64,10 +75,23 @@ jobs:
   integration-test:
     runs-on: ubuntu-22.04
     needs: build-kernel
+    strategy:
+          matrix:
+            scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install-deps-action
-      # cache rust deps, invalidate when kernel commit changes
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # get latest head commit of sched_ext for-next
       - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
 
@@ -78,7 +102,7 @@ jobs:
         with:
           path: |
             linux
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-build-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       # need to re-run job when kernel head changes between build and test running.
       - if: ${{ steps.cache-kernel.outputs.cache-hit != 'true' }}
@@ -92,25 +116,42 @@ jobs:
 
       # The actual build:
       - run: meson setup build -Dkernel=$(pwd)/linux -Dkernel_headers=./linux/usr/include -Denable_stress=true
-      - run: meson compile -C build
+      - run: meson compile -C build ${{ matrix.scheduler }}
 
       # Print CPU model before running the tests (this can be useful for
       # debugging purposes)
       - run: grep 'model name' /proc/cpuinfo | head -1
 
       # Test schedulers
-      - run: meson compile -C build test_sched
+      - run: meson compile -C build test_sched_${{ matrix.scheduler }}
       # Stress schedulers
-      - run: meson compile -C build stress_tests
-      - run: meson compile -C build veristat
+      - uses: cytopia/[email protected]
+        name: stress test
+        with:
+          retries: 3
+          command: meson compile -C build stress_tests_${{ matrix.scheduler }}
+      - run: meson compile -C build veristat_${{ matrix.scheduler }}
 
-  rust-test:
+  rust-test-core:
     runs-on: ubuntu-22.04
     needs: build-kernel
+    strategy:
+      matrix:
+        component: [scx_loader, scx_rustland_core, scx_stats, scx_utils]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/install-deps-action
-      # cache rust deps, invalidate when kernel commit changes
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
       # get latest head commit of sched_ext for-next
       - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
       # cache bzImage alone for rust tests
@@ -120,7 +161,7 @@ jobs:
         with:
           path: |
             linux/arch/x86/boot/bzImage
-          key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
 
       # need to re-run job when kernel head changes between build and test running.
       - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }}
@@ -129,9 +170,55 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
         with:
-          shared-key: ${{ env.SCHED_EXT_KERNEL_COMMIT }}
           workspaces: rust
+          key: ${{ matrix.component }}
+          prefix-key: "1"
+      - run: cargo build  --manifest-path rust/${{ matrix.component }}/Cargo.toml
+      - run: cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml --no-run
+      - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path rust/${{ matrix.component }}/Cargo.toml
+
+  rust-test-schedulers:
+    runs-on: ubuntu-22.04
+    needs: build-kernel
+    strategy:
+      matrix:
+        scheduler: [ scx_bpfland, scx_lavd, scx_layered, scx_rlfifo, scx_rustland, scx_rusty ]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/install-deps-action
+      # cache virtiofsd (goes away w/ 24.04)
+      - name: Cache virtiofsd
+        id: cache-virtiofsd
+        uses: actions/cache@v4
+        with:
+          path: |
+            /usr/lib/virtiofsd
+          key: virtiofsd-binary
+      - if: ${{ steps.cache-virtiofsd.outputs.cache-hit != 'true' }}          
+        run: cargo install virtiofsd && sudo cp -a ~/.cargo/bin/virtiofsd /usr/lib/
+
+      # get latest head commit of sched_ext for-next
+      - run: echo "SCHED_EXT_KERNEL_COMMIT=$(git ls-remote https://git.kernel.org/pub/scm/linux/kernel/git/tj/sched_ext.git heads/for-next | awk '{print $1}')" >> $GITHUB_ENV
+      # cache bzImage alone for rust tests
+      - name: Cache bzImage
+        id: cache-bzImage
+        uses: actions/cache@v4
+        with:
+          path: |
+            linux/arch/x86/boot/bzImage
+          key: kernel-bzImage-${{ env.SCHED_EXT_KERNEL_COMMIT }}
+
+      # need to re-run job when kernel head changes between build and test running.
+      - if: ${{ steps.cache-bzImage.outputs.cache-hit != 'true' }}
+        name: exit if cache stale
+        run: exit -1
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: scheds/rust
+          key: ${{ matrix.scheduler }}
           prefix-key: "1"
-      - run: cargo build  --manifest-path rust/Cargo.toml
-      - run: cargo test --manifest-path rust/Cargo.toml --no-run
-      - run: vng -v --memory 10G --cpu 8 --force-9p -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path rust/Cargo.toml
+      - run: cargo build  --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml
+      - run: cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml --no-run
+      - run: vng -v --memory 10G --cpu 8 -r linux/arch/x86/boot/bzImage  --net user -- cargo test --manifest-path scheds/rust/${{ matrix.scheduler }}/Cargo.toml
+
diff --git a/meson-scripts/run_stress_tests b/meson-scripts/run_stress_tests
@@ -48,7 +48,7 @@ def run_stress_test(
     timeout_sec = int(config.get("timeout_sec"))
     if vng_path:
         vm_input = f"{stress_cmd} & timeout --foreground --preserve-status {timeout_sec} {sched_cmd}"
-        cmd = [vng_path, "--user", "root", "--force-9p", "-v", "--", vm_input]
+        cmd = [vng_path, "--user", "root", "-v", "--", vm_input]
         err = sys.stderr if output == "-" else open(output, "w")
         out = sys.stdout if output == "-" else err
         proc = subprocess.Popen(
@@ -73,6 +73,9 @@ def stress_tests(args: Namespace) -> None:
     for test_name in configs.sections():
         if args.stress_test and test_name != args.stress_test:
             continue
+        if args.sched and configs[test_name].get('sched') != args.sched:
+            continue
+        print(f"Running stress test: {test_name}")
         return_codes[test_name] = run_stress_test(
             configs[test_name],
             args.build_dir,
@@ -106,6 +109,9 @@ if __name__ == "__main__":
         '-v', '--verbose', action='store_true', help='Verbose output')
     parser.add_argument(
         '--vng', action='store_true', default=True, help='Run in vng')
+    parser.add_argument(
+        '--sched', default='', help='Scheduler to test (default: all)'
+    )
 
     args = parser.parse_args()
     if args.verbose:

diff --git a/meson-scripts/test_sched b/meson-scripts/test_sched
@@ -18,13 +18,27 @@ GUEST_TIMEOUT=60
 #
 declare -A SCHEDS
 
-SCHEDS["scx_simple"]=""
-SCHEDS["scx_central"]=""
-SCHEDS["scx_nest"]=""
-SCHEDS["scx_rusty"]=""
-SCHEDS["scx_rustland"]=""
-SCHEDS["scx_bpfland"]=""
-SCHEDS["scx_layered"]="--run-example"
+# enable running tests on individual schedulers
+if [ $# -ge 2 ] ; then
+    SCHEDS[$2]=""
+else
+    SCHEDS["scx_simple"]=""
+    SCHEDS["scx_central"]=""
+    SCHEDS["scx_nest"]=""
+    SCHEDS["scx_rusty"]=""
+    SCHEDS["scx_rustland"]=""
+    SCHEDS["scx_bpfland"]=""
+    SCHEDS["scx_layered"]="--run-example"
+fi
+
+if [[ -v SCHEDS["scx_layered"] ]] ; then
+    SCHEDS["scx_layered"]="--run-example"
+fi
+
+printf "testing scheds:\n"
+for i in "${!SCHEDS[@]}"; do
+    printf "%s=%s\n" "$i" "${SCHEDS[$i]}";
+done
 
 if [ ! -x `which vng` ]; then
     echo "vng not found, please install virtme-ng to enable testing"
@@ -48,7 +62,7 @@ for sched in ${!SCHEDS[@]}; do
 
     rm -f /tmp/output
     timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v -r ${kernel} -- \
+        vng -m 2G -v -r ${kernel} -- \
             "timeout --foreground --preserve-status ${TEST_TIMEOUT} ${sched_path} ${args}" \
                 2> >(tee /tmp/output) </dev/null
         grep -v " Speculative Return Stack Overflow" /tmp/output | \

diff --git a/meson-scripts/veristat b/meson-scripts/veristat
@@ -20,7 +20,7 @@ if [ -n "${SCHED}" ]; then
   echo "Running veristat on ${BPF_PATH}"
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --user root -r ${KERNEL} -- \
+      vng -m 2G -v --user root -r ${KERNEL} -- \
         sudo veristat ${BPF_PATH}
     exit $?
   else
@@ -32,7 +32,7 @@ fi
 for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --user root -r ${KERNEL} -- \
+      vng -m 2G -v --user root -r ${KERNEL} -- \
         sudo veristat ${BPF_PATH}
   else
     echo "$BPF_PATH"

diff --git a/meson-scripts/veristat_diff b/meson-scripts/veristat_diff
@@ -21,12 +21,12 @@ if [ -n "${SCHED}" ]; then
   echo "Running veristat on ${BPF_PATH}"
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+      vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
         veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \
 	  || echo "failed to verify ${SCHED}: ${BPF_PATH}"
     if [ -n "${DIFF_DIR}" ]; then
       timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+        vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
           veristat -C "${DIFF_DIR}/${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \
 	    -e file,prog,verdict,insns \
 	    || echo "failed to compare ${SCHED}: ${BPF_PATH}"
@@ -47,12 +47,12 @@ for BPF_PATH in $(find ${BUILD_DIR} -type f -name bpf.bpf.o); do
   SCHED=$(echo "${BPF_PATH}" | sed 's/.*\/rust\///g' | sed 's/\/.*//g')
   if [ -n "${KERNEL}" ]; then
     timeout --preserve-status ${GUEST_TIMEOUT} \
-      vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+      vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
         veristat ${BPF_PATH} -o csv > "${DIFF_DIR}/${SCHED}_new.csv" \
           || echo "failed to verify ${SCHED}: ${BPF_PATH}"
     if [ -n "${DIFF_DIR}" ]; then
       timeout --preserve-status ${GUEST_TIMEOUT} \
-        vng -m 2G --force-9p -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
+        vng -m 2G -v --rwdir "${DIFF_DIR}" --user root -r ${KERNEL} -- \
           veristat -C "${SCHED}_new.csv" "${DIFF_DIR}/${SCHED}.csv" \
 	    -e file,prog,verdict,insns \
             || echo "failed to compare ${SCHED}: ${BPF_PATH}"

diff --git a/meson.build b/meson.build
@@ -330,8 +330,21 @@ endif
 
 run_target('test_sched', command: [test_sched, kernel])
 
+SCHEDULERS = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo',
+               'scx_rusty',
+               'scx_layered'] # scx_mitosis temporarily excluded                        
+
+foreach s : SCHEDULERS
+  run_target('test_sched_'+s, command: [test_sched, kernel, s])
+endforeach
+
 run_target('veristat', command: [run_veristat, meson.current_build_dir(),
                                  get_option('veristat_scheduler'), get_option('kernel')])
+foreach s : SCHEDULERS
+  run_target('veristat_'+s, command: [run_veristat, meson.current_build_dir(),
+                                 get_option('veristat_scheduler'), get_option('kernel'), s])
+endforeach
+
 run_target('veristat_diff', command: [run_veristat_diff, meson.current_build_dir(),
                                  get_option('veristat_scheduler'), get_option('kernel'),
                                  get_option('veristat_diff_dir')])
@@ -343,6 +356,10 @@ subdir('rust')
 if enable_stress
   run_target('stress_tests', command: [run_stress_tests, '-k', kernel, '-b',
     meson.current_build_dir()])
+foreach s : SCHEDULERS
+  run_target('stress_tests_'+s, command: [run_stress_tests, '-k', kernel, '-b',
+    meson.current_build_dir(), '--sched', s])
+endforeach
 endif
 
 systemd = dependency('systemd', required: get_option('systemd'))

diff --git a/scheds/rust/meson.build b/scheds/rust/meson.build
@@ -12,12 +12,7 @@ custom_target('rust_scheds',
               build_by_default: true,
               build_always_stale: true)
 
-# per-scheduler targets
-rust_scheds = ['scx_lavd', 'scx_bpfland', 'scx_rustland', 'scx_rlfifo',
-               'scx_rusty',
-               'scx_layered'] # scx_mitosis temporarily excluded
-
-foreach sched: rust_scheds
+foreach sched: SCHEDULERS
   custom_target(sched,
                 output: sched + '@PLAINNAME@.__PHONY__',
                 input: 'Cargo.toml',