From 623309fbcbf1c16476ab480f66e4a63fdcc28d9c Mon Sep 17 00:00:00 2001 From: Jiaqi Gao Date: Tue, 18 Jul 2023 11:54:57 -0400 Subject: [PATCH] CI: add virtio-serial build and integration test Signed-off-by: Jiaqi Gao --- .github/workflows/integration-tdx.yml | 74 ++++++++++++- .github/workflows/main.yml | 54 +++++++++- sh_script/build_final.sh | 17 ++- sh_script/integration_tdx.sh | 150 +++++++++++++++----------- 4 files changed, 220 insertions(+), 75 deletions(-) diff --git a/.github/workflows/integration-tdx.yml b/.github/workflows/integration-tdx.yml index 4e4b1546..2e40477c 100644 --- a/.github/workflows/integration-tdx.yml +++ b/.github/workflows/integration-tdx.yml @@ -15,8 +15,8 @@ env: TOOLCHAIN_PROFILE: minimal jobs: - system_compile: - name: Run TDX Integration Test + virtio_vsock: + name: Run TDX Integration Test (virtio-vsock) runs-on: [self-hosted, tdx] # timeout-minutes: 30 @@ -99,4 +99,72 @@ jobs: - name: Run Tests - Run migration test case 011 run: | - sudo bash sh_script/integration_tdx.sh -f Bin/migtd_011.bin \ No newline at end of file + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_011.bin + + virtio_serial: + name: Run TDX Integration Test (virtio-serial) + runs-on: [self-hosted, tdx] + + steps: + - name: Checkout sources + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Build Migration TD binary + run: bash sh_script/build_final.sh -t migtd -d serial -c -a on + + - name: Run Tests - Test Migration TD + run: | + sudo bash sh_script/integration_tdx.sh -f target/release/migtd.bin -d serial + + - name: Run Tests - Cycle Test Migration TD (20) + run: | + sudo bash sh_script/integration_tdx.sh -f target/release/migtd.bin -d serial -n 20 + + - name: Build all test binaries + run: bash sh_script/build_final.sh -t test -c -a on -d serial + + - name: Run Tests - Run migration test case 001 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_001.bin -d serial + + - name: Run Tests - Run migration test case 002 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_002.bin -d serial + + - name: Run Tests - Run migration test case 003 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_003.bin -d serial + + - name: Run Tests - Run migration test case 004 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_004.bin -d serial + + - name: Run Tests - Run migration test case 005 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_005.bin -d serial + + - name: Run Tests - Run migration test case 006 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_006.bin -d serial + + - name: Run Tests - Run migration test case 007 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_007.bin -d serial + + - name: Run Tests - Run migration test case 008 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_008.bin -d serial + + - name: Run Tests - Run migration test case 009 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_009.bin -d serial + + - name: Run Tests - Run migration test case 010 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_010.bin -d serial + + - name: Run Tests - Run migration test case 011 + run: | + sudo bash sh_script/integration_tdx.sh -f Bin/migtd_011.bin -d serial \ No newline at end of file diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6914d5e..1a1fd0e0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,7 @@ env: TOOLCHAIN_PROFILE: minimal jobs: - system_compile: + build_migtd: name: Build final migtd runs-on: ubuntu-20.04 timeout-minutes: 30 @@ -60,4 +60,54 @@ jobs: run: bash sh_script/build_final.sh -t migtd -c -a on - name: Build Migtd Final Binary Using Vmcall-Vsock - run: bash sh_script/build_final.sh -t migtd -v vmcall -c \ No newline at end of file + run: bash sh_script/build_final.sh -t migtd -d vmcall -c + + - name: Build Migtd Final Binary Using virtio-serial + run: bash sh_script/build_final.sh -t migtd -d serial -c + + build_migtd_features: + name: Build final migtd (vmcall-vsock/virtio-serial) + runs-on: ubuntu-20.04 + timeout-minutes: 30 + + steps: + # Install first since it's needed to build NASM + - name: Install LLVM and Clang + uses: KyleMayes/install-llvm-action@v1 + with: + version: "10.0" + directory: ${{ runner.temp }}/llvm + + - name: Install NASM + uses: ilammy/setup-nasm@v1 + + - name: Install tools for sgx lib + run: sudo apt-get install build-essential ocaml ocamlbuild automake autoconf libtool wget python-is-python3 libssl-dev git cmake perl + + - name: Checkout sources + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + profile: ${{ env.TOOLCHAIN_PROFILE }} + toolchain: ${{ env.RUST_TOOLCHAIN }} + override: true + components: rust-src + + - name: Run cargo install cargo-xbuild + uses: actions-rs/cargo@v1 + with: + command: install + args: cargo-xbuild + + - name: Preparation Work + run: bash sh_script/preparation.sh + + - name: Build Migtd Final Binary Using Vmcall-Vsock + run: bash sh_script/build_final.sh -t migtd -d vmcall -c + + - name: Build Migtd Final Binary Using virtio-serial + run: bash sh_script/build_final.sh -t migtd -d serial -c \ No newline at end of file diff --git a/sh_script/build_final.sh b/sh_script/build_final.sh index 8579ddba..413fbf5c 100644 --- a/sh_script/build_final.sh +++ b/sh_script/build_final.sh @@ -18,11 +18,11 @@ function cleanup() { } function proccess_args() { - while getopts ":t:a:v:c" option; do + while getopts ":t:a:v:d:c" option; do case "${option}" in t) type=${OPTARG};; a) attestation=${OPTARG};; - v) vsock=${OPTARG};; + d) device=${OPTARG};; c) cleanup;; esac done @@ -45,13 +45,12 @@ function proccess_args() { then MIGTD_FEATURE+=",remote-attestation" fi - - if [[ ${vsock} == "vmcall" ]]; - then - MIGTD_FEATURE+=",vmcall-vsock" - else - MIGTD_FEATURE+=",virtio-vsock" - fi + + case "${device}" in + vmcall) MIGTD_FEATURE+=",vmcall-vsock";; + serial) MIGTD_FEATURE+=",virtio-serial";; + *) MIGTD_FEATURE+=",virtio-vsock";; + esac } function check_file_exist() { diff --git a/sh_script/integration_tdx.sh b/sh_script/integration_tdx.sh index ed260194..128f23eb 100644 --- a/sh_script/integration_tdx.sh +++ b/sh_script/integration_tdx.sh @@ -28,9 +28,23 @@ cycle=0 # Test Configuration Info cpus=1 memory=32M +device=vsock # trap cleanup exit +# MigTD start command +migtd_qemu_cmd="${qemu_tdx_path} \ + -accel kvm \ + -M q35 \ + -cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ + -smp ${cpus},threads=1,sockets=${cpus} \ + -m ${memory} \ + -object tdx-guest,id=tdx0,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ + -object memory-backend-memfd-private,id=ram1,size=${memory} \ + -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ + -no-hpet \ + -nographic -vga none -nic none" + usage() { cat << EOM Usage: $(basename "$0") [OPTION]... @@ -42,13 +56,14 @@ Usage: $(basename "$0") [OPTION]... -c by default is 1. -m by defalt is 2G. -n Cycle test number. + -d Device(transport) used for host-guest communication, by default is `vsock`. -h Show help info EOM exit 0 } proccess_args() { - while getopts ":i:p:k:f:t:c:m:n:h" option; do + while getopts ":i:p:k:f:t:c:m:n:d:h" option; do case "${option}" in i) guest_image=${OPTARG};; p) qemu_tdx_path=${OPTARG};; @@ -58,6 +73,7 @@ proccess_args() { c) cpus=${OPTARG};; m) memory=${OPTARG};; n) cycle=${OPTARG};; + d) device=${OPTARG};; h) usage;; esac done @@ -83,6 +99,7 @@ proccess_args() { echo "Type : ${type}" echo "CPUs : ${cpus}" echo "Memmory Size : ${memory}" + echo "Device type : ${device}" echo "=========================================" } @@ -157,60 +174,65 @@ setup_agent() { } launch_src_migtd() { - nohup ${qemu_tdx_path} \ - -accel kvm \ - -M q35 \ - -cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ - -smp ${cpus},threads=1,sockets=${cpus} \ - -m ${memory} \ - -object tdx-guest,id=tdx0,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ - -object memory-backend-memfd-private,id=ram1,size=${memory} \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -bios $1 \ - -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=18,disable-legacy=on \ - -name migtd-src,process=migtd-src,debug-threads=on \ - -no-hpet \ - -nographic -vga none -nic none \ - -serial mon:stdio > ${mig_src_log} & + local cmd="${migtd_qemu_cmd} \ + -bios $1 \ + -name migtd-src,process=migtd-src,debug-threads=on \ + -serial mon:stdio" + + if [[ ${device} == serial ]] + then + cmd="${cmd} \ + -device virtio-serial-pci,id=virtio-serial0 \ + -chardev socket,host=127.0.0.1,port=1234,server=off,id=foo \ + -device virtserialport,chardev=foo,bus=virtio-serial0.0" + elif [[ ${device} == vsock ]] + then + cmd="${cmd} \ + -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=18,disable-legacy=on" + fi + + nohup ${cmd} > ${mig_src_log} & sleep 10 } launch_dst_migtd() { - nohup ${qemu_tdx_path} \ - -accel kvm \ - -M q35 \ - -cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ - -smp ${cpus},threads=1,sockets=${cpus} \ - -m ${memory} \ - -object tdx-guest,id=tdx0,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ - -object memory-backend-memfd-private,id=ram1,size=${memory} \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -bios $1 \ - -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=36,disable-legacy=on \ - -name migtd-dst,process=migtd-dst,debug-threads=on \ - -no-hpet \ - -nographic -vga none -nic none \ - -serial mon:stdio > ${mig_dst_log} & + local cmd="${migtd_qemu_cmd} \ + -bios $1 \ + -name migtd-dst,process=migtd-dst,debug-threads=on \ + -serial mon:stdio" + + if [[ ${device} == serial ]] + then + cmd="${cmd} \ + -device virtio-serial-pci,id=virtio-serial0 \ + -chardev socket,host=127.0.0.1,port=1234,server=on,id=foo \ + -device virtserialport,chardev=foo,bus=virtio-serial0.0" + elif [[ ${device} == vsock ]] + then + cmd="${cmd} \ + -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=36,disable-legacy=on" + fi + + nohup ${cmd} > ${mig_dst_log} & sleep 10 } -launch_dst_migtd_without_vsock() { - nohup ${qemu_tdx_path} \ - -accel kvm \ - -M q35 \ - -cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ - -smp ${cpus},threads=1,sockets=${cpus} \ - -m ${memory} \ - -object tdx-guest,id=tdx0,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ - -object memory-backend-memfd-private,id=ram1,size=${memory} \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -bios $1 \ - -name migtd-dst,process=migtd-dst,debug-threads=on \ - -no-hpet \ - -nographic -vga none -nic none \ - -serial mon:stdio > ${mig_dst_log} & +launch_src_migtd_without_device() { + local cmd="${migtd_qemu_cmd} \ + -bios $1 \ + -name migtd-src,process=migtd-src,debug-threads=on \ + -serial mon:stdio" + + # Connect to dst MigTD to make it run + if [[ ${device} == serial ]] + then + cmd="${cmd} \ + -chardev socket,host=127.0.0.1,port=1234,server=off,id=foo" + fi + + nohup ${cmd} > ${mig_src_log} & sleep 10 } @@ -274,25 +296,29 @@ test_migtd() { echo "-- start test migration td" local time_out=30 - echo "-- setup agent" - setup_agent + if [[ ${device} == vsock ]] + then + echo "-- setup agent" + setup_agent + fi + + echo "-- launch dst migtd" + launch_dst_migtd ${firmware} + echo "-- launch src migtd" if [[ ${firmware} == *004* ]] || [[ ${firmware} == *005* ]] || [[ ${firmware} == *006* ]] || [[ ${firmware} == *migtd.bin* ]] then launch_src_migtd ${firmware} - elif [[ ${firmware} == *007* ]] || [[ ${firmware} == *008* ]] || [[ ${firmware} == *009* ]] + elif [[ ${firmware} == *007* ]] || [[ ${firmware} == *008* ]] then launch_src_migtd "`dirname ${firmware}`/migtd_no.bin" - else - launch_src_migtd "`dirname ${firmware}`/migtd_001.bin" - fi - echo "-- launch dst migtd" - if [[ ${firmware} == *009* ]] + elif [[ ${firmware} == *009* ]] then - launch_dst_migtd_without_vsock ${firmware} + launch_src_migtd_without_device "`dirname ${firmware}`/migtd_no.bin" else - launch_dst_migtd ${firmware} + launch_src_migtd "`dirname ${firmware}`/migtd_001.bin" fi + echo "-- launch src td" launch_src_td echo "-- launch dst td" @@ -324,13 +350,15 @@ cycle_test_migtd() { echo "-- start test migration td" local time_out=30 - echo "-- setup agent" - setup_agent - echo "-- launch src migtd" - launch_src_migtd ${firmware} - + if [[ ${device} == vsock ]] + then + echo "-- setup agent" + setup_agent + fi echo "-- launch dst migtd" launch_dst_migtd ${firmware} + echo "-- launch src migtd" + launch_src_migtd ${firmware} for ((i=1;i<${cycle};i++)) do