diff --git a/.github/workflows/integration-tdx.yml b/.github/workflows/integration-tdx.yml index 86b5f07d..5050809f 100644 --- a/.github/workflows/integration-tdx.yml +++ b/.github/workflows/integration-tdx.yml @@ -33,162 +33,69 @@ jobs: # - name: Install tools for sgx lib # run: sudo dnf group install 'Development Tools' | sudo dnf --enablerepo=powertools install ocaml ocaml-ocamlbuild wget rpm-build pkgcon + - name: Clean test repository + run: | + sudo rm -rf sh_script/test/ - name: Checkout sources uses: actions/checkout@v4 with: submodules: recursive + + - name: Preparation Work + run: bash sh_script/preparation.sh - name: Build Migration TD binary - run: bash sh_script/build_final.sh -t migtd -c -a on - - - name: Run Tests - Test Migration TD - run: | - sudo bash sh_script/integration_tdx.sh -d target/release/migtd.bin -s target/release/migtd.bin + run: cargo image - - name: Run Tests - Cycle Test Migration TD (20) + - name: Run Tests - Test Migration TD 20 Cycles run: | - sudo bash sh_script/integration_tdx.sh -d target/release/migtd.bin -s target/release/migtd.bin -n 20 + pushd sh_script/test + sudo pytest -k "cycle" + popd - name: Build all test binaries run: bash sh_script/build_final.sh -t test -c -a on - - - name: Run Tests - Run test TD payload - run: | - sudo bash sh_script/integration_tdx.sh -f Bin/final-test.bin - - - name: Run Tests - Run migration test case 001 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_001.bin -s Bin/migtd_001.bin - - - name: Run Tests - Run migration test case 002 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_002.bin -s Bin/migtd_001.bin - - - name: Run Tests - Run migration test case 003 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_003.bin -s Bin/migtd_001.bin - - - name: Run Tests - Run migration test case 004 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_004.bin -s Bin/migtd_004.bin - - - name: Run Tests - Run migration test case 005 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_005.bin -s Bin/migtd_005.bin - - - name: Run Tests - Run migration test case 006 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_006.bin -s Bin/migtd_006.bin - - - name: Run Tests - Run migration test case 007 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_007.bin -s Bin/migtd_no.bin - - - name: Run Tests - Run migration test case 008 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_008.bin -s Bin/migtd_no.bin - - - name: Run Tests - Run migration test case 009 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_009.bin -s Bin/migtd_009.bin - - - name: Run Tests - Run migration test case 010 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_010.bin -s Bin/migtd_010.bin - - - name: Run Tests - Run migration test case 011 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_011.bin -s Bin/migtd_011.bin - - - name: Run Tests - Run migration test case 012 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_012.bin -s Bin/migtd_src_012.bin - - - name: Run Tests - Run migration test case 013 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_013.bin -s Bin/migtd_src_013.bin - - name: Run Tests - Run migration test case 014 + - name: Run Tests run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_014.bin -s Bin/migtd_src_014.bin + pushd sh_script/test + sudo pytest -k "not cycle" + popd virtio_serial: name: Run TDX Integration Test (virtio-serial) runs-on: [self-hosted, tdx] steps: + - name: Clean test repository + run: | + sudo rm -rf sh_script/test/ + - name: Checkout sources uses: actions/checkout@v4 with: submodules: recursive + + - name: Preparation Work + run: bash sh_script/preparation.sh - name: Build Migration TD binary - run: bash sh_script/build_final.sh -t migtd -d serial -c -a on - - - name: Run Tests - Test Migration TD - run: | - sudo bash sh_script/integration_tdx.sh -d target/release/migtd.bin -s target/release/migtd.bin -t serial + run: cargo image --no-default-features --features remote-attestation,stack-guard,virtio-serial - - name: Run Tests - Cycle Test Migration TD (20) + - name: Run Tests - Test Migration TD 20 Cycles run: | - sudo bash sh_script/integration_tdx.sh -d target/release/migtd.bin -s target/release/migtd.bin -t serial -n 20 + pushd sh_script/test + sudo pytest -k "cycle" --device_type serial + popd - name: Build all test binaries run: bash sh_script/build_final.sh -t test -c -a on -d serial - - name: Run Tests - Run migration test case 001 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_001.bin -s Bin/migtd_001.bin -t serial - - - name: Run Tests - Run migration test case 002 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_002.bin -s Bin/migtd_001.bin -t serial - - - name: Run Tests - Run migration test case 003 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_003.bin -s Bin/migtd_001.bin -t serial - - - name: Run Tests - Run migration test case 004 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_004.bin -s Bin/migtd_004.bin -t serial - - - name: Run Tests - Run migration test case 005 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_005.bin -s Bin/migtd_005.bin -t serial - - - name: Run Tests - Run migration test case 006 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_006.bin -s Bin/migtd_006.bin -t serial - - - name: Run Tests - Run migration test case 007 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_007.bin -s Bin/migtd_no.bin -t serial - - - name: Run Tests - Run migration test case 008 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_008.bin -s Bin/migtd_no.bin -t serial - - - name: Run Tests - Run migration test case 009 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_009.bin -s Bin/migtd_009.bin -t serial - - - name: Run Tests - Run migration test case 010 + - name: Run Tests run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_010.bin -s Bin/migtd_010.bin -t serial + pushd sh_script/test + sudo pytest -k "not cycle" --device_type serial + popd - - name: Run Tests - Run migration test case 011 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_011.bin -s Bin/migtd_011.bin -t serial - - - name: Run Tests - Run migration test case 012 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_012.bin -s Bin/migtd_src_012.bin -t serial - - - name: Run Tests - Run migration test case 013 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_013.bin -s Bin/migtd_src_013.bin -t serial - - - name: Run Tests - Run migration test case 014 - run: | - sudo bash sh_script/integration_tdx.sh -d Bin/migtd_dst_014.bin -s Bin/migtd_src_014.bin -t serial \ No newline at end of file + \ No newline at end of file diff --git a/doc/integration_test.md b/doc/integration_test.md new file mode 100644 index 00000000..2b5e5886 --- /dev/null +++ b/doc/integration_test.md @@ -0,0 +1,71 @@ +# Migration TD Integration Test +## Preparation +Download source code & test script: +``` +git clone https://github.com/intel/MigTD.git +git submodule update --init --recursive +./sh_script/preparation.sh +``` +## Setup pytest environment +Please use recommend configuration in [integration_test.py](../sh_script/test/integration_test.py). + +## Config test configuration file +``` +cd sh_script/test +``` +Config [test configration file](../sh_script/test/conf/pyproject.toml), for example: +``` +[migtd.config] +qemu="/usr/local/bin/qemu-system-x86_64" +mig_td_script = "mig-td.sh" +user_td_script = "user-td.sh" +connect_script = "connect.sh" +pre_mig_script = "pre-mig.sh" +user_td_bios_img = "/home/env/OVMF.fd" +kernel_img = "/home/env/bzImage" +guest_img = "/home/env/guest.img" +stress_test_cycles = 1 +``` +## Build & Test +### Build Migration TD binary - Vsock +``` +cargo image +``` +### Run Test +Set stress_test_cycles to 1 in configration file. +``` +pushd sh_script/test +sudo pytest -k "cycle" +popd +``` +### Build Migration TD Test binaries - Vsock +``` +bash sh_script/build_final.sh -t test -c -a on +``` +### Run Test +``` +pushd sh_script/test +sudo pytest -k "not cycle" +popd +``` +### Build Migration TD binary - Serial +``` +cargo image --no-default-features --features remote-attestation,stack-guard,virtio-serial +``` +### Run Test +Set stress_test_cycles to 1 in configration file. +``` +pushd sh_script/test +sudo pytest -k "cycle" --device_type serial +popd +``` +### Build Migration TD Test binaries - Serial +``` +bash sh_script/build_final.sh -t test -c -a on -d serial +``` +### Run Test +``` +pushd sh_script/test +sudo pytest -k "not cycle" --device_type serial +popd +``` diff --git a/sh_script/integration_tdx.sh b/sh_script/integration_tdx.sh deleted file mode 100644 index 3d886015..00000000 --- a/sh_script/integration_tdx.sh +++ /dev/null @@ -1,584 +0,0 @@ -#!/bin/bash -# -# Copyright (c) 2022 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -script_path=$(dirname "$0") -temp_dir=$(mktemp -d) -nohup_logfile="${temp_dir}/nohup.log" - -guest_image="/home/env/guest.img" -kernel="/home/env/bzImage" -qemu_tdx_path="/usr/local/bin/qemu-system-x86_64" - -qmp_sock_src="/tmp/qmp-sock-src" -qmp_sock_dst="/tmp/qmp-sock-dst" -tdvf_image="/home/env/OVMF.fd" - -mig_src_log="migtd-src.log" -mig_dst_log="migtd-dst.log" - -src_migtd="" -dst_migtd="" -payload="" - -cycle=0 - -# Test Configuration Info -cpus=1 -memory=32M -device=vsock - -# Function test case list -func_tcs_list=( - *001* - *002* - *004* - *005* - *010* - *011* - *012* - *013* -) - -# trap cleanup exit - -# MigTD start command -migtd_qemu_cmd="${qemu_tdx_path} \ - -accel kvm \ - -M q35 \ - -cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ - -smp ${cpus},threads=1,sockets=${cpus} \ - -m ${memory} \ - -object tdx-guest,id=tdx0,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ - -object memory-backend-memfd-private,id=ram1,size=${memory} \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -no-hpet \ - -nographic -vga none -nic none" - -usage() { - cat << EOM -Usage: $(basename "$0") [OPTION]... - -s required for pre migration test. - -d required for pre migration test. - -f required for test td payload test. - -p . - -i by default is td-guest.raw. - -k by default is vmlinuz. - -c by default is 1. - -m by defalt is 2G. - -n Cycle test number. - -t Device(transport) used for host-guest communication, by default is `vsock`. - -h Show help info -EOM - exit 0 -} - -proccess_args() { - while getopts ":f:i:p:k:s:t:c:m:n:d:h" option; do - case "${option}" in - i) guest_image=${OPTARG};; - p) qemu_tdx_path=${OPTARG};; - k) kernel=${OPTARG};; - f) payload=${OPTARG};; - s) src_migtd=${OPTARG};; - d) dst_migtd=${OPTARG};; - c) cpus=${OPTARG};; - m) memory=${OPTARG};; - n) cycle=${OPTARG};; - t) device=${OPTARG};; - h) usage;; - esac - done - - if [[ ${payload} == "" ]]; then - if [[ -z ${src_migtd} ]]; then - die "Please input correct source migtd binary path" - fi - [ -e ${src_migtd} ] || die "Source migtd binary path: ${src_migtd} is not exists" - - if [[ -z ${dst_migtd} ]]; then - die "Please input correct destination migtd binary path" - fi - [ -e ${dst_migtd} ] || die "Destination migtd binary path: ${dst_migtd} is not exists" - else - if [[ -z ${payload} ]]; then - die "Please input correct test td payload binary path" - fi - [ -e ${payload} ] || die "Test td payload path: ${payload} is not exists" - fi - - [ -e ${qemu_tdx_path} ] || die "TDX QEMU path: ${qemu_tdx_path} is not exists" - - echo "=========================================" - echo "Source migtd binary : ${src_migtd}" - echo "Destination migtd binary : ${dst_migtd}" - echo "Test td pyalod binary : ${payload}" - echo "Guest Image : ${guest_image}" - echo "Kernel binary : ${kernel}" - echo "CPUs : ${cpus}" - echo "Memmory Size : ${memory}" - echo "Device type : ${device}" - echo "=========================================" -} - -cleanup() { - rm -rf ${tmp_dir} - rm -rf ${mig_src_log} ${mig_dst_log} - - sudo dmesg --clear - - kill_qemu && kill_socat - echo "=========================================" -} - -kill_qemu() { - kill_mig_td - kill_user_td -} - -kill_mig_td() { - # kill proccess with vsock port - while true - do - exist_migtd=`ps aux | grep vsock:1:4050 | grep -v grep` - if [[ $exist_migtd == "" ]] - then - echo "All migtd proccesses have been killed" - break - else - ps aux | grep vsock:1:4050 | grep -v grep | awk -F ' ' '{print $2}' | xargs kill -9 - fi - done -} - -kill_user_td() { - while true - do - exist_srctd=`ps aux | grep lm_src_ci | grep -v grep` - if [[ $exist_srctd == "" ]] - then - echo "Source td proccess has been killed" - break - else - ps aux | grep lm_src_ci | grep -v grep | awk -F ' ' '{print $2}' | xargs kill -9 - fi - done - - while true - do - exist_dsttd=`ps aux | grep lm_dst_ci | grep -v grep` - if [[ $exist_dsttd == "" ]] - then - echo "Destination td proccess has been killed" - break - else - ps aux | grep lm_dst_ci | grep -v grep | awk -F ' ' '{print $2}' | xargs kill -9 - fi - done -} - -kill_socat() { - while true - do - exist_socat=`ps aux | grep socat | grep -v grep` - if [[ $exist_socat == "" ]] - then - echo "All socat proccesses have been killed" - break - else - ps aux | grep socat | grep -v grep | awk -F ' ' '{print $2}' | xargs kill -9 - fi - done -} - -die() { - echo "ERROR: $*" >&2 - exit 1 -} - -is_function_tcs() { - result=1 - for tcs_no in ${func_tcs_list[@]}; do - if [[ ${dst_migtd} == *${tcs_no}* ]] - then - result=0 - break - fi - done - - if [[ ${dst_migtd} == *migtd.bin* ]] - then - result=0 - fi - - return ${result} -} - -check_result() { - time=0 - result=1 - while ((${time}<=$3)) - do - sleep 1 - if [[ `grep -c "$2" "$1"` -ne 0 ]] - then - result=0 - break - fi - let "time++" - done - return ${result} -} - -check_migration_result() { - time=0 - result=1 - while ((${time}<=$2)) - do - sleep 1 - if [[ "`dmesg | tail -2 | grep "$1"`" != "" ]] - then - result=0 - break - fi - let "time++" - done - return ${result} -} - -install_qemu_tdx() { - echo "-- Install QEMU" - dnf update qemu --allowerasing -} - -setup_agent() { - time_out=30 - while ((${time_out}>0)) - do - socat TCP4-LISTEN:9001,reuseaddr VSOCK-LISTEN:1234,fork & - socat_pnum=`ps aux | grep socat | grep -v grep | wc -l` - if [[ $socat_pnum == 1 ]] - then - break - else - let "time_out--" - continue - fi - done - - [ $time_out -ne 0 ] || die "Faild to launch socat" - - time_out=30 - while ((${time_out}>0)) - do - socat TCP4-CONNECT:127.0.0.1:9001,reuseaddr VSOCK-LISTEN:1235,fork & - socat_pnum=`ps aux | grep socat | grep -v grep | wc -l` - if [[ $socat_pnum == 2 ]] - then - break - else - let "time_out--" - continue - fi - done - - [ $time_out -ne 0 ] || die "Faild to launch socat" -} - -launch_src_migtd() { - local cmd="${migtd_qemu_cmd} \ - -bios ${src_migtd} \ - -name migtd-src,process=migtd-src-ci,debug-threads=on \ - -serial mon:stdio" - - if [[ ${device} == serial ]] - then - cmd="${cmd} \ - -device virtio-serial-pci,id=virtio-serial0 \ - -chardev socket,host=127.0.0.1,port=1234,server=off,id=foo \ - -device virtserialport,chardev=foo,bus=virtio-serial0.0" - elif [[ ${device} == vsock ]] - then - cmd="${cmd} \ - -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=18,disable-legacy=on" - fi - - nohup ${cmd} > ${mig_src_log} & - - sleep 1 -} - -launch_dst_migtd() { - local cmd="${migtd_qemu_cmd} \ - -bios ${dst_migtd} \ - -name migtd-dst,process=migtd-dst-ci,debug-threads=on \ - -serial mon:stdio" - - if [[ ${device} == serial ]] - then - cmd="${cmd} \ - -device virtio-serial-pci,id=virtio-serial0 \ - -chardev socket,host=127.0.0.1,port=1234,server=on,id=foo \ - -device virtserialport,chardev=foo,bus=virtio-serial0.0" - elif [[ ${device} == vsock ]] - then - cmd="${cmd} \ - -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=36,disable-legacy=on" - fi - - nohup ${cmd} > ${mig_dst_log} & - - sleep 1 -} - -launch_src_migtd_without_device() { - local cmd="${migtd_qemu_cmd} \ - -bios ${src_migtd} \ - -name migtd-src,process=migtd-src-ci,debug-threads=on \ - -serial mon:stdio" - - # Connect to dst MigTD to make it run - if [[ ${device} == serial ]] - then - cmd="${cmd} \ - -chardev socket,host=127.0.0.1,port=1234,server=off,id=foo" - fi - - nohup ${cmd} > ${mig_src_log} & - - sleep 1 -} - -launch_src_td() { - local MIGTD_PID=$(pgrep migtd-src-ci) - time_out=30 - while ((${time_out}>0)) - do - nohup ${qemu_tdx_path} -accel kvm \ - -cpu host,host-phys-bits,pmu=off,-kvm-steal-time,-kvmclock,-kvm-asyncpf,-kvmclock-stable-bit,tsc-freq=1000000000 \ - -smp 1 \ - -m 2G \ - -object tdx-guest,id=tdx0,sept-ve-disable=on,debug=off,migtd-pid=${MIGTD_PID} \ - -object memory-backend-memfd-private,id=ram1,size=2G \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -bios ${tdvf_image} \ - -chardev stdio,id=mux,mux=on \ - -device virtio-serial,romfile= \ - -device virtconsole,chardev=mux -serial chardev:mux -monitor chardev:mux \ - -drive file=${guest_image},if=virtio,id=virtio-disk0,format=raw \ - -kernel ${kernel} \ - -append "root=/dev/vda1 rw console=hvc0" \ - -name process=lm_src_ci,debug-threads=on \ - -no-hpet -nodefaults \ - -D qemu_src.log -nographic -vga none \ - -monitor unix:${qmp_sock_src},server,nowait & - - sleep 1 - exist_srctd=`ps aux | grep lm_src_ci | grep -v grep` - if [[ $exist_srctd == "" ]] - then - let "time_out--" - continue - else - break - fi - done - - [ $time_out -ne 0 ] || die "Faild to launch source TD" -} - -launch_dst_td() { - local MIGTD_PID=$(pgrep migtd-dst-ci) - time_out=30 - while ((${time_out}>0)) - do - nohup ${qemu_tdx_path} -accel kvm \ - -cpu host,host-phys-bits,pmu=off,-kvm-steal-time,-kvmclock,-kvm-asyncpf,-kvmclock-stable-bit,tsc-freq=1000000000 \ - -smp 1 \ - -m 2G \ - -object tdx-guest,id=tdx0,sept-ve-disable=on,debug=off,migtd-pid=${MIGTD_PID} \ - -object memory-backend-memfd-private,id=ram1,size=2G \ - -machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ - -bios ${tdvf_image} \ - -chardev stdio,id=mux,mux=on \ - -device virtio-serial,romfile= \ - -device virtconsole,chardev=mux -serial chardev:mux -monitor chardev:mux \ - -drive file=${guest_image},if=virtio,id=virtio-disk0,format=raw \ - -name process=lm_dst_ci,debug-threads=on \ - -no-hpet -nodefaults \ - -D qemu_dst.log -nographic -vga none \ - -monitor unix:${qmp_sock_dst},server,nowait \ - -incoming tcp:0:6666 & - - sleep 1 - exist_dsttd=`ps aux | grep lm_dst_ci | grep -v grep` - if [[ $exist_dsttd == "" ]] - then - let "time_out--" - continue - else - break - fi - done - - [ $time_out -ne 0 ] || die "Faild to launch destination TD" -} - -send_mig_command() { - # Asking migtd-src to connect to the src socat - echo "qom-set /objects/tdx0/ vsockport 1234" | nc -U /tmp/qmp-sock-src - - # Asking migtd-dst to connect to the dst socat - echo "qom-set /objects/tdx0/ vsockport 1235" | nc -U /tmp/qmp-sock-dst -} - -test_migtd() { - echo "-- start test migration td" - local time_out=30 - - if [[ ${device} == vsock ]] - then - echo "-- setup agent" - setup_agent - fi - - echo "-- launch dst migtd" - launch_dst_migtd - - echo "-- launch src migtd" - if [[ ${src_migtd} == *009* ]] - then - launch_src_migtd_without_device - else - launch_src_migtd - fi - - echo "-- launch src td" - launch_src_td - echo "-- launch dst td" - launch_dst_td - echo "-- send migration command" - send_mig_command - - if is_function_tcs - then - # TBD need to update to avoid no run scenario - local key_str="Pre-migration is done" - else - # TBD need to update to avoid no run scenario - local key_str="pre-migration failed" - fi - - check_migration_result "${key_str}" ${time_out} - if [[ $? -eq 0 ]] - then - kill_qemu - echo "-- migration td test: Pass" - else - kill_qemu - echo "-- migration td test: Fail" && exit 1 - fi -} - -cycle_test_migtd() { - echo "-- start test migration td" - local time_out=30 - - if [[ ${device} == vsock ]] - then - echo "-- setup agent" - setup_agent - fi - echo "-- launch dst migtd" - launch_dst_migtd - echo "-- launch src migtd" - launch_src_migtd - - for ((i=1;i<${cycle};i++)) - do - echo "#########################" - echo "Cycling test ${i}" - echo "#########################" - echo "-- launch src td" - launch_src_td - - echo "-- launch dst td" - launch_dst_td - - echo "-- send migration command" - send_mig_command - - local key_str="Pre-migration is done" - - check_migration_result "${key_str}" ${time_out} - if [[ $? -eq 0 ]] - then - kill_user_td - echo "-- migration td test: Pass" - else - kill_qemu - echo "-- migration td test: Fail" && exit 1 - fi - sudo dmesg --clear - done - echo "-- migration td cycling test: Pass" - kill_qemu -} - -launch_td_test_payload() { - echo "-- launch td test payload" - local time_out=20 - local key_str="0 failed" - - nohup ${qemu_tdx_path} -accel kvm \ - -M q35 \ - -name process=tdxvm \ - -smp ${cpus}\ - -object tdx-guest,id=tdx,quote-generation-service=vsock:1:4050,sept-ve-disable=off,debug=off \ - -machine q35,kernel_irqchip=split,confidential-guest-support=tdx \ - -no-hpet \ - -cpu host,host-phys-bits,-kvm-steal-time,pmu=off,-amx-tile,-amx-int8 \ - -bios ${payload} \ - -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=37,disable-legacy=on \ - -m ${memory} -nographic -vga none -nic none \ - -serial mon:stdio > ${nohup_logfile} 2>&1 & - - check_result ${nohup_logfile} "${key_str}" ${time_out} - - if [[ $? -eq 0 ]] - then - kill_qemu - cat ${nohup_logfile} && echo "-- launch td test payload: Pass" - else - kill_qemu - cat ${nohup_logfile} && echo "-- launch td test payload: Fail" && exit 1 - fi -} - -run_test() { - echo "=========================================" - echo " Run Test " - echo "=========================================" - if [[ ${payload} == "" ]] - then - if [[ ${cycle} -eq 0 ]] - then - test_migtd - else - cycle_test_migtd - fi - else - launch_td_test_payload - fi -} - -main() { - cleanup - run_test -} - -proccess_args $@ -main diff --git a/sh_script/test/__init__.py b/sh_script/test/__init__.py new file mode 100644 index 00000000..6605c685 --- /dev/null +++ b/sh_script/test/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 \ No newline at end of file diff --git a/sh_script/test/conf/pyproject.toml b/sh_script/test/conf/pyproject.toml new file mode 100644 index 00000000..8ff4a6eb --- /dev/null +++ b/sh_script/test/conf/pyproject.toml @@ -0,0 +1,10 @@ +[migtd.config] +qemu="/usr/local/bin/qemu-system-x86_64" +mig_td_script = "mig-td.sh" +user_td_script = "user-td.sh" +connect_script = "connect.sh" +pre_mig_script = "pre-mig.sh" +user_td_bios_img = "/home/env/OVMF.fd" +kernel_img = "/home/env/bzImage" +guest_img = "/home/env/guest.img" +stress_test_cycles = 20 diff --git a/sh_script/test/conftest.py b/sh_script/test/conftest.py new file mode 100644 index 00000000..c94d3ecd --- /dev/null +++ b/sh_script/test/conftest.py @@ -0,0 +1,250 @@ +# Copyright (c) 2022 - 2023 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import time +import logging +import subprocess +import threading +from contextlib import contextmanager +from typing import Dict + +import psutil +import tomli + +LOG = logging.getLogger(__name__) + + +def pytest_addoption(parser): + parser.addoption("--device_type", action="store", default="vsock", help="Device type vsock/serial") + +@pytest.fixture() +def device_type(request): + return request.config.getoption("--device_type") + +@contextmanager +def migtd_context(): + """ + Create a MigTD instance with default user id. Cleanup when out of scope. + + In most cases that are not necessary to run multiple pairs of TDs at the same time, use this method with `with` statement. + """ + tool = MigtdTest() + try: + yield tool + finally: + tool.cleanup() + + +class MigtdTest: + log_dir_name = f"log_{int(time.time())}" + + def __init__(self): + # variable from the config file + self.qemu: str = None + self.mig_td_script: str = None + self.user_td_script: str = None + self.connect_script: str = None + self.pre_mig_script: str = None + self.user_td_bios_img: str = None + self.kernel_img: str = None + self.guest_img: str = None + self.stress_test_cycles: str = None + + cfg = self._parse_toml_config() + for k, v in cfg.items(): + setattr(self, k, v) + + # other variables to init + self._threads: Dict[str, threading.Thread] = {} + self._host_procs: Dict[str, subprocess.Popen] = {} + self.ssh = None + # clear dmesg + self.clear_dmesg() + # terminate all tds & socat before test execution + self.cleanup() + + def _parse_toml_config(self) -> dict: + with open("conf/pyproject.toml", "rb") as f: + cfg = tomli.load(f) + return cfg["migtd"]["config"] + + def _try_terminate_procs(self, popen: subprocess.Popen): + if not popen: + return + try: + root = psutil.Process(popen.pid) + root_cmd = " ".join(root.cmdline()) + childs = root.children(recursive=True) + LOG.debug( + f"Terminate subprocess {list(map(lambda p: p.cmdline()[0], childs))} launched by '{root_cmd}'" + ) + except psutil.NoSuchProcess: + LOG.debug("Failed to terminate subprocess: NoSuchProcess") + return + for child in childs: + child.terminate() + root.terminate() + + def _wait_td_threads(self): + for _, thread in self._threads.items(): + thread.join() + self._threads = {} + + def _exec_shell_cmd(self, command: str, tag: str = None) -> tuple: + LOG.debug(f"{command}\n") + proc = subprocess.Popen( + command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if tag: + self._host_procs[tag] = proc + output, error = proc.communicate() + stdout, stderr = output.decode().strip(), error.decode().strip() + + LOG.debug(f"{command}\n{stdout}\n") + + return (stdout, stderr) + + def start_test_payload(self, bios_img=None, type=None, device=None): + """ + Start migration td + """ + LOG.debug(f"Starting launch test td payload") + arg=f"sudo bash {self.mig_td_script} -q {self.qemu} -m {bios_img} -t {type}" + + if device == "serial": + arg=arg + " -s" + + thread_migtd =threading.Thread( + target=self._exec_shell_cmd, + args=(arg, + f"mig_td_{type}") + ) + + thread_migtd.start() + + self._threads[f"mig_td_{type}"] = thread_migtd + time.sleep(1) + + # type has src dst + def start_mig_td(self, bios_img=None, type=None, no_device=False, device=None): + """ + Start migration td + """ + arg=f"sudo bash {self.mig_td_script} -q {self.qemu} -m {bios_img} -t {type}" + if no_device: + arg=arg + " -n" + if device == "serial": + arg=arg + " -s" + + thread_migtd =threading.Thread( + target=self._exec_shell_cmd, + args=(arg, + f"mig_td_{type}") + ) + + LOG.debug(f"Starting {type} migration td") + thread_migtd.start() + self._threads[f"mig_td_{type}"] = thread_migtd + time.sleep(1) + + # type has src dst + def start_user_td(self, type=None): + """ + Start user td + """ + thread_usertd = threading.Thread( + target=self._exec_shell_cmd, + args=(f"sudo bash {self.user_td_script} -q {self.qemu} -i {self.guest_img} -k {self.kernel_img} -o {self.user_td_bios_img} -t {type}", + f"user_td_{type}") + ) + + LOG.debug(f"Starting {type} user td") + thread_usertd.start() + self._threads[f"user_td_{type}"] = thread_usertd + time.sleep(1) + + def connect(self): + """ + Create a channel for MigTD_src and MigTD_dst + """ + thread_connect =threading.Thread( + target=self._exec_shell_cmd, + args=(f"sudo bash {self.connect_script}", + "connect") + ) + + LOG.debug(f"Create a channel for MigTD_src and MigTD_dst") + thread_connect.start() + self._threads["connect"] = thread_connect + + def pre_migration(self): + """ + Execute pre migration + """ + LOG.debug(f"Start pre migration") + self._exec_shell_cmd(f"sudo bash {self.pre_mig_script}") + + def check_migration_result(self, negative=False, wait_time=2): + """ + Check pre-migration result + """ + time.sleep(wait_time) + runner = self._exec_shell_cmd("dmesg | tail -2") + if runner[0] == "": + time.sleep(3) + runner = self._exec_shell_cmd("dmesg | tail -2") + + LOG.debug(runner[0]) + if negative: + assert "pre-migration failed" in runner[0] or "Pre-migration is done" not in runner[0], "Negative: Not found 'Pre-migration failed'" + else: + assert "Pre-migration is done" in runner[0], "Function: Pre-migration failed" + + def terminate_mig_td(self, type=None): + """ + Terminate the migration td qemu process with SIGTERM alone. + """ + root_proc = self._host_procs.get(f"mig_td_{type}") + self._try_terminate_procs(root_proc) + + def terminate_user_td(self, type=None): + """ + Terminate the user td qemu process with SIGTERM alone. + """ + root_proc = self._host_procs.get(f"user_td_{type}") + self._try_terminate_procs(root_proc) + + def terminate_socat(self, type=None): + """ + Terminate the migration td qemu process with SIGTERM alone. + """ + self._exec_shell_cmd("ps aux | grep socat | grep -v grep | awk -F ' ' '{print $2}' | xargs sudo kill -9") + + def terminate_all_tds(self): + """ + Terminate all td qemu processes with SIGTERM. + """ + if len(self._host_procs) == 0: + return + self.terminate_user_td(type="src") + self.terminate_user_td(type="dst") + self.terminate_mig_td(type="src") + self.terminate_mig_td(type="dst") + self._host_procs = {} + # Avoid terminate failed and cleanup existing migtd related td before running pytest + self._exec_shell_cmd("ps aux | grep migtd | grep -v grep | awk -F ' ' '{print $2}' | xargs sudo kill -9") + + def clear_dmesg(self): + self._exec_shell_cmd("sudo dmesg --clear") + + def cleanup(self): + """ + Clear storage of threads and processes. And make sure all qemu procs have been terminated. + """ + self.terminate_socat() + self.terminate_all_tds() + self._wait_td_threads() + self._threads = {} + self._host_procs = {} \ No newline at end of file diff --git a/sh_script/test/connect.sh b/sh_script/test/connect.sh new file mode 100644 index 00000000..42e6db7f --- /dev/null +++ b/sh_script/test/connect.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +TYPE="local" +DEST_IP="" + +usage() { + cat << EOM +Usage: $(basename "$0") [OPTION]... + -i Destination platform ip address + -t Use single or cross host live migration + -h Show this help +EOM +} + +process_args() { + while getopts "i:t:h" option; do + case "${option}" in + i) DEST_IP=$OPTARG;; + t) TYPE=$OPTARG;; + h) usage + exit 0 + ;; + *) + echo "Invalid option '-$OPTARG'" + usage + exit 1 + ;; + esac + done + + case ${TYPE} in + "local");; + "remote") + if [[ -z ${DEST_IP} ]]; then + error "Please use -i specify DEST_IP in remote type" + fi + ;; + *) + error "Invalid ${TYPE}, must be [local|remote]" + ;; + esac +} + +error() { + echo -e "\e[1;31mERROR: $*\e[0;0m" + exit 1 +} + +connect() { + modprobe vhost_vsock + if [[ ${TYPE} == "local" ]]; then + socat TCP4-LISTEN:9009,reuseaddr VSOCK-LISTEN:1235,fork & + socat TCP4-CONNECT:127.0.0.1:9009,reuseaddr VSOCK-LISTEN:1234,fork & + else + ssh root@"${DEST_IP}" -o ConnectTimeout=30 "modprobe vhost_vsock; nohup socat TCP4-LISTEN:9009,reuseaddr VSOCK-LISTEN:1235,fork > foo.out 2> foo.err < /dev/null &" + sleep 3 + socat TCP4-CONNECT:"${DEST_IP}":9009,reuseaddr VSOCK-LISTEN:1234,fork & + fi +} + +process_args "$@" +connect \ No newline at end of file diff --git a/sh_script/test/integration_test.py b/sh_script/test/integration_test.py new file mode 100644 index 00000000..fd7ae124 --- /dev/null +++ b/sh_script/test/integration_test.py @@ -0,0 +1,335 @@ +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + +""" +integration_test.py + +Readme: +-------------- +A migtd test framework to help to test pre-migration and check dump logs. + +- Please edit migtd.config variables in `pyproject.toml` before use. +- Please note running pytest requires sudo permission without being prompted to input a password. + +Example: +Recommend to use python 3.10 +-------------- +$ python3 -m venv .venv +$ source .venv/bin/activate +$ (.venv) pip install pytest psutil pytest-html +$ (.venv) vim conf/pyproject.toml +$ (.venv) pytest + +""" +import logging + +from .conftest import migtd_context + +LOG = logging.getLogger(__name__) + +""" +Test Secure boot +""" +def test_function_001(device_type): + migtd_src = "../../Bin/migtd_001.bin" + migtd_dst = "../../Bin/migtd_001.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +RTMR1 of src and dst are not equal, but RTMR1 is not in policy file - SVN +""" +def test_function_002(device_type): + migtd_src = "../../Bin/migtd_002.bin" + migtd_dst = "../../Bin/migtd_001.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +RTMR1 of src and dst are not equal and RTMR1 is in policy file +""" +def test_negative_003(device_type): + migtd_src = "../../Bin/migtd_003.bin" + migtd_dst = "../../Bin/migtd_001.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and svn(13) in range(13..18) of policy +""" +def test_function_004(device_type): + migtd_src = "../../Bin/migtd_004.bin" + migtd_dst = "../../Bin/migtd_004.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and svn(17) in range(13..18) of policy +""" +def test_function_005(device_type): + migtd_src = "../../Bin/migtd_005.bin" + migtd_dst = "../../Bin/migtd_005.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and svn(18) out of range(13..18) +""" +def test_negative_006(device_type): + migtd_src = "../../Bin/migtd_006.bin" + migtd_dst = "../../Bin/migtd_006.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Different policy file and check "Digest.MigTdPolicy" +""" +def test_negative_007(device_type): + migtd_src = "../../Bin/migtd_007.bin" + migtd_dst = "../../Bin/migtd_no.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Invalid json +""" +def test_negative_008(device_type): + migtd_src = "../../Bin/migtd_008.bin" + migtd_dst = "../../Bin/migtd_no.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Test without vsock device init +""" +def test_negative_009(device_type): + migtd_src = "../../Bin/migtd_009.bin" + migtd_dst = "../../Bin/migtd_009.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_mig_td(bios_img=migtd_src, type="src", no_device=True, device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True, wait_time=10) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +def test_function_010(device_type): + migtd_src = "../../Bin/migtd_010.bin" + migtd_dst = "../../Bin/migtd_010.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +def test_function_011(device_type): + migtd_src = "../../Bin/migtd_011.bin" + migtd_dst = "../../Bin/migtd_011.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and dst svn(2) greater than src svn(1) +""" +def test_function_012(device_type): + migtd_src = "../../Bin/migtd_src_012.bin" + migtd_dst = "../../Bin/migtd_dst_012.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and dst svn(1) equal than src svn(1) +""" +def test_function_013(device_type): + migtd_src = "../../Bin/migtd_src_013.bin" + migtd_dst = "../../Bin/migtd_dst_013.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result() + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Migration Policy Check: +Secure boot and dst svn(1) smaller than src svn(2) +""" +def test_negative_014(device_type): + migtd_src = "../../Bin/migtd_src_014.bin" + migtd_dst = "../../Bin/migtd_dst_014.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.connect() + ctx.pre_migration() + ctx.check_migration_result(negative=True) + + ctx.terminate_all_tds() + ctx.terminate_socat() + +""" +Test TD payload: +- MSR RW +- MMIO RW +- Quote Sevice Query +- Quote Attestation +""" +def test_function_015(device_type): + test_bin = "../../Bin/final-test.bin" + + with migtd_context() as ctx: + ctx.start_test_payload(bios_img=test_bin, type="src", device=device_type) + ctx.terminate_all_tds() + +def test_cycle(device_type): + migtd_src = "../../target/release/migtd.bin" + migtd_dst = "../../target/release/migtd.bin" + + with migtd_context() as ctx: + ctx.start_mig_td(bios_img=migtd_src, type="src", device=device_type) + ctx.start_mig_td(bios_img=migtd_dst, type="dst", device=device_type) + ctx.connect() + + for i in range(ctx.stress_test_cycles): + LOG.debug(f"#### Cycle Test: {i} ####") + ctx.start_user_td(type="src") + ctx.start_user_td(type="dst") + ctx.pre_migration() + ctx.check_migration_result() + ctx.terminate_user_td(type="src") + ctx.terminate_user_td(type="dst") + + ctx.terminate_mig_td() + ctx.terminate_socat() \ No newline at end of file diff --git a/sh_script/test/mig-td.sh b/sh_script/test/mig-td.sh new file mode 100644 index 00000000..df3f6166 --- /dev/null +++ b/sh_script/test/mig-td.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +QEMU_EXEC="/usr/local/bin/qemu-system-x86_64" +GUEST_CID=18 +MIGTD="/usr/share/td-migration/migtd.bin" +VIRTIO_SERIAL=false +MIGTD_TYPE="" +DEST_IP="127.0.0.1" + +usage() { + cat << EOM +Usage: $(basename "$0") [OPTION]... + -q QEMU path + -m MigTD file + -t Must set migtd type, src or dst + -h Show this help +EOM +} + +process_args() { + while getopts "i:m:t:q:hsn" option; do + case "${option}" in + i) DEST_IP=$OPTARG;; + m) MIGTD=$OPTARG;; + t) MIGTD_TYPE=$OPTARG;; + s) VIRTIO_SERIAL=true;; + n) NO_DEVICE=true;; + q) QEMU_EXEC=$OPTARG;; + h) usage + exit 0 + ;; + *) + echo "Invalid option '-$OPTARG'" + usage + exit 1 + ;; + esac + done + + if [[ -z ${MIGTD_TYPE} ]]; then + usage + error "Must set MIGTD_TYPE -t [src|dst]" + fi + + + case ${MIGTD_TYPE} in + "src") + GUEST_CID=18 + ;; + "dst") + GUEST_CID=36 + ;; + *) + error "Invalid ${MIGTD_TYPE}, must be [src|dst]" + ;; + esac +} + +error() { + echo -e "\e[1;31mERROR: $*\e[0;0m" + exit 1 +} + +launch_migTD() { +QEMU_CMD="${QEMU_EXEC} \ +-accel kvm \ +-M q35 \ +-cpu host,host-phys-bits,-kvm-steal-time,pmu=off \ +-smp 1,threads=1,sockets=1 \ +-m 32M \ +-object tdx-guest,id=tdx0,sept-ve-disable=off,debug=off,quote-generation-service=vsock:1:4050 \ +-object memory-backend-memfd-private,id=ram1,size=32M \ +-machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ +-bios ${MIGTD} \ +-name migtd-${MIGTD_TYPE},process=migtd-${MIGTD_TYPE},debug-threads=on \ +-no-hpet \ +-nographic -vga none -nic none \ +-serial mon:stdio \ +-pidfile /var/run/migtd-${MIGTD_TYPE}.pid" + if [[ $NO_DEVICE != true ]]; then + if [[ ${VIRTIO_SERIAL} == true ]]; then + QEMU_CMD+=" -device virtio-serial-pci,id=virtio-serial0 " + if [[ ${MIGTD_TYPE} == "src" ]]; then + QEMU_CMD+=" -chardev socket,host=0.0.0.0,port=1236,server=on,id=foo " + else + QEMU_CMD+=" -chardev socket,host=${DEST_IP},port=1236,server=off,id=foo " + fi + QEMU_CMD+=" -device virtserialport,chardev=foo,bus=virtio-serial0.0 " + else + QEMU_CMD+=" -device vhost-vsock-pci,id=vhost-vsock-pci1,guest-cid=${GUEST_CID},disable-legacy=on " + fi + fi + + eval "${QEMU_CMD}" +} + +process_args "$@" +launch_migTD \ No newline at end of file diff --git a/sh_script/test/pre-mig.sh b/sh_script/test/pre-mig.sh new file mode 100644 index 00000000..be64d6fd --- /dev/null +++ b/sh_script/test/pre-mig.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +TYPE="local" +DEST_IP="" + +usage() { + cat << EOM +Usage: $(basename "$0") [OPTION]... + -i Destination platform ip address + -t Use single or cross host live migration + -h Show this help +EOM +} + +process_args() { + while getopts "i:t:h" option; do + case "${option}" in + i) DEST_IP=$OPTARG;; + t) TYPE=$OPTARG;; + h) usage + exit 0 + ;; + *) + echo "Invalid option '-$OPTARG'" + usage + exit 1 + ;; + esac + done + + case ${TYPE} in + "local");; + "remote") + if [[ -z ${DEST_IP} ]]; then + error "Please use -i specify DEST_IP in remote type" + fi + ;; + *) + error "Invalid ${TYPE}, must be [local|remote]" + ;; + esac +} + +error() { + echo -e "\e[1;31mERROR: $*\e[0;0m" + exit 1 +} + +pre_mig(){ + # Asking migtd-dst to connect to the dst socat + if [[ ${TYPE} == "local" ]]; then + echo "qom-set /objects/tdx0/ vsockport 1235" | nc -U /tmp/qmp-sock-dst + else + ssh root@"${DEST_IP}" -o ConnectTimeout=30 "echo qom-set /objects/tdx0/ vsockport 1235 | nc -U /tmp/qmp-sock-dst" + fi + + # Asking migtd-dst to connect to the src socat + echo "qom-set /objects/tdx0/ vsockport 1234" | nc -U /tmp/qmp-sock-src +} + +process_args "$@" +pre_mig \ No newline at end of file diff --git a/sh_script/test/pytest.ini b/sh_script/test/pytest.ini new file mode 100644 index 00000000..8ce78731 --- /dev/null +++ b/sh_script/test/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts=-v -s -p no:warnings +log_cli = True +log_cli_level = DEBUG +log_cli_format = %(asctime)s [%(levelname)s] | %(filename)s:%(lineno)s | %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/sh_script/test/user-td.sh b/sh_script/test/user-td.sh new file mode 100644 index 00000000..99fe3e99 --- /dev/null +++ b/sh_script/test/user-td.sh @@ -0,0 +1,170 @@ +#!/bin/bash +# +# Copyright (c) 2022 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# +set -e + +QEMU_EXEC="/usr/local/bin/qemu-system-x86_64" +TD_TYPE="" +KERNEL="" +GUEST_IMG="" +OVMF="/home/env/OVMF.fd" +BOOT_TYPE="direct" +TARGET_PID="" +ROOT_PARTITION="/dev/vda3" +QUOTE_TYPE="" +GUEST_CID=3 +TELNET_PORT=9088 +INCOMING_PORT=6666 + + +usage() { + cat << EOM +Usage: $(basename "$0") [OPTION]... + -q QEMU path + -o OVMF image path + -i Guest image file + -k Kernel file + -b [direct|grub] Boot type, default is "direct" which requires kernel binary specified via "-k" + -p incoming port + -q [tdvmcall|vsock] Support for TD quote using tdvmcall or vsock + -r root partition for direct boot, default is /dev/vda1 + -t Must set userTD type, src or dst + -h Show this help +EOM +} + +process_args() { + while getopts "i:k:b:o:p:q:r:t:h" option; do + case "${option}" in + i) GUEST_IMG=$OPTARG;; + k) KERNEL=$OPTARG;; + o) OVMF=$OPTARG;; + b) BOOT_TYPE=$OPTARG;; + p) INCOMING_PORT=$OPTARG;; + q) QEMU_EXEC=$OPTARG;; + a) QUOTE_TYPE=$OPTARG;; + r) ROOT_PARTITION=$OPTARG;; + t) TD_TYPE=$OPTARG;; + h) usage + exit 0 + ;; + *) + echo "Invalid option '-$OPTARG'" + usage + exit 1 + ;; + esac + done + + if [[ -z ${TD_TYPE} ]]; then + usage + error "Must set TD_TYPE -t [src|dst]" + fi + + case ${TD_TYPE} in + "src") + GUEST_CID=3 + TELNET_PORT=9088 + TARGET_PID=$(pgrep migtd-src) + ;; + "dst") + GUEST_CID=4 + TELNET_PORT=9089 + TARGET_PID=$(pgrep migtd-dst) + ;; + *) + error "Invalid ${TD_TYPE}, must be [src|dst]" + ;; + esac + + case ${BOOT_TYPE} in + "direct") + if [[ ! -f ${KERNEL} ]]; then + usage + error "Kernel image file ${KERNEL} not exist. Please specify via option \"-k\"" + fi + ;; + "grub") + ;; + *) + echo "Invalid ${BOOT_TYPE}, must be [direct|grub]" + exit 1 + ;; + esac + + if [[ ! -f ${GUEST_IMG} ]]; then + usage + error "Guest image file ${GUEST_IMG} not exist. Please specify via option \"-i\"" + fi + + if [[ ! -f ${QEMU_EXEC} ]]; then + error "Please install QEMU which supports TDX." + fi + + if [[ ! -f ${OVMF} ]]; then + error "Could not find OVMF, please install first" + fi + + if [[ -n ${QUOTE_TYPE} ]]; then + case ${QUOTE_TYPE} in + "tdvmcall") ;; + "vsock");; + *) + error "Invalid quote type \"$QUOTE_TYPE\", must be [vsock|tdvmcall]" + ;; + esac + fi +} + +error() { + echo -e "\e[1;31mERROR: $*\e[0;0m" + exit 1 +} + +launch_TDVM() { +QEMU_CMD="${QEMU_EXEC} \ +-accel kvm \ +-cpu host,pmu=off,-kvm-steal-time,-shstk,tsc-freq=1000000000 \ +-smp 2,threads=1,sockets=1 \ +-m 8G \ +-object memory-backend-memfd-private,id=ram1,size=8G \ +-machine q35,memory-backend=ram1,confidential-guest-support=tdx0,kernel_irqchip=split \ +-bios ${OVMF} \ +-chardev stdio,id=mux,mux=on,logfile=lm-${TD_TYPE}.log \ +-drive file=$(readlink -f "${GUEST_IMG}"),if=virtio,id=virtio-disk0,format=raw \ +-name process=lm-${TD_TYPE},debug-threads=on \ +-no-hpet -nodefaults \ +-D /run/qemu-${TD_TYPE}.log -nographic -vga none \ +-monitor unix:/tmp/qmp-sock-${TD_TYPE},server,nowait \ +-monitor telnet:127.0.0.1:${TELNET_PORT},server,nowait \ +-device virtio-serial,romfile= \ +-device virtconsole,chardev=mux -serial chardev:mux -monitor chardev:mux" + + if [[ -n ${QUOTE_TYPE} ]]; then + if [[ ${QUOTE_TYPE} == "tdvmcall" ]]; then + QEMU_CMD+=" -object tdx-guest,id=tdx0,sept-ve-disable=on,debug=off,migtd-pid=${TARGET_PID},quote-generation-service=vsock:2:4050" + else + QEMU_CMD+=" -object tdx-guest,id=tdx0,sept-ve-disable=on,debug=off,migtd-pid=${TARGET_PID}" + QEMU_CMD+=" -device vhost-vsock-pci,guest-cid=${GUEST_CID}" + fi + else + QEMU_CMD+=" -object tdx-guest,id=tdx0,sept-ve-disable=on,debug=off,migtd-pid=${TARGET_PID}" + fi + + if [[ ${BOOT_TYPE} == "direct" ]]; then + QEMU_CMD+=" -kernel $(readlink -f "${KERNEL}")" + QEMU_CMD+=" -append \"root=${ROOT_PARTITION} rw console=hvc0\" " + fi + + if [[ ${TD_TYPE} == "dst" ]]; then + QEMU_CMD+=" -incoming tcp:0:${INCOMING_PORT}" + fi + + eval "${QEMU_CMD}" +} + +process_args "$@" +launch_TDVM \ No newline at end of file