implement shadow stacks

kernel/Cargo.toml

@@ -46,6 +46,7 @@ enable-gdb = ["dep:gdbstub", "dep:gdbstub_arch"]
 mstpm = ["dep:libmstpm"]
 nosmep = []
 nosmap = []
+shadow-stacks = []
 
 [dev-dependencies]
 

kernel/src/address.rs

@@ -227,6 +227,11 @@ impl VirtAddr {
         self.0 as *mut T
     }
 
+    #[inline]
+    pub const fn as_usize(&self) -> usize {
+        self.0
+    }
+
     /// Converts the `VirtAddr` to a reference to the given type, checking
     /// that the address is not NULL and properly aligned.
     ///

kernel/src/cpu/extable.rs

@@ -74,7 +74,7 @@ pub fn handle_exception_table(ctx: &mut X86ExceptionContext) -> bool {
     // If an exception hit in an area covered by the exception table, set rcx to -1
     if new_rip != ex_rip {
         ctx.regs.rcx = !0usize;
-        ctx.frame.rip = new_rip.bits();
+        ctx.set_rip(new_rip.bits());
         return true;
     }
 

kernel/src/cpu/idt/common.rs

@@ -11,6 +11,7 @@ use crate::cpu::control_regs::{read_cr0, read_cr4};
 use crate::cpu::efer::read_efer;
 use crate::cpu::gdt::gdt;
 use crate::cpu::registers::{X86GeneralRegs, X86InterruptFrame};
+use crate::cpu::shadow_stack::is_cet_ss_supported;
 use crate::insn_decode::{InsnError, InsnMachineCtx, InsnMachineMem, Register, SegRegister};
 use crate::locking::{RWLock, ReadLockGuard, WriteLockGuard};
 use crate::mm::GuestPtr;
@@ -61,11 +62,32 @@ bitflags::bitflags! {
 #[repr(C, packed)]
 #[derive(Default, Debug, Clone, Copy)]
 pub struct X86ExceptionContext {
+    pub ssp: u64,
+    _padding: [u8; 8],
     pub regs: X86GeneralRegs,
     pub error_code: usize,
     pub frame: X86InterruptFrame,
 }
 
+impl X86ExceptionContext {
+    pub fn set_rip(&mut self, new_rip: usize) {
+        self.frame.rip = new_rip;
+
+        if is_cet_ss_supported() {
+            // Update the instruction pointer on the shadow stack.
+            let return_on_stack = (self.ssp + 8) as *const usize;
+            let return_on_stack_val = new_rip;
+            unsafe {
+                asm!(
+                    "wrssq [{}], {}",
+                    in(reg) return_on_stack,
+                    in(reg) return_on_stack_val
+                );
+            }
+        }
+    }
+}
+
 impl InsnMachineCtx for X86ExceptionContext {
     fn read_efer(&self) -> u64 {
         read_efer().bits()

kernel/src/cpu/idt/entry.S

@@ -29,6 +29,12 @@ HV_DOORBELL_ADDR:
         pushq   %r13
         pushq   %r14
         pushq   %r15
+	pushq   $0
+	# rdsspq is a nop when shadow stacks are not supported. Make sure that
+	# rax is 0 if that's the case.
+	xorl	%eax, %eax
+	rdsspq  %rax
+	pushq	%rax
 .endm
 
 .macro default_entry_no_ist name: req handler:req error_code:req vector:req
@@ -107,6 +113,14 @@ asm_entry_hv:
 	// RIP is in the return window, so update RIP to the cancel point.
 	leaq	switch_vmpl_cancel(%rip), %rbx
 	movq	%rbx, 0x20(%rsp)
+	.if CFG_SHADOW_STACKS
+	// Update RIP on the shadow stack to the cancel point.
+	cmpb    $0, {IS_CET_SUPPORTED}(%rip)
+        je 2f
+	rdsspq  %rax
+        wrssq	%rbx, 8(%rax)
+    2:
+	.endif
 	// Defer any further processing until interrupts can be processed.
 	jmp	postpone_hv
 hv_not_vmpl_switch:
@@ -141,12 +155,21 @@ hv_not_vmpl_switch:
 	// required.  This could not be performed before the RIP check because
 	// the previous RIP determines where to find the previous EFLAGS.IF
 	// value on the stack.
-	testl	${IF}, 18*8(%rcx)
+	testl	${IF}, 20*8(%rcx)
 	jz	postpone_hv
 	// Switch to the stack pointer from the previous exception, which
 	// points to the register save area, and continue with #HV
 	// processing.
 	movq	%rcx, %rsp
+        .if CFG_SHADOW_STACKS
+	// Pop the current stack frame, so that the previous stack frame sits
+	// on top of the shadow stack.
+	cmpb	$0, {IS_CET_SUPPORTED}(%rip)
+        je	2f
+	movl	$3, %eax
+	incsspq	%rax
+    2:
+	.endif
 	jmp	handle_as_hv
 
 postpone_hv:
@@ -189,6 +212,15 @@ restart_hv:
 	movq	2*8(%rsp), %rax
 	movq	%rax, -2*8(%rcx)
 	leaq	-4*8(%rcx), %rsp
+        .if CFG_SHADOW_STACKS
+	// Pop the current stack frame, so that the previous stack frame sits
+	// on top of the shadow stack.
+	cmpb	$0, {IS_CET_SUPPORTED}(%rip)
+        je	2f
+	movl	$3, %eax
+	incsspq	%rax
+    2:
+	.endif
 
 continue_hv:
 	// At this point, only the dummy error code and first three registers
@@ -206,6 +238,12 @@ continue_hv:
         pushq   %r13
         pushq   %r14
         pushq   %r15
+	pushq   $0
+        # rdsspq is a nop when shadow stacks are not supported. Make sure that
+	# rdx is 0 if that's the case.
+	xorl	%edx, %edx
+	rdsspq  %rdx
+        pushq	%rdx
 handle_as_hv:
 	// Load the address of the #HV doorbell page.  The global address
 	// might not yet be configured, and the per-CPU page might also not
@@ -225,13 +263,13 @@ handle_as_hv_with_doorbell:
 default_return:
 	// Ensure that interrupts are disabled before attempting any return.
 	cli
-	testb	$3, 17*8(%rsp) // Check CS in exception frame
+	testb	$3, 19*8(%rsp) // Check CS in exception frame
 	jnz 	return_user
 return_all_paths:
 	// If interrupts were previously available, then check whether any #HV
 	// events are pending.  If so, proceed as if the original trap was
 	// #HV.
-	testl 	${IF}, 18*8(%rsp) // check EFLAGS.IF in exception frame
+	testl 	${IF}, 20*8(%rsp) // check EFLAGS.IF in exception frame
 	jz 	begin_iret_return
 	movq 	HV_DOORBELL_ADDR(%rip), %rdi
 	test 	%rdi, %rdi
@@ -251,23 +289,23 @@ iret_return_window:
 begin_iret_return:
 	// Reload registers without modifying the stack pointer so that if #HV
 	// occurs within this window, the saved registers are still intact.
-	movq 	0*8(%rsp), %r15
-	movq 	1*8(%rsp), %r14
-	movq 	2*8(%rsp), %r13
-	movq 	3*8(%rsp), %r12
-	movq 	4*8(%rsp), %r11
-	movq 	5*8(%rsp), %r10
-	movq 	6*8(%rsp), %r9
-	movq 	7*8(%rsp), %r8
-	movq 	8*8(%rsp), %rbp
-	movq 	9*8(%rsp), %rdi
-	movq 	10*8(%rsp), %rsi
-	movq 	11*8(%rsp), %rdx
-	movq 	12*8(%rsp), %rcx
-	movq 	13*8(%rsp), %rbx
-	movq 	14*8(%rsp), %rax
-
-	addq 	$16*8, %rsp
+	movq 	2*8(%rsp), %r15
+	movq 	3*8(%rsp), %r14
+	movq 	4*8(%rsp), %r13
+	movq 	5*8(%rsp), %r12
+	movq 	6*8(%rsp), %r11
+	movq 	7*8(%rsp), %r10
+	movq 	8*8(%rsp), %r9
+	movq 	9*8(%rsp), %r8
+	movq 	10*8(%rsp), %rbp
+	movq 	11*8(%rsp), %rdi
+	movq 	12*8(%rsp), %rsi
+	movq 	13*8(%rsp), %rdx
+	movq 	14*8(%rsp), %rcx
+	movq 	15*8(%rsp), %rbx
+	movq 	16*8(%rsp), %rax
+
+	addq 	$18*8, %rsp
 
 default_iret:
 	iretq
@@ -343,7 +381,11 @@ default_entry_no_ist	name=xf		handler=panic			error_code=0	vector=19
 // Vector 20 not defined
 
 // #CP Control-Protection Exception (Vector 21)
+.if CFG_SHADOW_STACKS
+default_entry_no_ist	name=cp		handler=control_protection			error_code=1	vector=21
+.else
 default_entry_no_ist	name=cp		handler=panic			error_code=1	vector=21
+.endif
 
 // Vectors 22-27 not defined
 

kernel/src/cpu/idt/svsm.rs

@@ -17,8 +17,10 @@ use super::common::{
 };
 use crate::address::VirtAddr;
 use crate::cpu::registers::RFlags;
+use crate::cpu::shadow_stack::IS_CET_SUPPORTED;
 use crate::cpu::X86ExceptionContext;
 use crate::debug::gdbstub::svsm_gdbstub::handle_debug_exception;
+use crate::mm::GuestPtr;
 use crate::platform::SVSM_PLATFORM;
 use crate::task::{is_task_fault, terminate};
 use core::arch::global_asm;
@@ -205,6 +207,51 @@ extern "C" fn ex_handler_page_fault(ctxt: &mut X86ExceptionContext, vector: usiz
     }
 }
 
+// Control-Protection handler
+#[no_mangle]
+extern "C" fn ex_handler_control_protection(ctxt: &mut X86ExceptionContext, _vector: usize) {
+    // From AMD64 Architecture Programmer's Manual, Volume 2, 8.4.3
+    // Control-Protection Error Code:
+    /// A RET (near) instruction encountered a return address mismatch.
+    const NEAR_RET: usize = 1;
+    /// A RET (far) or IRET instruction encountered a return address mismatch.
+    const FAR_RET_IRET: usize = 2;
+    /// An RSTORSSP instruction encountered an invalid shadow stack restore
+    /// token.
+    const RSTORSSP: usize = 4;
+    /// A SETSSBSY instruction encountered an invalid supervisor shadow stack
+    /// token.
+    const SETSSBSY: usize = 5;
+
+    let rip = ctxt.frame.rip;
+    match ctxt.error_code & 0x7fff {
+        code @ (NEAR_RET | FAR_RET_IRET) => {
+            // Read the return address on the normal stack.
+            let ret_ptr: GuestPtr<u64> = GuestPtr::new(VirtAddr::from(ctxt.frame.rsp));
+            let ret = unsafe { ret_ptr.read() }.expect("Failed to read return address");
+
+            // Read the return address on the shadow stack.
+            let prev_rssp_ptr: GuestPtr<u64> = GuestPtr::new(VirtAddr::from(ctxt.ssp));
+            let prev_rssp = unsafe { prev_rssp_ptr.read() }
+                .expect("Failed to read address of previous shadow stack pointer");
+            // The offset to the return pointer is different for RET and IRET.
+            let offset = if code == NEAR_RET { 0 } else { 8 };
+            let ret_ptr: GuestPtr<u64> = GuestPtr::new(VirtAddr::from(prev_rssp + offset));
+            let ret_on_ssp =
+                unsafe { ret_ptr.read() }.expect("Failed to read return address on shadow stack");
+
+            panic!("thread at {rip:#018x} tried to return to {ret:#x}, but return address on shadow stack was {ret_on_ssp:#x}!");
+        }
+        RSTORSSP => {
+            panic!("rstorssp instruction encountered an unexpected shadow stack restore token at RIP {rip:#018x}");
+        }
+        SETSSBSY => {
+            panic!("setssbsy instruction encountered an unexpected supervisor shadow stack token at RIP {rip:#018x}");
+        }
+        code => unreachable!("unexpected code for #CP exception: {code}"),
+    }
+}
+
 // VMM Communication handler
 #[no_mangle]
 extern "C" fn ex_handler_vmm_communication(ctxt: &mut X86ExceptionContext, vector: usize) {
@@ -279,8 +326,13 @@ global_asm!(
         .set const_true, 1
     "#,
     concat!(".set CFG_NOSMAP, const_", cfg!(feature = "nosmap")),
+    concat!(
+        ".set CFG_SHADOW_STACKS, const_",
+        cfg!(feature = "shadow-stacks")
+    ),
     include_str!("../x86/smap.S"),
     include_str!("entry.S"),
     IF = const RFlags::IF.bits(),
+    IS_CET_SUPPORTED = sym IS_CET_SUPPORTED,
     options(att_syntax)
 );

kernel/src/cpu/isst.rs

@@ -0,0 +1,18 @@
+use core::num::NonZeroU8;
+
+use crate::address::VirtAddr;
+
+#[derive(Debug, Default, Clone, Copy)]
+#[repr(C)]
+pub struct Isst {
+    _reserved: u64,
+    entries: [VirtAddr; 7],
+}
+
+impl Isst {
+    pub fn set(&mut self, index: NonZeroU8, addr: VirtAddr) {
+        // ISST entries start at index 1
+        let index = usize::from(index.get() - 1);
+        self.entries[index] = addr;
+    }
+}

kernel/src/cpu/mod.rs

@@ -13,10 +13,12 @@ pub mod features;
 pub mod gdt;
 pub mod idt;
 pub mod irq_state;
+pub mod isst;
 pub mod mem;
 pub mod msr;
 pub mod percpu;
 pub mod registers;
+pub mod shadow_stack;
 pub mod smp;
 pub mod sse;
 pub mod tlb;