From b3391da590ceadaa5683e4ce2a99a9c21c4781f7 Mon Sep 17 00:00:00 2001 From: Jiaqi Gao Date: Tue, 17 Dec 2024 04:45:04 -0500 Subject: [PATCH] td-exception: use global asm to replace naked interrupt handler An interrupt handler table is used to handle all 256 interrupts. They share common handler that saves the interrupt vector and context. If the interrupt is not an error, a zero is pushed to the stack to align with the error one. Function `generic_interrupt_handler` is called to find and call the corresponding callback in the `CALLBACK_TABLE`. For virtualization exception, when the CET shadow stack is enabled, we need to update the LIP value in the shadow stack. As shadow stack saves the latest two return address after interrupt entry and the SSP, the position of saved LIP value is the top address of shadow stack minus 0x18 bytes. Signed-off-by: Jiaqi Gao --- td-exception/src/asm/handler.asm | 60 ++++++ td-exception/src/asm/mod.rs | 7 + td-exception/src/idt.rs | 46 +--- td-exception/src/interrupt.rs | 330 +++++++++++++---------------- td-exception/src/lib.rs | 7 +- td-payload/src/arch/x86_64/apic.rs | 56 ----- 6 files changed, 228 insertions(+), 278 deletions(-) create mode 100644 td-exception/src/asm/handler.asm create mode 100644 td-exception/src/asm/mod.rs diff --git a/td-exception/src/asm/handler.asm b/td-exception/src/asm/handler.asm new file mode 100644 index 00000000..10b0f2d8 --- /dev/null +++ b/td-exception/src/asm/handler.asm @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Intel Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent + +.section .text +interrupt_handler_entry: + push rax + push rcx + push rdx + push rdi + push rsi + push r8 + push r9 + push r10 + push r11 + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov rdi, rsp + call generic_interrupt_handler + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + pop r11 + pop r10 + pop r9 + pop r8 + pop rsi + pop rdi + pop rdx + pop rcx + pop rax + + # vector number and error code + add rsp, 16 + + iretq + +.align 32 +.global interrupt_handler_table +interrupt_handler_table: + i = 0 + .rept 256 + .align 32 + .if ((0x20027d00 >> i) & 1) == 0 + push 0 + .endif + push i + jmp interrupt_handler_entry + i = i + 1 + .endr + + ret diff --git a/td-exception/src/asm/mod.rs b/td-exception/src/asm/mod.rs new file mode 100644 index 00000000..b06d53ec --- /dev/null +++ b/td-exception/src/asm/mod.rs @@ -0,0 +1,7 @@ +use core::arch::global_asm; + +global_asm!(include_str!("handler.asm")); + +extern "C" { + pub(crate) static interrupt_handler_table: u8; +} diff --git a/td-exception/src/idt.rs b/td-exception/src/idt.rs index a4258e67..5f56e926 100644 --- a/td-exception/src/idt.rs +++ b/td-exception/src/idt.rs @@ -22,9 +22,10 @@ use x86_64::{ VirtAddr, }; -use crate::interrupt; +use crate::asm::interrupt_handler_table; +use crate::interrupt::init_interrupt_callbacks; -const IDT_ENTRY_COUNT: usize = 256; +pub(crate) const IDT_ENTRY_COUNT: usize = 256; lazy_static! { static ref INIT_IDT: Mutex = Mutex::new(Idt::new()); @@ -71,44 +72,13 @@ impl Idt { pub fn init(&mut self) { let current_idt = &mut self.entries; + let handler_table = unsafe { &interrupt_handler_table as *const u8 as usize }; - // Set up exceptions handler according to Intel64 & IA32 Software Developer Manual - // Reference: https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html - current_idt[0].set_func(interrupt::divide_by_zero as usize); - current_idt[1].set_func(interrupt::debug as usize); - current_idt[2].set_func(interrupt::non_maskable as usize); - current_idt[3].set_func(interrupt::breakpoint as usize); - current_idt[4].set_func(interrupt::overflow as usize); - current_idt[5].set_func(interrupt::bound_range as usize); - current_idt[6].set_func(interrupt::invalid_opcode as usize); - current_idt[7].set_func(interrupt::device_not_available as usize); - current_idt[8].set_func(interrupt::double_fault as usize); - // 9 no longer available - current_idt[9].set_func(interrupt::default_exception as usize); - current_idt[10].set_func(interrupt::invalid_tss as usize); - current_idt[11].set_func(interrupt::segment_not_present as usize); - current_idt[12].set_func(interrupt::stack_segment as usize); - current_idt[13].set_func(interrupt::protection as usize); - current_idt[14].set_func(interrupt::page as usize); - // 15 reserved - current_idt[15].set_func(interrupt::default_exception as usize); - current_idt[16].set_func(interrupt::fpu as usize); - current_idt[17].set_func(interrupt::alignment_check as usize); - current_idt[18].set_func(interrupt::machine_check as usize); - current_idt[19].set_func(interrupt::simd as usize); - #[cfg(feature = "tdx")] - current_idt[20].set_func(interrupt::virtualization as usize); - #[cfg(not(feature = "tdx"))] - current_idt[20].set_func(interrupt::default_exception as usize); - current_idt[21].set_func(interrupt::control_flow as usize); - // reset exception reserved - for idt in current_idt.iter_mut().take(32).skip(22) { - idt.set_func(interrupt::default_exception as usize); - } - // Setup reset potential interrupt handler. - for idt in current_idt.iter_mut().take(IDT_ENTRY_COUNT).skip(32) { - idt.set_func(interrupt::default_interrupt as usize); + for (idx, idt) in current_idt.iter_mut().enumerate() { + idt.set_func(handler_table + idx * 32); } + + init_interrupt_callbacks(); } // Construct the Interrupt Descriptor Table Pointer (IDTR) based diff --git a/td-exception/src/interrupt.rs b/td-exception/src/interrupt.rs index 659fa847..c5b53f06 100644 --- a/td-exception/src/interrupt.rs +++ b/td-exception/src/interrupt.rs @@ -3,9 +3,12 @@ // SPDX-License-Identifier: BSD-2-Clause-Patent use core::arch::asm; +use spin::Mutex; #[cfg(feature = "tdx")] use tdx_tdcall::tdx; +use crate::{idt::IDT_ENTRY_COUNT, ExceptionError}; + // the order is aligned with scratch_push!() and scratch_pop!() #[repr(C, packed)] pub struct ScratchRegisters { @@ -34,40 +37,6 @@ impl ScratchRegisters { } } -#[macro_export] -macro_rules! scratch_push { - () => { - " - push rax - push rcx - push rdx - push rdi - push rsi - push r8 - push r9 - push r10 - push r11 - " - }; -} - -#[macro_export] -macro_rules! scratch_pop { - () => { - " - pop r11 - pop r10 - pop r9 - pop r8 - pop rsi - pop rdi - pop rdx - pop rcx - pop rax - " - }; -} - #[repr(C, packed)] pub struct PreservedRegisters { pub r15: usize, @@ -89,34 +58,6 @@ impl PreservedRegisters { } } -#[macro_export] -macro_rules! preserved_push { - () => { - " - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - " - }; -} - -#[macro_export] -macro_rules! preserved_pop { - () => { - " - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - " - }; -} - #[repr(packed)] pub struct IretRegisters { pub rip: usize, @@ -133,235 +74,254 @@ impl IretRegisters { } #[repr(packed)] -pub struct InterruptNoErrorStack { +pub struct InterruptStack { pub preserved: PreservedRegisters, pub scratch: ScratchRegisters, + pub vector: usize, + pub code: usize, pub iret: IretRegisters, } -impl InterruptNoErrorStack { +impl InterruptStack { pub fn dump(&self) { self.iret.dump(); + log::info!("CODE: {:>016X}\n", { self.code }); + log::info!("VECTOR: {:>016X}\n", { self.vector }); self.scratch.dump(); self.preserved.dump(); } } -#[repr(packed)] -pub struct InterruptErrorStack { - pub preserved: PreservedRegisters, - pub scratch: ScratchRegisters, - pub code: usize, - pub iret: IretRegisters, +#[derive(Debug, Copy, Clone)] +pub struct InterruptCallback { + func: fn(&mut InterruptStack), } -impl InterruptErrorStack { - pub fn dump(&self) { - self.iret.dump(); - log::info!("CODE: {:>016X}\n", { self.code }); - self.scratch.dump(); - self.preserved.dump(); +impl InterruptCallback { + const fn new(func: fn(&mut InterruptStack)) -> Self { + InterruptCallback { func } } } -#[macro_export] -macro_rules! interrupt_common { - ($name:ident, $stack: ident, $stack_type:ty, $func:block, $asm_epilogue:literal) => { - #[naked] - #[no_mangle] - pub unsafe extern fn $name () { - #[inline(never)] - unsafe extern "win64" fn inner($stack: &mut $stack_type) { - $func - } +struct InterruptCallbackTable { + table: [InterruptCallback; IDT_ENTRY_COUNT], +} - // Push scratch registers - core::arch::asm!( concat!( - $crate::scratch_push!(), - $crate::preserved_push!(), - " - mov rcx, rsp - call {inner} - ", - $crate::preserved_pop!(), - $crate::scratch_pop!(), - $asm_epilogue - ), - inner = sym inner, - options(noreturn), - ) +impl InterruptCallbackTable { + const fn init() -> Self { + InterruptCallbackTable { + table: [InterruptCallback::new(default_callback); IDT_ENTRY_COUNT], } - }; + } } -#[macro_export] -macro_rules! interrupt_no_error { - ($name:ident, $stack: ident, $func:block) => { - $crate::interrupt_common!( - $name, - $stack, - $crate::interrupt::InterruptNoErrorStack, - $func, - " - iretq - " - ); - }; +static CALLBACK_TABLE: Mutex = Mutex::new(InterruptCallbackTable::init()); + +pub(crate) fn init_interrupt_callbacks() { + let mut callbacks = CALLBACK_TABLE.lock(); + // Set up exceptions handler according to Intel64 & IA32 Software Developer Manual + // Reference: https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html + callbacks.table[0].func = divide_by_zero; + callbacks.table[1].func = debug; + callbacks.table[2].func = non_maskable; + callbacks.table[3].func = breakpoint; + callbacks.table[4].func = overflow; + callbacks.table[5].func = bound_range; + callbacks.table[6].func = invalid_opcode; + callbacks.table[7].func = device_not_available; + callbacks.table[8].func = double_fault; + // 9 no longer available + callbacks.table[10].func = invalid_tss; + callbacks.table[11].func = segment_not_present; + callbacks.table[12].func = stack_segment; + callbacks.table[13].func = protection; + callbacks.table[14].func = page; + // 15 reserved + callbacks.table[16].func = fpu; + callbacks.table[17].func = alignment_check; + callbacks.table[18].func = machine_check; + callbacks.table[19].func = simd; + #[cfg(feature = "tdx")] + { + callbacks.table[20].func = virtualization; + } + callbacks.table[21].func = control_flow; } -#[macro_export] -macro_rules! interrupt_error { - ($name:ident, $stack: ident, $func:block) => { - $crate::interrupt_common!( - $name, - $stack, - $crate::interrupt::InterruptErrorStack, - $func, - " - add rsp, 8 - iretq +pub fn register_interrupt_callback( + index: usize, + callback: InterruptCallback, +) -> Result<(), ExceptionError> { + if index > IDT_ENTRY_COUNT { + return Err(ExceptionError::InvalidParameter); + } + CALLBACK_TABLE.lock().table[index] = callback; + Ok(()) +} + +fn eoi() { + // Write the end-of-interrupt (EOI) register (0x80B) at the end of the handler + // routine, sometime before the IRET instruction + unsafe { + asm!( " - ); - }; + mov rcx, 0x80B + mov edx, 0 + mov eax, 0 + wrmsr + " + ) + } } -interrupt_no_error!(default_exception, stack, { - log::info!("default exception\n"); - stack.dump(); - deadloop(); -}); +#[no_mangle] +fn generic_interrupt_handler(stack: &mut InterruptStack) { + if stack.vector >= IDT_ENTRY_COUNT { + log::error!("Invalid interrupt vector number!\n"); + return; + } + + (CALLBACK_TABLE.lock().table[stack.vector].func)(stack); + + // If we are handling an interrupt, signal a end-of-interrupt before return. + if stack.vector > 31 { + eoi(); + } +} -interrupt_no_error!(default_interrupt, stack, { - log::info!("default interrupt\n"); +fn default_callback(stack: &mut InterruptStack) { + log::info!("default interrupt callback\n"); stack.dump(); deadloop(); -}); +} #[cfg(feature = "integration-test")] -interrupt_no_error!(divide_by_zero, stack, { +fn divide_by_zero(stack: &mut InterruptStack) { log::info!("Divide by zero\n"); crate::DIVIDED_BY_ZERO_EVENT_COUNT.fetch_add(1, core::sync::atomic::Ordering::AcqRel); stack.iret.rip += 7; log::info!("divide_by_zero done\n"); return; -}); +} #[cfg(not(feature = "integration-test"))] -interrupt_no_error!(divide_by_zero, stack, { +fn divide_by_zero(stack: &mut InterruptStack) { log::info!("Divide by zero\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(debug, stack, { +fn debug(stack: &mut InterruptStack) { log::info!("Debug trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(non_maskable, stack, { +fn non_maskable(stack: &mut InterruptStack) { log::info!("Non-maskable interrupt\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(breakpoint, stack, { +fn breakpoint(stack: &mut InterruptStack) { log::info!("Breakpoint trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(overflow, stack, { +fn overflow(stack: &mut InterruptStack) { log::info!("Overflow trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(bound_range, stack, { +fn bound_range(stack: &mut InterruptStack) { log::info!("Bound range exceeded fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(invalid_opcode, stack, { +fn invalid_opcode(stack: &mut InterruptStack) { log::info!("Invalid opcode fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(device_not_available, stack, { +fn device_not_available(stack: &mut InterruptStack) { log::info!("Device not available fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(double_fault, stack, { +fn double_fault(stack: &mut InterruptStack) { log::info!("Double fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(invalid_tss, stack, { +fn invalid_tss(stack: &mut InterruptStack) { log::info!("Invalid TSS fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(segment_not_present, stack, { +fn segment_not_present(stack: &mut InterruptStack) { log::info!("Segment not present fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(stack_segment, stack, { +fn stack_segment(stack: &mut InterruptStack) { log::info!("Stack segment fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(protection, stack, { +fn protection(stack: &mut InterruptStack) { log::info!("Protection fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(page, stack, { +fn page(stack: &mut InterruptStack) { let cr2: usize; - asm!("mov {}, cr2", out(reg) cr2); + unsafe { + asm!("mov {}, cr2", out(reg) cr2); + } log::info!("Page fault: {:>016X}\n", cr2); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(fpu, stack, { +fn fpu(stack: &mut InterruptStack) { log::info!("FPU floating point fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(alignment_check, stack, { +fn alignment_check(stack: &mut InterruptStack) { log::info!("Alignment check fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(machine_check, stack, { +fn machine_check(stack: &mut InterruptStack) { log::info!("Machine check fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(simd, stack, { +fn simd(stack: &mut InterruptStack) { log::info!("SIMD floating point fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(control_flow, stack, { +fn control_flow(stack: &mut InterruptStack) { log::info!("Control Flow Exception\n"); stack.dump(); deadloop(); -}); +} #[cfg(feature = "tdx")] const EXIT_REASON_CPUID: u32 = 10; @@ -385,7 +345,7 @@ const EXIT_REASON_MONITOR_INSTRUCTION: u32 = 39; const EXIT_REASON_WBINVD: u32 = 54; #[cfg(feature = "tdx")] -interrupt_no_error!(virtualization, stack, { +fn virtualization(stack: &mut InterruptStack) { // Firstly get VE information from TDX module, halt it error occurs let ve_info = tdx::tdcall_get_ve_info().expect("#VE handler: fail to get VE info\n"); @@ -439,7 +399,7 @@ interrupt_no_error!(virtualization, stack, { // stack and the `RIP` value saved in the normal stack when executing a return from an // exception handler and cause a control protection exception if they do not match. #[cfg(feature = "cet-shstk")] - { + unsafe { use x86_64::registers::control::{Cr4, Cr4Flags}; use x86_64::registers::model_specific::Msr; @@ -463,8 +423,12 @@ interrupt_no_error!(virtualization, stack, { ssp = out(reg) ssp, ); - // SSP -> Return address of current function | SSP | LIP | CS - let lip_ptr = ssp + 0x10; + // SSP -> return address of func [virtualization] + // return address of func [generic_interrupt_handler] + // SSP + // LIP + // CS + let lip_ptr = ssp + 0x18; let lip = *(lip_ptr as *const u64) + ve_info.exit_instruction_length as u64; // Enables the WRSSD/WRSSQ instructions by setting the `WR_SHSTK_E` @@ -481,14 +445,14 @@ interrupt_no_error!(virtualization, stack, { // Clear the `WR_SHSTK_E` msr_cet.write(msr_cet.read() & !WR_SHSTK_E); } -}); +} // Handle IO exit from TDX Module // // Use TDVMCALL to realize IO read/write operation // Return false if VE info is invalid #[cfg(feature = "tdx")] -fn handle_tdx_ioexit(ve_info: &tdx::TdVeInfo, stack: &mut InterruptNoErrorStack) -> bool { +fn handle_tdx_ioexit(ve_info: &tdx::TdVeInfo, stack: &mut InterruptStack) -> bool { let size = ((ve_info.exit_qualification & 0x7) + 1) as usize; // 0 - 1bytes, 1 - 2bytes, 3 - 4bytes let read = (ve_info.exit_qualification >> 3) & 0x1 == 1; let string = (ve_info.exit_qualification >> 4) & 0x1 == 1; diff --git a/td-exception/src/lib.rs b/td-exception/src/lib.rs index da7a0d3a..dc79487c 100644 --- a/td-exception/src/lib.rs +++ b/td-exception/src/lib.rs @@ -3,8 +3,8 @@ // SPDX-License-Identifier: BSD-2-Clause-Patent #![no_std] -#![feature(naked_functions)] +mod asm; pub mod idt; pub mod interrupt; @@ -17,3 +17,8 @@ pub fn setup_exception_handlers() { lazy_static::lazy_static! { pub static ref DIVIDED_BY_ZERO_EVENT_COUNT: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); } + +pub enum ExceptionError { + // Caller gives an input that is not expected. + InvalidParameter, +} diff --git a/td-payload/src/arch/x86_64/apic.rs b/td-payload/src/arch/x86_64/apic.rs index 2e37afe4..89fc4441 100644 --- a/td-payload/src/arch/x86_64/apic.rs +++ b/td-payload/src/arch/x86_64/apic.rs @@ -2,9 +2,6 @@ // // SPDX-License-Identifier: BSD-2-Clause-Patent -pub use td_exception::interrupt::InterruptNoErrorStack; -pub use td_exception::{preserved_pop, preserved_push, scratch_pop, scratch_push}; - // MSR registers pub const MSR_LVTT: u32 = 0x832; pub const MSR_INITIAL_COUNT: u32 = 0x838; @@ -54,56 +51,3 @@ pub fn one_shot_tsc_deadline_mode(period: u64) -> Option { pub fn one_shot_tsc_deadline_mode_reset() { unsafe { x86::msr::wrmsr(MSR_TSC_DEADLINE, 0) } } - -#[macro_export] -macro_rules! interrupt_handler_template { - ($name:ident, $stack: ident, $func:block) => { - #[naked] - #[no_mangle] - /// # Safety - /// - /// Interrupt handler will handle the register reserve and restore - pub unsafe extern fn $name () { - #[inline(never)] - /// # Safety - /// - /// Interrupt handler will handle the register reserve and restore - unsafe extern "win64" fn inner($stack: &mut $crate::arch::apic::InterruptNoErrorStack) { - $func - } - - // Push scratch registers - core::arch::asm!( concat!( - $crate::arch::apic::scratch_push!(), - $crate::arch::apic::preserved_push!(), - " - mov rcx, rsp - call {inner} - ", - $crate::eoi!(), - $crate::arch::apic::preserved_pop!(), - $crate::arch::apic::scratch_pop!(), - " - iretq - " - ), - inner = sym inner, - options(noreturn), - ) - } - }; -} - -#[macro_export] -macro_rules! eoi { - // Write the end-of-interrupt (EOI) register (0x80B) at the end of the handler - // routine, sometime before the IRET instruction - () => { - " - mov rcx, 0x80B - mov edx, 0 - mov eax, 0 - wrmsr - " - }; -}