diff --git a/td-exception/src/asm/handler.asm b/td-exception/src/asm/handler.asm new file mode 100644 index 00000000..10b0f2d8 --- /dev/null +++ b/td-exception/src/asm/handler.asm @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Intel Corporation +# SPDX-License-Identifier: BSD-2-Clause-Patent + +.section .text +interrupt_handler_entry: + push rax + push rcx + push rdx + push rdi + push rsi + push r8 + push r9 + push r10 + push r11 + push rbx + push rbp + push r12 + push r13 + push r14 + push r15 + + mov rdi, rsp + call generic_interrupt_handler + + pop r15 + pop r14 + pop r13 + pop r12 + pop rbp + pop rbx + pop r11 + pop r10 + pop r9 + pop r8 + pop rsi + pop rdi + pop rdx + pop rcx + pop rax + + # vector number and error code + add rsp, 16 + + iretq + +.align 32 +.global interrupt_handler_table +interrupt_handler_table: + i = 0 + .rept 256 + .align 32 + .if ((0x20027d00 >> i) & 1) == 0 + push 0 + .endif + push i + jmp interrupt_handler_entry + i = i + 1 + .endr + + ret diff --git a/td-exception/src/asm/mod.rs b/td-exception/src/asm/mod.rs new file mode 100644 index 00000000..b06d53ec --- /dev/null +++ b/td-exception/src/asm/mod.rs @@ -0,0 +1,7 @@ +use core::arch::global_asm; + +global_asm!(include_str!("handler.asm")); + +extern "C" { + pub(crate) static interrupt_handler_table: u8; +} diff --git a/td-exception/src/idt.rs b/td-exception/src/idt.rs index a4258e67..5f56e926 100644 --- a/td-exception/src/idt.rs +++ b/td-exception/src/idt.rs @@ -22,9 +22,10 @@ use x86_64::{ VirtAddr, }; -use crate::interrupt; +use crate::asm::interrupt_handler_table; +use crate::interrupt::init_interrupt_callbacks; -const IDT_ENTRY_COUNT: usize = 256; +pub(crate) const IDT_ENTRY_COUNT: usize = 256; lazy_static! { static ref INIT_IDT: Mutex = Mutex::new(Idt::new()); @@ -71,44 +72,13 @@ impl Idt { pub fn init(&mut self) { let current_idt = &mut self.entries; + let handler_table = unsafe { &interrupt_handler_table as *const u8 as usize }; - // Set up exceptions handler according to Intel64 & IA32 Software Developer Manual - // Reference: https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html - current_idt[0].set_func(interrupt::divide_by_zero as usize); - current_idt[1].set_func(interrupt::debug as usize); - current_idt[2].set_func(interrupt::non_maskable as usize); - current_idt[3].set_func(interrupt::breakpoint as usize); - current_idt[4].set_func(interrupt::overflow as usize); - current_idt[5].set_func(interrupt::bound_range as usize); - current_idt[6].set_func(interrupt::invalid_opcode as usize); - current_idt[7].set_func(interrupt::device_not_available as usize); - current_idt[8].set_func(interrupt::double_fault as usize); - // 9 no longer available - current_idt[9].set_func(interrupt::default_exception as usize); - current_idt[10].set_func(interrupt::invalid_tss as usize); - current_idt[11].set_func(interrupt::segment_not_present as usize); - current_idt[12].set_func(interrupt::stack_segment as usize); - current_idt[13].set_func(interrupt::protection as usize); - current_idt[14].set_func(interrupt::page as usize); - // 15 reserved - current_idt[15].set_func(interrupt::default_exception as usize); - current_idt[16].set_func(interrupt::fpu as usize); - current_idt[17].set_func(interrupt::alignment_check as usize); - current_idt[18].set_func(interrupt::machine_check as usize); - current_idt[19].set_func(interrupt::simd as usize); - #[cfg(feature = "tdx")] - current_idt[20].set_func(interrupt::virtualization as usize); - #[cfg(not(feature = "tdx"))] - current_idt[20].set_func(interrupt::default_exception as usize); - current_idt[21].set_func(interrupt::control_flow as usize); - // reset exception reserved - for idt in current_idt.iter_mut().take(32).skip(22) { - idt.set_func(interrupt::default_exception as usize); - } - // Setup reset potential interrupt handler. - for idt in current_idt.iter_mut().take(IDT_ENTRY_COUNT).skip(32) { - idt.set_func(interrupt::default_interrupt as usize); + for (idx, idt) in current_idt.iter_mut().enumerate() { + idt.set_func(handler_table + idx * 32); } + + init_interrupt_callbacks(); } // Construct the Interrupt Descriptor Table Pointer (IDTR) based diff --git a/td-exception/src/interrupt.rs b/td-exception/src/interrupt.rs index 659fa847..c5b53f06 100644 --- a/td-exception/src/interrupt.rs +++ b/td-exception/src/interrupt.rs @@ -3,9 +3,12 @@ // SPDX-License-Identifier: BSD-2-Clause-Patent use core::arch::asm; +use spin::Mutex; #[cfg(feature = "tdx")] use tdx_tdcall::tdx; +use crate::{idt::IDT_ENTRY_COUNT, ExceptionError}; + // the order is aligned with scratch_push!() and scratch_pop!() #[repr(C, packed)] pub struct ScratchRegisters { @@ -34,40 +37,6 @@ impl ScratchRegisters { } } -#[macro_export] -macro_rules! scratch_push { - () => { - " - push rax - push rcx - push rdx - push rdi - push rsi - push r8 - push r9 - push r10 - push r11 - " - }; -} - -#[macro_export] -macro_rules! scratch_pop { - () => { - " - pop r11 - pop r10 - pop r9 - pop r8 - pop rsi - pop rdi - pop rdx - pop rcx - pop rax - " - }; -} - #[repr(C, packed)] pub struct PreservedRegisters { pub r15: usize, @@ -89,34 +58,6 @@ impl PreservedRegisters { } } -#[macro_export] -macro_rules! preserved_push { - () => { - " - push rbx - push rbp - push r12 - push r13 - push r14 - push r15 - " - }; -} - -#[macro_export] -macro_rules! preserved_pop { - () => { - " - pop r15 - pop r14 - pop r13 - pop r12 - pop rbp - pop rbx - " - }; -} - #[repr(packed)] pub struct IretRegisters { pub rip: usize, @@ -133,235 +74,254 @@ impl IretRegisters { } #[repr(packed)] -pub struct InterruptNoErrorStack { +pub struct InterruptStack { pub preserved: PreservedRegisters, pub scratch: ScratchRegisters, + pub vector: usize, + pub code: usize, pub iret: IretRegisters, } -impl InterruptNoErrorStack { +impl InterruptStack { pub fn dump(&self) { self.iret.dump(); + log::info!("CODE: {:>016X}\n", { self.code }); + log::info!("VECTOR: {:>016X}\n", { self.vector }); self.scratch.dump(); self.preserved.dump(); } } -#[repr(packed)] -pub struct InterruptErrorStack { - pub preserved: PreservedRegisters, - pub scratch: ScratchRegisters, - pub code: usize, - pub iret: IretRegisters, +#[derive(Debug, Copy, Clone)] +pub struct InterruptCallback { + func: fn(&mut InterruptStack), } -impl InterruptErrorStack { - pub fn dump(&self) { - self.iret.dump(); - log::info!("CODE: {:>016X}\n", { self.code }); - self.scratch.dump(); - self.preserved.dump(); +impl InterruptCallback { + const fn new(func: fn(&mut InterruptStack)) -> Self { + InterruptCallback { func } } } -#[macro_export] -macro_rules! interrupt_common { - ($name:ident, $stack: ident, $stack_type:ty, $func:block, $asm_epilogue:literal) => { - #[naked] - #[no_mangle] - pub unsafe extern fn $name () { - #[inline(never)] - unsafe extern "win64" fn inner($stack: &mut $stack_type) { - $func - } +struct InterruptCallbackTable { + table: [InterruptCallback; IDT_ENTRY_COUNT], +} - // Push scratch registers - core::arch::asm!( concat!( - $crate::scratch_push!(), - $crate::preserved_push!(), - " - mov rcx, rsp - call {inner} - ", - $crate::preserved_pop!(), - $crate::scratch_pop!(), - $asm_epilogue - ), - inner = sym inner, - options(noreturn), - ) +impl InterruptCallbackTable { + const fn init() -> Self { + InterruptCallbackTable { + table: [InterruptCallback::new(default_callback); IDT_ENTRY_COUNT], } - }; + } } -#[macro_export] -macro_rules! interrupt_no_error { - ($name:ident, $stack: ident, $func:block) => { - $crate::interrupt_common!( - $name, - $stack, - $crate::interrupt::InterruptNoErrorStack, - $func, - " - iretq - " - ); - }; +static CALLBACK_TABLE: Mutex = Mutex::new(InterruptCallbackTable::init()); + +pub(crate) fn init_interrupt_callbacks() { + let mut callbacks = CALLBACK_TABLE.lock(); + // Set up exceptions handler according to Intel64 & IA32 Software Developer Manual + // Reference: https://www.intel.com/content/www/us/en/developer/articles/technical/intel-sdm.html + callbacks.table[0].func = divide_by_zero; + callbacks.table[1].func = debug; + callbacks.table[2].func = non_maskable; + callbacks.table[3].func = breakpoint; + callbacks.table[4].func = overflow; + callbacks.table[5].func = bound_range; + callbacks.table[6].func = invalid_opcode; + callbacks.table[7].func = device_not_available; + callbacks.table[8].func = double_fault; + // 9 no longer available + callbacks.table[10].func = invalid_tss; + callbacks.table[11].func = segment_not_present; + callbacks.table[12].func = stack_segment; + callbacks.table[13].func = protection; + callbacks.table[14].func = page; + // 15 reserved + callbacks.table[16].func = fpu; + callbacks.table[17].func = alignment_check; + callbacks.table[18].func = machine_check; + callbacks.table[19].func = simd; + #[cfg(feature = "tdx")] + { + callbacks.table[20].func = virtualization; + } + callbacks.table[21].func = control_flow; } -#[macro_export] -macro_rules! interrupt_error { - ($name:ident, $stack: ident, $func:block) => { - $crate::interrupt_common!( - $name, - $stack, - $crate::interrupt::InterruptErrorStack, - $func, - " - add rsp, 8 - iretq +pub fn register_interrupt_callback( + index: usize, + callback: InterruptCallback, +) -> Result<(), ExceptionError> { + if index > IDT_ENTRY_COUNT { + return Err(ExceptionError::InvalidParameter); + } + CALLBACK_TABLE.lock().table[index] = callback; + Ok(()) +} + +fn eoi() { + // Write the end-of-interrupt (EOI) register (0x80B) at the end of the handler + // routine, sometime before the IRET instruction + unsafe { + asm!( " - ); - }; + mov rcx, 0x80B + mov edx, 0 + mov eax, 0 + wrmsr + " + ) + } } -interrupt_no_error!(default_exception, stack, { - log::info!("default exception\n"); - stack.dump(); - deadloop(); -}); +#[no_mangle] +fn generic_interrupt_handler(stack: &mut InterruptStack) { + if stack.vector >= IDT_ENTRY_COUNT { + log::error!("Invalid interrupt vector number!\n"); + return; + } + + (CALLBACK_TABLE.lock().table[stack.vector].func)(stack); + + // If we are handling an interrupt, signal a end-of-interrupt before return. + if stack.vector > 31 { + eoi(); + } +} -interrupt_no_error!(default_interrupt, stack, { - log::info!("default interrupt\n"); +fn default_callback(stack: &mut InterruptStack) { + log::info!("default interrupt callback\n"); stack.dump(); deadloop(); -}); +} #[cfg(feature = "integration-test")] -interrupt_no_error!(divide_by_zero, stack, { +fn divide_by_zero(stack: &mut InterruptStack) { log::info!("Divide by zero\n"); crate::DIVIDED_BY_ZERO_EVENT_COUNT.fetch_add(1, core::sync::atomic::Ordering::AcqRel); stack.iret.rip += 7; log::info!("divide_by_zero done\n"); return; -}); +} #[cfg(not(feature = "integration-test"))] -interrupt_no_error!(divide_by_zero, stack, { +fn divide_by_zero(stack: &mut InterruptStack) { log::info!("Divide by zero\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(debug, stack, { +fn debug(stack: &mut InterruptStack) { log::info!("Debug trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(non_maskable, stack, { +fn non_maskable(stack: &mut InterruptStack) { log::info!("Non-maskable interrupt\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(breakpoint, stack, { +fn breakpoint(stack: &mut InterruptStack) { log::info!("Breakpoint trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(overflow, stack, { +fn overflow(stack: &mut InterruptStack) { log::info!("Overflow trap\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(bound_range, stack, { +fn bound_range(stack: &mut InterruptStack) { log::info!("Bound range exceeded fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(invalid_opcode, stack, { +fn invalid_opcode(stack: &mut InterruptStack) { log::info!("Invalid opcode fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(device_not_available, stack, { +fn device_not_available(stack: &mut InterruptStack) { log::info!("Device not available fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(double_fault, stack, { +fn double_fault(stack: &mut InterruptStack) { log::info!("Double fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(invalid_tss, stack, { +fn invalid_tss(stack: &mut InterruptStack) { log::info!("Invalid TSS fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(segment_not_present, stack, { +fn segment_not_present(stack: &mut InterruptStack) { log::info!("Segment not present fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(stack_segment, stack, { +fn stack_segment(stack: &mut InterruptStack) { log::info!("Stack segment fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(protection, stack, { +fn protection(stack: &mut InterruptStack) { log::info!("Protection fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(page, stack, { +fn page(stack: &mut InterruptStack) { let cr2: usize; - asm!("mov {}, cr2", out(reg) cr2); + unsafe { + asm!("mov {}, cr2", out(reg) cr2); + } log::info!("Page fault: {:>016X}\n", cr2); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(fpu, stack, { +fn fpu(stack: &mut InterruptStack) { log::info!("FPU floating point fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(alignment_check, stack, { +fn alignment_check(stack: &mut InterruptStack) { log::info!("Alignment check fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(machine_check, stack, { +fn machine_check(stack: &mut InterruptStack) { log::info!("Machine check fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_no_error!(simd, stack, { +fn simd(stack: &mut InterruptStack) { log::info!("SIMD floating point fault\n"); stack.dump(); deadloop(); -}); +} -interrupt_error!(control_flow, stack, { +fn control_flow(stack: &mut InterruptStack) { log::info!("Control Flow Exception\n"); stack.dump(); deadloop(); -}); +} #[cfg(feature = "tdx")] const EXIT_REASON_CPUID: u32 = 10; @@ -385,7 +345,7 @@ const EXIT_REASON_MONITOR_INSTRUCTION: u32 = 39; const EXIT_REASON_WBINVD: u32 = 54; #[cfg(feature = "tdx")] -interrupt_no_error!(virtualization, stack, { +fn virtualization(stack: &mut InterruptStack) { // Firstly get VE information from TDX module, halt it error occurs let ve_info = tdx::tdcall_get_ve_info().expect("#VE handler: fail to get VE info\n"); @@ -439,7 +399,7 @@ interrupt_no_error!(virtualization, stack, { // stack and the `RIP` value saved in the normal stack when executing a return from an // exception handler and cause a control protection exception if they do not match. #[cfg(feature = "cet-shstk")] - { + unsafe { use x86_64::registers::control::{Cr4, Cr4Flags}; use x86_64::registers::model_specific::Msr; @@ -463,8 +423,12 @@ interrupt_no_error!(virtualization, stack, { ssp = out(reg) ssp, ); - // SSP -> Return address of current function | SSP | LIP | CS - let lip_ptr = ssp + 0x10; + // SSP -> return address of func [virtualization] + // return address of func [generic_interrupt_handler] + // SSP + // LIP + // CS + let lip_ptr = ssp + 0x18; let lip = *(lip_ptr as *const u64) + ve_info.exit_instruction_length as u64; // Enables the WRSSD/WRSSQ instructions by setting the `WR_SHSTK_E` @@ -481,14 +445,14 @@ interrupt_no_error!(virtualization, stack, { // Clear the `WR_SHSTK_E` msr_cet.write(msr_cet.read() & !WR_SHSTK_E); } -}); +} // Handle IO exit from TDX Module // // Use TDVMCALL to realize IO read/write operation // Return false if VE info is invalid #[cfg(feature = "tdx")] -fn handle_tdx_ioexit(ve_info: &tdx::TdVeInfo, stack: &mut InterruptNoErrorStack) -> bool { +fn handle_tdx_ioexit(ve_info: &tdx::TdVeInfo, stack: &mut InterruptStack) -> bool { let size = ((ve_info.exit_qualification & 0x7) + 1) as usize; // 0 - 1bytes, 1 - 2bytes, 3 - 4bytes let read = (ve_info.exit_qualification >> 3) & 0x1 == 1; let string = (ve_info.exit_qualification >> 4) & 0x1 == 1; diff --git a/td-exception/src/lib.rs b/td-exception/src/lib.rs index da7a0d3a..dc79487c 100644 --- a/td-exception/src/lib.rs +++ b/td-exception/src/lib.rs @@ -3,8 +3,8 @@ // SPDX-License-Identifier: BSD-2-Clause-Patent #![no_std] -#![feature(naked_functions)] +mod asm; pub mod idt; pub mod interrupt; @@ -17,3 +17,8 @@ pub fn setup_exception_handlers() { lazy_static::lazy_static! { pub static ref DIVIDED_BY_ZERO_EVENT_COUNT: core::sync::atomic::AtomicU32 = core::sync::atomic::AtomicU32::new(0); } + +pub enum ExceptionError { + // Caller gives an input that is not expected. + InvalidParameter, +} diff --git a/td-payload/src/arch/x86_64/apic.rs b/td-payload/src/arch/x86_64/apic.rs index 2e37afe4..89fc4441 100644 --- a/td-payload/src/arch/x86_64/apic.rs +++ b/td-payload/src/arch/x86_64/apic.rs @@ -2,9 +2,6 @@ // // SPDX-License-Identifier: BSD-2-Clause-Patent -pub use td_exception::interrupt::InterruptNoErrorStack; -pub use td_exception::{preserved_pop, preserved_push, scratch_pop, scratch_push}; - // MSR registers pub const MSR_LVTT: u32 = 0x832; pub const MSR_INITIAL_COUNT: u32 = 0x838; @@ -54,56 +51,3 @@ pub fn one_shot_tsc_deadline_mode(period: u64) -> Option { pub fn one_shot_tsc_deadline_mode_reset() { unsafe { x86::msr::wrmsr(MSR_TSC_DEADLINE, 0) } } - -#[macro_export] -macro_rules! interrupt_handler_template { - ($name:ident, $stack: ident, $func:block) => { - #[naked] - #[no_mangle] - /// # Safety - /// - /// Interrupt handler will handle the register reserve and restore - pub unsafe extern fn $name () { - #[inline(never)] - /// # Safety - /// - /// Interrupt handler will handle the register reserve and restore - unsafe extern "win64" fn inner($stack: &mut $crate::arch::apic::InterruptNoErrorStack) { - $func - } - - // Push scratch registers - core::arch::asm!( concat!( - $crate::arch::apic::scratch_push!(), - $crate::arch::apic::preserved_push!(), - " - mov rcx, rsp - call {inner} - ", - $crate::eoi!(), - $crate::arch::apic::preserved_pop!(), - $crate::arch::apic::scratch_pop!(), - " - iretq - " - ), - inner = sym inner, - options(noreturn), - ) - } - }; -} - -#[macro_export] -macro_rules! eoi { - // Write the end-of-interrupt (EOI) register (0x80B) at the end of the handler - // routine, sometime before the IRET instruction - () => { - " - mov rcx, 0x80B - mov edx, 0 - mov eax, 0 - wrmsr - " - }; -}