stage2: add TD Partitioning support

Cargo.toml

@@ -32,16 +32,17 @@ aes-gcm = { version = "0.10.3", default-features = false }
 arbitrary = "1.3.0"
 bitfield-struct = "0.6.2"
 bitflags = "2.4"
-clap = { version = "4.4.14", default-features = false}
+clap = { version = "4.4.14", default-features = false }
 gdbstub = { version = "0.6.6", default-features = false }
 gdbstub_arch = { version = "0.2.4" }
-sha2 = "0.10.8"
-igvm_defs = { version = "0.3.2", default-features = false}
-igvm = { version = "0.3.2", default-features = false}
+igvm = { version = "0.3.2", default-features = false }
+igvm_defs = { version = "0.3.2", default-features = false }
 intrusive-collections = "0.9.6"
 libfuzzer-sys = "0.4"
 log = "0.4.17"
 p384 = { version = "0.13.0" }
+sha2 = "0.10.8"
+tdx-tdcall = "0.2.1"
 uuid = "1.6.1"
 # Add the derive feature by default because all crates use it.
 zerocopy = { version = "0.8.2", features = ["alloc", "derive"] }

igvmbuilder/src/igvm_builder.rs

@@ -428,6 +428,23 @@ impl IgvmBuilder {
                 IgvmPageDataType::SECRETS,
             )?;
         }
+        if COMPATIBILITY_MASK.contains(TDP_COMPATIBILITY_MASK) {
+            // Insert a zero page in place of the CPUID page
+            self.add_empty_pages(
+                self.gpa_map.cpuid_page.get_start(),
+                self.gpa_map.cpuid_page.get_size(),
+                TDP_COMPATIBILITY_MASK,
+                IgvmPageDataType::NORMAL,
+            )?;
+
+            // Insert a zero page in place of the secrets page
+            self.add_empty_pages(
+                self.gpa_map.secrets_page.get_start(),
+                self.gpa_map.secrets_page.get_size(),
+                TDP_COMPATIBILITY_MASK,
+                IgvmPageDataType::NORMAL,
+            )?;
+        }
 
         // Add optional stage 1 binary.
         if let Some(stage1) = &self.options.tdx_stage1 {

kernel/Cargo.toml

@@ -33,6 +33,7 @@ igvm_defs = { workspace = true, features = ["unstable"] }
 intrusive-collections.workspace = true
 log = { workspace = true, features = ["max_level_info", "release_max_level_info"] }
 packit.workspace = true
+tdx-tdcall.workspace = true
 libmstpm = { workspace = true, optional = true }
 zerocopy.workspace = true
 

kernel/src/mm/address_space.rs

@@ -5,11 +5,9 @@
 // Author: Joerg Roedel <[email protected]>
 
 use crate::address::{PhysAddr, VirtAddr};
+use crate::mm::pagetable::{PageFrame, PageTable};
 use crate::utils::immut_after_init::ImmutAfterInitCell;
 
-#[cfg(target_os = "none")]
-use crate::mm::pagetable::PageTable;
-
 #[derive(Debug, Copy, Clone)]
 #[allow(dead_code)]
 pub struct FixedAddressMappingRange {
@@ -67,7 +65,16 @@ pub fn init_kernel_mapping_info(
 
 #[cfg(target_os = "none")]
 pub fn virt_to_phys(vaddr: VirtAddr) -> PhysAddr {
-    match PageTable::virt_to_phys(vaddr) {
+    match PageTable::virt_to_frame(vaddr) {
+        Some(paddr) => paddr.address(),
+        None => {
+            panic!("Invalid virtual address {:#018x}", vaddr);
+        }
+    }
+}
+
+pub fn virt_to_frame(vaddr: VirtAddr) -> PageFrame {
+    match PageTable::virt_to_frame(vaddr) {
         Some(paddr) => paddr,
         None => {
             panic!("Invalid virtual address {:#018x}", vaddr);

kernel/src/mm/pagetable.rs

@@ -16,7 +16,7 @@ use crate::mm::{
     SVSM_PTE_BASE,
 };
 use crate::platform::SvsmPlatform;
-use crate::types::{PageSize, PAGE_SIZE, PAGE_SIZE_2M};
+use crate::types::{PageSize, PAGE_SIZE, PAGE_SIZE_1G, PAGE_SIZE_2M};
 use crate::utils::immut_after_init::{ImmutAfterInitCell, ImmutAfterInitResult};
 use crate::utils::MemoryRegion;
 use crate::BIT_MASK;
@@ -450,6 +450,41 @@ pub enum Mapping<'a> {
     Level0(&'a mut PTEntry),
 }
 
+/// A physical address within a page frame
+#[derive(Debug)]
+pub enum PageFrame {
+    Size4K(PhysAddr),
+    Size2M(PhysAddr),
+    Size1G(PhysAddr),
+}
+
+impl PageFrame {
+    pub fn address(&self) -> PhysAddr {
+        match *self {
+            Self::Size4K(pa) => pa,
+            Self::Size2M(pa) => pa,
+            Self::Size1G(pa) => pa,
+        }
+    }
+
+    fn size(&self) -> usize {
+        match self {
+            Self::Size4K(_) => PAGE_SIZE,
+            Self::Size2M(_) => PAGE_SIZE_2M,
+            Self::Size1G(_) => PAGE_SIZE_1G,
+        }
+    }
+
+    pub fn start(&self) -> PhysAddr {
+        let end = self.address().bits() & !(self.size() - 1);
+        end.into()
+    }
+
+    pub fn end(&self) -> PhysAddr {
+        self.start() + self.size()
+    }
+}
+
 /// Page table structure containing a root page with multiple entries.
 #[repr(C)]
 #[derive(Default, Debug)]
@@ -608,12 +643,12 @@ impl PageTable {
     /// Perform a virtual to physical translation using the self-map.
     ///
     /// # Parameters
-    /// - `vaddr': The virtual address to transalte.
+    /// - `vaddr': The virtual address to translate.
     ///
     /// # Returns
-    /// Some(PhysAddr) if the virtual address is valid.
+    /// Some(PageFrame) if the virtual address is valid.
     /// None if the virtual address is not valid.
-    pub fn virt_to_phys(vaddr: VirtAddr) -> Option<PhysAddr> {
+    pub fn virt_to_frame(vaddr: VirtAddr) -> Option<PageFrame> {
         // Calculate the virtual addresses of each level of the paging
         // hierarchy in the self-map.
         let pte_addr = Self::get_pte_address(vaddr);
@@ -640,20 +675,23 @@ impl PageTable {
             return None;
         }
         if pdpe.huge() {
-            return Some(pdpe.address() + (usize::from(vaddr) & 0x3FFF_FFFF));
+            let pa = pdpe.address() + (usize::from(vaddr) & 0x3FFF_FFFF);
+            return Some(PageFrame::Size1G(pa));
         }
 
         let pde = unsafe { PTEntry::read_pte(pde_addr) };
         if !pde.present() {
             return None;
         }
         if pde.huge() {
-            return Some(pde.address() + (usize::from(vaddr) & 0x001F_FFFF));
+            let pa = pde.address() + (usize::from(vaddr) & 0x001F_FFFF);
+            return Some(PageFrame::Size2M(pa));
         }
 
         let pte = unsafe { PTEntry::read_pte(pte_addr) };
         if pte.present() {
-            Some(pte.address() + (usize::from(vaddr) & 0xFFF))
+            let pa = pte.address() + (usize::from(vaddr) & 0xFFF);
+            Some(PageFrame::Size4K(pa))
         } else {
             None
         }

kernel/src/platform/mod.rs

@@ -98,7 +98,7 @@ pub trait SvsmPlatform {
 
     /// Marks a virtual range of pages as valid or invalid for use as private
     /// pages.  Provided primarily for use in stage2 where validation by
-    /// physical address cannot e supported.
+    /// physical address cannot be supported.
     fn validate_virtual_page_range(
         &self,
         region: MemoryRegion<VirtAddr>,

kernel/src/platform/tdp.rs

@@ -9,12 +9,18 @@ use crate::console::init_svsm_console;
 use crate::cpu::cpuid::CpuidResult;
 use crate::cpu::percpu::PerCpu;
 use crate::error::SvsmError;
-use crate::io::{IOPort, DEFAULT_IO_DRIVER};
+use crate::io::IOPort;
+use crate::mm::{virt_to_frame, PerCPUPageMappingGuard};
 use crate::platform::{PageEncryptionMasks, PageStateChangeOp, PageValidateOp, SvsmPlatform};
 use crate::types::PageSize;
 use crate::utils::immut_after_init::ImmutAfterInitCell;
-use crate::utils::MemoryRegion;
+use crate::utils::{zero_mem_region, MemoryRegion};
+use tdx_tdcall::tdx::{
+    td_accept_memory, tdvmcall_io_read_16, tdvmcall_io_read_32, tdvmcall_io_read_8,
+    tdvmcall_io_write_16, tdvmcall_io_write_32, tdvmcall_io_write_8,
+};
 
+static GHCI_IO_DRIVER: GHCIIOPort = GHCIIOPort::new();
 static VTOM: ImmutAfterInitCell<usize> = ImmutAfterInitCell::uninit();
 
 #[derive(Clone, Copy, Debug)]
@@ -33,12 +39,14 @@ impl Default for TdpPlatform {
 }
 
 impl SvsmPlatform for TdpPlatform {
-    fn env_setup(&mut self, _debug_serial_port: u16, vtom: usize) -> Result<(), SvsmError> {
-        VTOM.init(&vtom).map_err(|_| SvsmError::PlatformInit)
+    fn env_setup(&mut self, debug_serial_port: u16, vtom: usize) -> Result<(), SvsmError> {
+        VTOM.init(&vtom).map_err(|_| SvsmError::PlatformInit)?;
+        // Serial console device can be initialized immediately
+        init_svsm_console(&GHCI_IO_DRIVER, debug_serial_port)
     }
 
-    fn env_setup_late(&mut self, debug_serial_port: u16) -> Result<(), SvsmError> {
-        init_svsm_console(&DEFAULT_IO_DRIVER, debug_serial_port)
+    fn env_setup_late(&mut self, _debug_serial_port: u16) -> Result<(), SvsmError> {
+        Ok(())
     }
 
     fn env_setup_svsm(&self) -> Result<(), SvsmError> {
@@ -61,7 +69,7 @@ impl SvsmPlatform for TdpPlatform {
             private_pte_mask: 0,
             shared_pte_mask: vtom,
             addr_mask_width: vtom.trailing_zeros(),
-            phys_addr_sizes: res.eax & 0xff,
+            phys_addr_sizes: res.eax,
         }
     }
 
@@ -72,9 +80,7 @@ impl SvsmPlatform for TdpPlatform {
     fn setup_guest_host_comm(&mut self, _cpu: &PerCpu, _is_bsp: bool) {}
 
     fn get_io_port(&self) -> &'static dyn IOPort {
-        // FIXME - the default I/O port implementation doesn't work on TDP,
-        // but the platform does not yet have an alternative available.
-        &DEFAULT_IO_DRIVER
+        &GHCI_IO_DRIVER
     }
 
     fn page_state_change(
@@ -88,18 +94,42 @@ impl SvsmPlatform for TdpPlatform {
 
     fn validate_physical_page_range(
         &self,
-        _region: MemoryRegion<PhysAddr>,
-        _op: PageValidateOp,
+        region: MemoryRegion<PhysAddr>,
+        op: PageValidateOp,
     ) -> Result<(), SvsmError> {
-        Err(SvsmError::Tdx)
+        match op {
+            PageValidateOp::Validate => {
+                td_accept_memory(region.start().into(), region.len().try_into().unwrap());
+            }
+            PageValidateOp::Invalidate => {
+                let mapping = PerCPUPageMappingGuard::create(region.start(), region.end(), 0)?;
+                zero_mem_region(mapping.virt_addr(), mapping.virt_addr() + region.len());
+            }
+        }
+        Ok(())
     }
 
     fn validate_virtual_page_range(
         &self,
-        _region: MemoryRegion<VirtAddr>,
-        _op: PageValidateOp,
+        region: MemoryRegion<VirtAddr>,
+        op: PageValidateOp,
     ) -> Result<(), SvsmError> {
-        Err(SvsmError::Tdx)
+        match op {
+            PageValidateOp::Validate => {
+                let mut va = region.start();
+                while va < region.end() {
+                    let pa = virt_to_frame(va);
+                    let sz = pa.end() - pa.address();
+                    // td_accept_memory() will take care of alignment
+                    td_accept_memory(pa.address().into(), sz.try_into().unwrap());
+                    va = va + sz;
+                }
+            }
+            PageValidateOp::Invalidate => {
+                zero_mem_region(region.start(), region.end());
+            }
+        }
+        Ok(())
     }
 
     fn configure_alternate_injection(&mut self, _alt_inj_requested: bool) -> Result<(), SvsmError> {
@@ -124,3 +154,38 @@ impl SvsmPlatform for TdpPlatform {
         todo!();
     }
 }
+
+#[derive(Clone, Copy, Debug, Default)]
+struct GHCIIOPort {}
+
+impl GHCIIOPort {
+    pub const fn new() -> Self {
+        GHCIIOPort {}
+    }
+}
+
+impl IOPort for GHCIIOPort {
+    fn outb(&self, port: u16, value: u8) {
+        tdvmcall_io_write_8(port, value);
+    }
+
+    fn inb(&self, port: u16) -> u8 {
+        tdvmcall_io_read_8(port)
+    }
+
+    fn outw(&self, port: u16, value: u16) {
+        tdvmcall_io_write_16(port, value);
+    }
+
+    fn inw(&self, port: u16) -> u16 {
+        tdvmcall_io_read_16(port)
+    }
+
+    fn outl(&self, port: u16, value: u32) {
+        tdvmcall_io_write_32(port, value);
+    }
+
+    fn inl(&self, port: u16) -> u32 {
+        tdvmcall_io_read_32(port)
+    }
+}

kernel/src/stage2.rs

@@ -443,6 +443,8 @@ pub extern "C" fn stage2_main(launch_info: &Stage2LaunchInfo) {
 
     let valid_bitmap = valid_bitmap_addr();
 
+    log::info!("Starting SVSM kernel...");
+
     // Shut down the GHCB
     unsafe {
         shutdown_percpu();

kernel/src/types.rs

@@ -9,8 +9,10 @@ use crate::sev::vmsa::VMPL_MAX;
 
 pub const PAGE_SHIFT: usize = 12;
 pub const PAGE_SHIFT_2M: usize = 21;
+pub const PAGE_SHIFT_1G: usize = 30;
 pub const PAGE_SIZE: usize = 1 << PAGE_SHIFT;
-pub const PAGE_SIZE_2M: usize = PAGE_SIZE * 512;
+pub const PAGE_SIZE_2M: usize = 1 << PAGE_SHIFT_2M;
+pub const PAGE_SIZE_1G: usize = 1 << PAGE_SHIFT_1G;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum PageSize {

stage1/stage1.S

@@ -87,7 +87,7 @@ startup_32:
 	 * Stage 2 launch info has been prepared
 	 * Make sure platform type is TDP
 	 */
-	movl	$(STAGE1_STACK - 32), %eax
+	movl	$(STAGE1_STACK - 40), %eax
 	movl	(%eax), %eax
 	cmpl	$2, %eax
 	je	.Lsetup_td
@@ -104,7 +104,7 @@ startup_32:
 
 .Lsetup_bsp_stack:
 	/* Set up BSP stack for stage 2 */
-	movl	$(STAGE1_STACK - 40), %esp
+	movl	$(STAGE1_STACK - 48), %esp
 	/* %ebx is initialized with GPAW - save (1u64 << (GPAW - 1)) to vtom */
 	mov	%esp, %eax
 	/* GPAW must be either 48 or 52 */

stage1/stage1.lds

@@ -10,7 +10,6 @@ OUTPUT_ARCH(i386:x86-64)
 
 SECTIONS
 {
-	. = 0x800000;
 	.stext = ALIGN(.sdata  - SIZEOF(.text) - 4095, 4096);
 	. = .stext;
 	.text : { *(.startup.*) *(.text) *(.text.*) }