From a0070fa799872e2be6aa7ae7eba9eebe69dbd2ee Mon Sep 17 00:00:00 2001 From: Klaus Jensen Date: Thu, 26 Sep 2024 15:36:24 +0200 Subject: [PATCH] examples/cmb-p2p: fix the p2p example The cmb-p2p example is broken. Fix it. After mapping the BAR into virtual memory, map it in the iommu as well and use the resulting IOVA as the CMB Controller Base Address. Fixes: 4de6927088e3 ("add cmb-p2p example") Signed-off-by: Klaus Jensen --- examples/cmb-p2p.c | 55 ++++++++++++++++++++++++---------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/examples/cmb-p2p.c b/examples/cmb-p2p.c index bda13f03..75137526 100644 --- a/examples/cmb-p2p.c +++ b/examples/cmb-p2p.c @@ -16,6 +16,16 @@ * more details. */ +/* + * Basic example of doing DMA P2P using a CMB. + * + * This example expects two devices (a source and a destination) where the + * destination MUST have a CMB. For instance, testing with QEMU: + * + * -device nvme,serial=deadc0de,drive=... + * -device nvme,serial=deadbeef,drive=...,cmb_size_mb=1 + */ + #include #include @@ -49,15 +59,12 @@ static inline off_t bar_offset(int bar) return PCI_BASE_ADDRESS_0 + bar * 4; } -static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64_t *cba) +static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *iova, size_t *len) { - uint64_t cap, szu, ofst; + uint64_t cap, szu, ofst, hwaddr; uint32_t cmbloc, cmbsz, v32; - size_t len; int bir; void *cmb; - struct iommu_iova_range *iova_ranges; - int num_iova_ranges; cap = le64_to_cpu(mmio_read64(ctrl->regs + NVME_REG_CAP)); @@ -68,42 +75,39 @@ static void *nvme_configure_cmb(struct nvme_ctrl *ctrl, uint64_t *hwaddr, uint64 cmbloc = le32_to_cpu(mmio_read32(ctrl->regs + NVME_REG_CMBLOC)); szu = 1 << (12 + 4 * NVME_GET(cmbsz, CMBSZ_SZU)); - len = szu * NVME_GET(cmbsz, CMBSZ_SZ); + *len = szu * NVME_GET(cmbsz, CMBSZ_SZ); ofst = szu * NVME_GET(cmbloc, CMBLOC_OFST); bir = NVME_GET(cmbloc, CMBLOC_BIR); - printf("cmb bar is %d\n", bir); + printf("cmb bir is %d\n", bir); if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir)) < 0) err(1, "failed to read pci config"); - *hwaddr = v32 & PCI_BASE_ADDRESS_MEM_MASK; + hwaddr = v32 & PCI_BASE_ADDRESS_MEM_MASK; if (vfio_pci_read_config(&ctrl->pci, &v32, sizeof(v32), bar_offset(bir + 1)) < 0) err(1, "failed to read pci config"); - *hwaddr |= (uint64_t)v32 << 32; + hwaddr |= (uint64_t)v32 << 32; - printf("cmb bar is mapped at physical address 0x%lx\n", *hwaddr); + printf("cmb bar is mapped at physical address 0x%lx\n", hwaddr); /* map the cmb */ - cmb = vfio_pci_map_bar(&ctrl->pci, bir, len, ofst, PROT_READ | PROT_WRITE); + cmb = vfio_pci_map_bar(&ctrl->pci, bir, *len, ofst, PROT_READ | PROT_WRITE); if (!cmb) err(1, "failed to map cmb"); - if (!NVME_GET(cap, CAP_CMBS)) { - *cba = *hwaddr; - return cmb; - } + if (iommu_map_vaddr(__iommu_ctx(ctrl), cmb, *len, iova, 0x0)) + err(1, "failed to map cmb in iommu (try VFN_IOMMU_FORCE_VFIO=1)"); - /* choose a base address that is guaranteed not to be involved in dma */ - num_iova_ranges = iommu_get_iova_ranges(__iommu_ctx(ctrl), &iova_ranges); - *cba = ALIGN_UP(iova_ranges[num_iova_ranges - 1].last + 1, 4096); - printf("assigned cmb base address is 0x%lx\n", *cba); + if (NVME_GET(cap, CAP_CMBS)) { + printf("assigned cmb base address is 0x%lx\n", *iova); - /* set the base address and enable the memory space */ - mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*cba | 0x3)); + /* set the base address and enable the memory space */ + mmio_hl_write64(ctrl->regs + NVME_REG_CMBMSC, cpu_to_le64(*iova | 0x3)); + } return cmb; } @@ -112,7 +116,8 @@ int main(int argc, char **argv) { struct nvme_ctrl src = {}, dst = {}; - uint64_t hwaddr, cba; + uint64_t iova; + size_t len; void *cmb; union nvme_cmd cmd = {}; @@ -136,14 +141,12 @@ int main(int argc, char **argv) if (nvme_init(&dst, bdfs[1], &ctrl_opts)) err(1, "failed to initialize destination nvme controller"); - cmb = nvme_configure_cmb(&dst, &hwaddr, &cba); + cmb = nvme_configure_cmb(&dst, &iova, &len); cmd.identify = (struct nvme_cmd_identify) { .opcode = nvme_admin_identify, .cns = NVME_IDENTIFY_CNS_CTRL, - - /* external reference; use the system address */ - .dptr.prp1 = cpu_to_le64(hwaddr), + .dptr.prp1 = cpu_to_le64(iova), }; if (nvme_admin(&src, &cmd, NULL, 0, NULL))