Test BPFire in KVM virtual machine with SR-IOV provisioned

[vincentmli]

Host Ubuntu 22.04 grub setting

GRUB_CMDLINE_LINUX_DEFAULT="intel_iommu=on"
echo 2 > /sys/class/net/enp4s0f0/device/sriov_numvfs
echo 2 > /sys/class/net/enp4s0f1/device/sriov_numvfs
#echo 1 >  /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts
#echo 1 >  /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts

root@r730:~# ip l list

7: enp4s0f0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether e8:ea:6a:06:1b:1a brd ff:ff:ff:ff:ff:ff
    vf 0     link/ether 2a:bc:10:1e:9d:b7 brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust off, query_rss off
    vf 1     link/ether 1e:ab:db:2d:a6:9d brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust off, query_rss off
8: enp7s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether 00:1b:21:50:bc:39 brd ff:ff:ff:ff:ff:ff
9: enp4s0f1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
    link/ether e8:ea:6a:06:1b:1b brd ff:ff:ff:ff:ff:ff
    vf 0     link/ether 72:a7:43:f9:7a:fe brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust off, query_rss off
    vf 1     link/ether d2:da:25:a7:25:cf brd ff:ff:ff:ff:ff:ff, spoof checking on, link-state auto, trust off, query_rss off

virtual machine libvirt xml

<domain type='kvm'>
  <name>ipfiretest</name>
  <uuid>f2d408df-454a-4be5-8651-8142bd3b6942</uuid>
  <memory unit='KiB'>8109568</memory>
  <currentMemory unit='KiB'>8108864</currentMemory>
  <vcpu placement='static'>4</vcpu>
  <resource>
    <partition>/machine</partition>
  </resource>
  <os>
    <type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
    <boot dev='hd'/>
  </os>
  <features>
    <acpi/>
    <apic/>
    <pae/>
  </features>
  <cpu mode='custom' match='exact' check='full'>
    <model fallback='forbid'>qemu64</model>
    <feature policy='require' name='x2apic'/>
    <feature policy='require' name='popcnt'/>
    <feature policy='require' name='hypervisor'/>
    <feature policy='require' name='lahf_lm'/>
    <feature policy='disable' name='svm'/>
    <feature policy='disable' name='abm'/>
    <feature policy='disable' name='sse4a'/>
  </cpu>
  <clock offset='utc'/>
  <on_poweroff>destroy</on_poweroff>
  <on_reboot>restart</on_reboot>
  <on_crash>restart</on_crash>
  <devices>
    <emulator>/usr/bin/kvm-spice</emulator>
    <disk type='file' device='disk'>
      <driver name='qemu' type='qcow2'/>
      <source file='/mnt/sdb1/images/ipfire-bpf.qcow2'/>
      <backingStore/>
      <target dev='vda' bus='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
    </disk>
    <controller type='usb' index='0' model='piix3-uhci'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
    </controller>
    <controller type='pci' index='0' model='pci-root'/>
    <controller type='ide' index='0'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
    </controller>
    <interface type='bridge'>
      <mac address='52:54:00:dd:62:26'/>
      <source bridge='br0'/>
      <model type='virtio'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='52:54:00:92:02:63'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x04' slot='0x10' function='0x0'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'/>
    </interface>
    <interface type='hostdev' managed='yes'>
      <mac address='52:54:00:dd:6c:53'/>
      <source>
        <address type='pci' domain='0x0000' bus='0x04' slot='0x10' function='0x1'/>
      </source>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x08' function='0x0'/>
    </interface>
    <serial type='pty'>
      <target type='isa-serial' port='0'>
        <model name='isa-serial'/>
      </target>
    </serial>
    <console type='pty'>
      <target type='serial' port='0'/>
    </console>
    <input type='mouse' bus='ps2'/>
    <input type='keyboard' bus='ps2'/>
    <graphics type='vnc' port='-1' autoport='yes' listen='127.0.0.1'>
      <listen type='address' address='127.0.0.1'/>
    </graphics>
    <audio id='1' type='none'/>
    <video>
      <model type='cirrus' vram='16384' heads='1' primary='yes'/>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
    </video>
    <memballoon model='virtio'>
      <address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'/>
    </memballoon>
  </devices>
  <seclabel type='dynamic' model='apparmor' relabel='yes'/>
  <seclabel type='dynamic' model='dac' relabel='yes'/>
</domain>

virsh start ipfiretest may got error like:

error: Failed to start domain 'ipfiretest'
error: internal error: qemu unexpectedly closed the monitor: /usr/bin/kvm-spice: W: this is an old compat wrapper script for kvm-spice
/usr/bin/kvm-spice: W: please use qemu-system-x86_64 instead of /usr/bin/kvm-spice
2024-04-25T16:53:18.491941Z qemu-system-x86_64: -device vfio-pci,host=0000:04:10.0,id=hostdev0,bus=pci.0,addr=0x7: vfio 0000:04:10.0: failed to setup container for group 43: Failed to set iommu for container: Operation not permitted

fix it by:

root@r730:~# echo 1 >  /sys/module/vfio_iommu_type1/parameters/allow_unsafe_interrupts