GPU Passthrough

Single

Edit /etc/default/grub, and update,


####
GRUB_CMDLINE_LINUX_DEFAULT="amd_iommu=on iommu=pt iommu=1 rd.driver.pre=vfio-pci quiet splash"
####
update-grub
reboot
cat /proc/cmdline # check if working

Find PCI Bus address for GPU and sound, for RTX there will be 4, GTX will be 2,


lspci -nnk | grep -i NVIDIA
## 09:00:0 --> 10de:1b81, 09:00.1 --> 10de:10f0, GTX 1070

Edit /etc/libvirt/libvirtd.conf,


####
unix_sock_group = "libvirt"
unix_sock_rw_perms = "0770"
log_filters="1:qemu"
log_outputs="1:file:/var/log/libvirt/libvirtd.log"
####
systemctl restart libvirt
systemctl restart libvirtd
sudo usermod -a -G libvirt $(whoami)
sudo systemctl start libvirtd
sudo systemctl enable libvirtd
sudo reboot

Configure /etc/libvirt/qemu.conf,


####
user = "yui"
group = "yui"
####
systemctl restart libvirtd
sudo usermod -a -G kvm "yui"
sudo usermod -a -G libvirt "yui"
sudo reboot

Create VM in QEMU/KVM Virt Manager.


V Customize configuration before install
Chipset Q35
Firmware UEFI x86_64:/usr/share/OVMF/OVMF_CODE.fd
# Begin installation
# Turn off, then delete
SATA CDROM, Tablet, Display Spice, SOund ich9, Video QXL, Channel spice
# Add hardware
PCI Host Devices for GPU, 4 for RTX, 2 for GTX

Create Libvirt Hook.


mkdir /etc/libvirt/hooks
touch /etc/libvirt/hooks/qemu
chmod +x /etc/libvirt/hooks/qemu
####
#!/bin/bash

GUEST_NAME="$1"
HOOK_NAME="$2"
STATE_NAME="$3"
MISC="${@:4}"

BASEDIR="$(dirname $0)"

HOOKPATH="$BASEDIR/qemu.d/$GUEST_NAME/$HOOK_NAME/$STATE_NAME"
set -e # If a script exits with an error, we should as well.

if [ -f "$HOOKPATH" ]; then
eval \""$HOOKPATH"\" "$@"
elif [ -d "$HOOKPATH" ]; then
while read file; do
  eval \""$file"\" "$@"
done <<< "$(find -L "$HOOKPATH" -maxdepth 1 -type f -executable -print;)"
fi
####

Create Start Script,


mkdir -p /etc/libvirt/hooks/qemu.d/win10/prepare/begin
touch /etc/libvirt/hooks/qemu.d/win10/prepare/begin/start.sh
chmod +x /etc/libvirt/hooks/qemu.d/win10/prepare/begin/start.sh
####
#!/bin/bash
set -x

# Stop display manager
sudo systemctl stop display-manager
# rc-service xdm stop

# Unbind EFI Framebuffer
echo efi-framebuffer.0 > /sys/bus/platform/drivers/efi-framebuffer/unbind

# Unload NVIDIA kernel modules
modprobe -r nvidia_drm nvidia_modeset nvidia_uvm nvidia drm_kms_helper drm i2c_nvidia_gpu

# Unload AMD kernel module
# modprobe -r amdgpu

# Detach GPU devices from host
# Use your GPU and HDMI Audio PCI host device
virsh nodedev-detach pci_0000_09_00_0
virsh nodedev-detach pci_0000_09_00_1
virsh nodedev-detach pci_0000_09_00_2
virsh nodedev-detach pci_0000_09_00_3

# Load vfio module
modprobe vfio
modprobe vfio-pci
modprobe vfio_iommu_type1
####

Create Stop Script,


mkdir -p /etc/libvirt/hooks/qemu.d/win10/release/end
touch /etc/libvirt/hooks/qemu.d/win10/release/end/stop.sh
chmod +x /etc/libvirt/hooks/qemu.d/win10/release/end/stop.sh
####
#!/bin/bash
set -x

# Unload vfio module
modprobe -r vfio_pci
modprobe -r vfio_iommu_type1
modprobe -r vfio

# Attach GPU devices to host
# Use your GPU and HDMI Audio PCI host device
virsh nodedev-reattach pci_0000_09_00_0
virsh nodedev-reattach pci_0000_09_00_1
virsh nodedev-reattach pci_0000_09_00_2
virsh nodedev-reattach pci_0000_09_00_3

# Rebind framebuffer to host
echo "efi-framebuffer.0" > /sys/bus/platform/drivers/efi-framebuffer/bind

# Load NVIDIA kernel modules
modprobe nvidia_drm
modprobe nvidia_modeset
modprobe nvidia_uvm
modprobe nvidia
modprobe drm_kms_helper
modprobe i2c_nvidia_gpu
modprobe drm

# Load AMD kernel module
# modprobe amdgpu

# Restart Display Manager
sudo systemctl start display-manager
# rc-service xdm start
####

Download VGA Bios for the Graphic Card used to passthrough, check more info from NVIDIA X Server Settings, like VBIOS Version


sudo mv ~/Downloads/xxx.rom /usr/share/vgabios/
sudo chmod -R 775 xxx.rom
sudo chown root:root xxx.rom
# For Pascal GPU, like 10xx series, one extra step is needed to convert rom,
git clone https://github.com/Matoking/NVIDIA-vBIOS-VFIO-Patcher.git
python nvidia_vbios_vfio_patcher.py -i xxx.rom -o yyy.rom

Edit virsh file for the specific vm,


virsh edit win10
####
# 1. Edit tag
## Before
<domain>>
## After 
<domain type='kvm' xmlns:qemu='http://libvirt.org/schemas/domain/qemu/1.0'>
####
# 2. Add mouse and keyboard as devices
## ls /dev/input/by-id to get event-mouse/kbd
...
  <qemu:commandline>
    <qemu:arg value='-object'/>
    <qemu:arg value='input-linux,id=mouse1,evdev=/dev/input/by-id/usb-1bcf_USB_Optical_Mouse-event-mouse'/>
    <qemu:arg value='-object'/>
    <qemu:arg value='input-linux,id=kbd1,evdev=/dev/input/by-id/usb-Usb_KeyBoard_Usb_KeyBoard-event-kbd,grab_all=on,repeat=on'/>
  </qemu:commandline>
</domain>
####
# 3. Change devices input to virtio
...
<devices>
  ...
  <input type='mouse' bus='virtio'/>
  <input type='keyboard' bus='virtio'/>
  ...
</devices>
...
####
4. Video card driver virtualization detection
...
<features>
  ...
  <hyperv>
    ...
    <vendor_id state='on' value='whatever'/>
    ...
  </hyperv>
  <kvm>
    <hidden state='on'/>
  </kvm>
  ...
</features>
####
5. vBIOs Patching
...
<hostdev mode='subsystem' type='pci' managed='yes'>
  <source>
    ...
  </source>
  <rom file='/usr/share/vgabios/output.rom'/>
  ...
</hostdev>
...
####
...

Edit qemu configuration file,


...
cgroup_device_acl = [
    "/dev/input/by-id/KEYBOARD_NAME",
    "/dev/input/by-id/MOUSE_NAME",
    "/dev/null", "/dev/full", "/dev/zero",
    "/dev/random", "/dev/urandom",
    "/dev/ptmx", "/dev/kvm", "/dev/kqemu",
    "/dev/rtc","/dev/hpet", "/dev/sev"
]
...

Change GPU

Download rom file and change rom path. (patch rom file if Pascal)
Add and remove necessary pcie hardware.
Login VM, should show now, then install nvidia driver for the changed GPU.

GPU Passthrough

Single

Change GPU

References