# cp /etc/default/grub /etc/default/grub.org
# sed -i ‘s@GRUB_CMDLINE_LINUX=”@GRUB_CMDLINE_LINUX=”modprobe.blacklist=nouveau @g’ /etc/default/grub

5-2. 시작 시 Nouveau 로드되지 않도록 설정

# cat << EOL >> /etc/modprobe.d/nouveau-blacklist.conf
blacklist nouveau
options nouveau modeset=0
EOL

5-3. GRUB2(GRand Unified Bootloader version 2)의 구성 파일 업데이트

5-3-1. BIOS(Basic Input/Output System)

# grub2-mkconfig -o /boot/grub2/grub.cfg

5-3-2. UEFI(Unified Extensible Firmware Interface)

# grub2-mkconfig -o /boot/efi/EFI/rocky/grub.cfg

5-4. 재부팅

# systemctl reboot

6. NVIDIA

6-1. NVIDIA driver 설치

vGPU Trial URL

# mkdir NVIDIA
# mv NVIDIA-GRID-Linux-KVM-550.54.16-550.54.15-551.78.zip NVIDIA
# cd NVIDIA
# unzip NVIDIA-GRID-Linux-KVM-550.54.16-550.54.15-551.78.zip
# cd Host_Drivers/
# sh NVIDIA-Linux-x86_64-550.54.16-vgpu-kvm.run

6-2. vGPU 설정

6-2-1. 커널 모듈 확인

# lsmod | grep nvidia_vgpu_vfio

nvidia_vgpu_vfio       73728  0
nvidia               8597504  1 nvidia_vgpu_vfio
mdev                   24576  2 vfio_mdev,nvidia_vgpu_vfio
vfio                   36864  3 vfio_mdev,nvidia_vgpu_vfio,vfio_iommu_type1
kvm                   970752  2 nvidia_vgpu_vfio,kvm_intel
irqbypass              16384  2 nvidia_vgpu_vfio,kvm

6-2-2. vGPU 관리 데몬 실행

# systemctl start nvidia-vgpu-mgr.service
# systemctl enable nvidia-vgpu-mgr.service
# systemctl status nvidia-vgpu-mgr.service

● nvidia-vgpu-mgr.service - NVIDIA vGPU Manager Daemon
   Loaded: loaded (/usr/lib/systemd/system/nvidia-vgpu-mgr.service; enabled; vendor preset: disabled)
   Active: active (running) since Tue 2024-06-04 07:33:16 EDT; 14min ago
 Main PID: 1768 (nvidia-vgpu-mgr)
    Tasks: 1 (limit: 2470715)
   Memory: 36.4M
   CGroup: /system.slice/nvidia-vgpu-mgr.service
           └─1768 /usr/bin/nvidia-vgpu-mgr

Jun 04 07:33:16 KVM02 systemd[1]: Starting NVIDIA vGPU Manager Daemon...
Jun 04 07:33:16 KVM02 systemd[1]: Started NVIDIA vGPU Manager Daemon.
Jun 04 07:33:18 KVM02 nvidia-vgpu-mgr[1768]: notice: vmiop_env_log: nvidia-vgpu-mgr daemon started

6-2-3. 사용 가능한 vGPU 장치 확인

# nvidia-smi vgpu -s

GPU 00000000:98:00.0
NVIDIA L40S-1B
NVIDIA L40S-2B
NVIDIA L40S-1Q
NVIDIA L40S-2Q
NVIDIA L40S-3Q
NVIDIA L40S-4Q
NVIDIA L40S-6Q
NVIDIA L40S-8Q
NVIDIA L40S-12Q
NVIDIA L40S-16Q
NVIDIA L40S-24Q
NVIDIA L40S-48Q
NVIDIA L40S-1A
NVIDIA L40S-2A
NVIDIA L40S-3A
NVIDIA L40S-4A
NVIDIA L40S-6A
NVIDIA L40S-8A
NVIDIA L40S-12A
NVIDIA L40S-16A
NVIDIA L40S-24A
NVIDIA L40S-48A

6-3. IOMMU 활성

# vim /etc/default/grub

GRUB_TIMEOUT=5
GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)"
GRUB_DEFAULT=saved
GRUB_DISABLE_SUBMENU=true
GRUB_TERMINAL_OUTPUT="console"
+ GRUB_CMDLINE_LINUX="modprobe.blacklist=nouveau crashkernel=auto resume=UUID=14d180db-ae42-4cfd-a571-e268fce2fd96 intel_iommu=on rhgb quiet"
GRUB_DISABLE_RECOVERY="true"
GRUB_ENABLE_BLSCFG=true

# grub2-mkconfig -o /boot/efi/EFI/rocky/grub.cfg
# reboot

6-4. SR-IOV

6-4-1. SR-IOV 활성화

# nvidia-smi vgpu

Tue Jun  4 07:48:35 2024
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 550.54.16              Driver Version: 550.54.16                 |
|---------------------------------+------------------------------+------------+
| GPU  Name                       | Bus-Id                       | GPU-Util   |
|      vGPU ID     Name           | VM ID     VM Name            | vGPU-Util  |
|=================================+==============================+============|
|   0  NVIDIA L40S                | 00000000:98:00.0             |   0%       |
+---------------------------------+------------------------------+------------+

# /usr/lib/nvidia/sriov-manage -e 00000000:98:00.0

Enabling VFs on 0000:98:00.0

# echo ‘/usr/lib/nvidia/sriov-manage -e 00000000:98:00.0’ >> /etc/rc.local

6-4-2. SR-IOV 활성화된 GPU 장치의 VFS 개수 확인

# ls -al /sys/bus/pci/devices/0000:98:00.0/ | grep virtfn

6-5. vGPU 설정

6-5-1. Mediated Device 유형 이름 확인

# cat /sys/bus/pci/devices/0000:98:00.0/virtfn0/mdev_supported_types/nvidia-1152/name

NVIDIA L40S-8Q

6-5-2. Mediated Device 유형의 사용 가능한 인스턴스 수 확인

# cat /sys/bus/pci/devices/0000:98:00.0/virtfn0/mdev_supported_types/nvidia-1152/available_instances

6-5-3. Mediated Device 생성

# UUID=$(uuidgen)
# echo $UUID > /sys/bus/pci/devices/0000:98:00.0/virtfn0/mdev_supported_types/nvidia-1152/create
# cat /sys/bus/pci/devices/0000:98:00.0/virtfn0/mdev_supported_types/nvidia-1152/available_instances

6-5-4. Mediated Device 인스턴스의 수 확인

# ls /sys/bus/mdev/devices/ | wc -l

6-5-5. Mediated Device 인스턴스 추가 생성

# UUID=$(uuidgen)
# echo $UUID > /sys/bus/pci/devices/0000:98:00.0/virtfn1/mdev_supported_types/nvidia-1152/create
# ls /sys/bus/mdev/devices/ | wc -l