From c2a65758e2a56ed26f9210b4842fbdd93f9c6a65 Mon Sep 17 00:00:00 2001
From: Julie-CAO <julie_caohe@163.com>
Date: Thu, 11 Jul 2024 15:23:16 +0800
Subject: [PATCH] Support vgpu automation test

---
 .../autoyast/dev_host_12.xml.ep               |   2 +-
 .../autoyast/dev_host_15.xml.ep               |  25 +-
 data/virt_autotest/vgpu_guest_logging.sh      |  11 +
 lib/ipmi_backend_utils.pm                     |   4 +-
 lib/virt_autotest/utils.pm                    |  29 +-
 products/sle/main.pm                          |   1 +
 tests/virt_autotest/install_package.pm        |  40 +-
 tests/virt_autotest/login_console.pm          |   4 +-
 tests/virt_autotest/vgpu.pm                   | 356 ++++++++++++++++++
 9 files changed, 421 insertions(+), 51 deletions(-)
 create mode 100644 data/virt_autotest/vgpu_guest_logging.sh
 create mode 100644 tests/virt_autotest/vgpu.pm

diff --git a/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_12.xml.ep b/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_12.xml.ep
index b991b02f2f45..8a732e98628b 100644
--- a/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_12.xml.ep
+++ b/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_12.xml.ep
@@ -36,7 +36,7 @@
       <gfxmode>auto</gfxmode>
       <hiddenmenu>false</hiddenmenu>
       <os_prober>false</os_prober>
-      <serial>serial --speed=115200 --unit=<%= $get_var->('SERIALDEV') =~ s/ttyS//r %> --word=8 --parity=no --stop=1</serial>
+      <serial>serial --speed=115200 --unit=<%= $get_var->('SERIALDEV', 'ttyS1') =~ s/ttyS//r %> --word=8 --parity=no --stop=1</serial>
       <timeout config:type="integer">15</timeout>
       <trusted_grub>false</trusted_grub>
       <secure_boot>false</secure_boot>
diff --git a/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_15.xml.ep b/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_15.xml.ep
index 8d2ffa9dc7f6..ec4e91eb9e6e 100644
--- a/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_15.xml.ep
+++ b/data/virt_autotest/host_unattended_installation_files/autoyast/dev_host_15.xml.ep
@@ -49,12 +49,12 @@
       <hiddenmenu>false</hiddenmenu>
       <os_prober>false</os_prober>
       <secure_boot>false</secure_boot>
-      <serial>serial --speed=115200 --unit=<%= $get_var->('SERIALDEV') =~ s/ttyS//r %> --word=8 --parity=no --stop=1</serial>
+      <serial>serial --speed=115200 --unit=<%= $get_var->('SERIALDEV', 'ttyS1') =~ s/ttyS//r %> --word=8 --parity=no --stop=1</serial>
       <timeout config:type="integer">15</timeout>
       <trusted_grub>false</trusted_grub>
       <terminal>console</terminal>
       % if ($check_var->('SYSTEM_ROLE', 'xen')) {
-      <xen_append>splash=silent quiet console=tty <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "loglvl=all guest_loglvl=all" : "loglvl=debug guest_loglvl=debug" %> <%= defined $bmwqemu::vars{"OPT_KERNEL_PARAMS"} ? $bmwqemu::vars{"OPT_KERNEL_PARAMS"} : "" %></xen_append>
+      <xen_append>splash=silent quiet console=tty <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "loglvl=all guest_loglvl=all" : "loglvl=debug guest_loglvl=debug" %> <%= $get_var->('OPT_KERNEL_PARAMS') %></xen_append>
       <!-- For some special beremetal machines, such as unreal2/3, their serial console:
       SERIALDEV='ttyS2', XEN_SERIAL_CONSOLE="com1=115200,8n1,0x3e8,5 console=com1" -->
       <xen_kernel_append><%= defined $bmwqemu::vars{"XEN_SERIAL_CONSOLE"} ? $bmwqemu::vars{"XEN_SERIAL_CONSOLE"} : "console=com2,115200" %> vga=gfx-1024x768x16 <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "loglvl=all guest_loglvl=all" : "loglvl=debug guest_loglvl=debug" %> sync_console</xen_kernel_append>
@@ -64,15 +64,15 @@
           % if ($check_var->('VIRT_SEV_ES_GUEST_INSTALL', '1')) {
             <!-- Do not add mem_encrypt=on option on 15-SP6 KVM host due to bsc#1224107 -->
             % if ($check_var->('VERSION', '15-SP6')) { 
-      <append>splash=silent console=<%= $get_var->('SERIALDEV') %>,115200 console=tty kvm_amd.sev=1 <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= defined $bmwqemu::vars{"OPT_KERNEL_PARAMS"} ? $bmwqemu::vars{"OPT_KERNEL_PARAMS"} : "" %></append>
+      <append>splash=silent console=<%= $get_var->('SERIALDEV', 'ttyS1') %>,115200 console=tty kvm_amd.sev=1 <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= $get_var->('OPT_KERNEL_PARAMS') %></append>
             % } else {
-      <append>splash=silent console=<%= $get_var->('SERIALDEV') %>,115200 console=tty mem_encrypt=on kvm_amd.sev=1 <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= defined $bmwqemu::vars{"OPT_KERNEL_PARAMS"} ? $bmwqemu::vars{"OPT_KERNEL_PARAMS"} : "" %></append>
+      <append>splash=silent console=<%= $get_var->('SERIALDEV', 'ttyS1') %>,115200 console=tty mem_encrypt=on kvm_amd.sev=1 <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= $get_var->('OPT_KERNEL_PARAMS') %></append>
             % }
           % } else {
-      <append>splash=silent console=<%= $get_var->('SERIALDEV') %>,115200 console=tty <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= defined $bmwqemu::vars{"OPT_KERNEL_PARAMS"} ? $bmwqemu::vars{"OPT_KERNEL_PARAMS"} : "" %></append>
+      <append>splash=silent console=<%= $get_var->('SERIALDEV', 'ttyS1') %>,115200 console=tty <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= $get_var->('OPT_KERNEL_PARAMS') %></append>
           % }
         % } else { 
-      <append>splash=silent console=<%= $get_var->('SERIALDEV') %>,115200 console=tty intel_iommu=on <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= defined $bmwqemu::vars{"OPT_KERNEL_PARAMS"} ? $bmwqemu::vars{"OPT_KERNEL_PARAMS"} : "" %></append>
+      <append>splash=silent console=<%= $get_var->('SERIALDEV', 'ttyS1') %>,115200 console=tty intel_iommu=on <%= defined $bmwqemu::vars{"ENABLE_CONSOLE_KERNEL_LOG"} ? "ignore_loglevel" : "loglevel=5" %> <%= $get_var->('OPT_KERNEL_PARAMS') %></append>
         % }
       % }
     </global>
@@ -278,6 +278,19 @@
     </user>
   </users>
   <scripts>
+    <chroot-scripts config:type="list">
+      <script>
+        <!-- Keep ssh connection alive for long-time run test -->
+        <chrooted t="boolean">true</chrooted>
+        <filename>configure_sshd.sh</filename>
+        <interpreter>/bin/bash -x</interpreter>
+        <source><![CDATA[
+sshd_config_file="/etc/ssh/sshd_config.d/01-virt-test.conf"
+echo -e "TCPKeepAlive yes\nClientAliveInterval 60\nClientAliveCountMax 60" > $sshd_config_file
+]]>
+        </source>
+      </script>
+    </chroot-scripts>
     <post-scripts config:type="list">
       % if ($check_var->('SYSTEM_ROLE', 'xen') && $check_var->('XEN_DEFAULT_BOOT_IS_SET', 1)) {
       <script>
diff --git a/data/virt_autotest/vgpu_guest_logging.sh b/data/virt_autotest/vgpu_guest_logging.sh
new file mode 100644
index 000000000000..a9ebd5b40cff
--- /dev/null
+++ b/data/virt_autotest/vgpu_guest_logging.sh
@@ -0,0 +1,11 @@
+#!/bin/bash -x
+#vgpu test guest debugging
+date
+echo ""
+lspci
+echo ""
+lsmod | grep -i -e nvidia -e kvm
+echo ""
+nvidia-smi
+echo ""
+journalctl --cursor-file /tmp/cursor.txt | grep -i -e 'kernel:' -e nvidia
diff --git a/lib/ipmi_backend_utils.pm b/lib/ipmi_backend_utils.pm
index c62fbd835194..4713508022b9 100644
--- a/lib/ipmi_backend_utils.pm
+++ b/lib/ipmi_backend_utils.pm
@@ -121,8 +121,8 @@ sub setup_console_in_grub {
     elsif (${virt_type} eq "kvm") {
         #enable Intel VT-d for SR-IOV test running on intel SUTs
         my $intel_option = "";
-        if (get_var("ENABLE_SRIOV_NETWORK_CARD_PCI_PASSTHROUGH") && script_run("grep Intel /proc/cpuinfo") == 0) {
-            $intel_option = "intel_iommu=on";
+        if (get_var("ENABLE_SRIOV_NETWORK_CARD_PCI_PASSTHROUGH") or get_var("VGPU_TEST")) {
+            $intel_option = "intel_iommu=on" if script_run("grep Intel /proc/cpuinfo") == 0;
         }
 
         $cmd
diff --git a/lib/virt_autotest/utils.pm b/lib/virt_autotest/utils.pm
index 7588c2ae7b7b..9cedb1222695 100644
--- a/lib/virt_autotest/utils.pm
+++ b/lib/virt_autotest/utils.pm
@@ -26,12 +26,12 @@ use Utils::Architectures;
 use IO::Socket::INET;
 use Carp;
 
-our @EXPORT = qw(is_vmware_virtualization is_hyperv_virtualization is_fv_guest is_pv_guest guest_is_sle is_guest_ballooned is_xen_host is_kvm_host reset_log_cursor check_failures_in_journal check_host_health check_guest_health
-  is_monolithic_libvirtd turn_on_libvirt_debugging_log
-  print_cmd_output_to_file ssh_setup ssh_copy_id create_guest import_guest install_default_packages ensure_default_net_is_active ensure_guest_started
-  ensure_online add_guest_to_hosts restart_libvirtd check_libvirtd remove_additional_disks remove_additional_nic collect_virt_system_logs shutdown_guests wait_guest_online start_guests restore_downloaded_guests save_original_guest_xmls restore_original_guests save_guests_xml_for_change restore_xml_changed_guests
-  is_guest_online wait_guests_shutdown remove_vm setup_common_ssh_config add_alias_in_ssh_config parse_subnet_address_ipv4 backup_file manage_system_service setup_rsyslog_host
-  check_port_state is_registered_system do_system_registration check_system_registration subscribe_extensions_and_modules download_script download_script_and_execute is_sev_es_guest upload_virt_logs recreate_guests download_vm_import_disks enable_nm_debug check_activate_network_interface upload_nm_debug_log restart_modular_libvirt_daemons check_modular_libvirt_daemons get_guest_regcode);
+our @EXPORT = qw(is_vmware_virtualization is_hyperv_virtualization is_fv_guest is_pv_guest is_sev_es_guest guest_is_sle is_guest_ballooned is_xen_host is_kvm_host
+  is_monolithic_libvirtd turn_on_libvirt_debugging_log restart_libvirtd check_libvirtd restart_modular_libvirt_daemons check_modular_libvirt_daemons
+  reset_log_cursor check_failures_in_journal check_host_health check_guest_health print_cmd_output_to_file collect_virt_system_logs setup_rsyslog_host download_script download_script_and_execute upload_virt_logs enable_nm_debug upload_nm_debug_log
+  ssh_setup setup_common_ssh_config add_alias_in_ssh_config install_default_packages parse_subnet_address_ipv4 backup_file manage_system_service check_port_state is_registered_system do_system_registration check_system_registration subscribe_extensions_and_modules check_activate_network_interface wait_for_host_reboot
+  create_guest import_guest ssh_copy_id add_guest_to_hosts ensure_default_net_is_active ensure_guest_started remove_additional_disks remove_additional_nic start_guests is_guest_online ensure_online wait_guest_online restore_downloaded_guests save_original_guest_xmls restore_original_guests save_guests_xml_for_change restore_xml_changed_guests shutdown_guests wait_guests_shutdown remove_vm recreate_guests download_vm_import_disks get_guest_regcode
+);
 
 my %log_cursors;
 
@@ -1274,4 +1274,21 @@ sub get_guest_regcode {
     return $regcode, $regcode_ltss;
 }
 
+sub wait_for_host_reboot {
+    select_console 'sol', await_console => 0;
+    # Wait for reboot and show screenshots
+    foreach (1 .. 10) {
+        save_screenshot;
+        sleep 20;
+    }
+    assert_screen([qw(sol-console-wait-typing-ret linux-login text-login)], 120);
+    if (match_has_tag('sol-console-wait-typing-ret')) {
+        send_key 'ret';
+        assert_screen([qw(inux-login text-login)], 120);
+    }
+    record_info("Host rebooted");
+    reset_consoles;
+    select_console('root-ssh');
+}
+
 1;
diff --git a/products/sle/main.pm b/products/sle/main.pm
index 625bad026a3d..dcb5d37925fa 100644
--- a/products/sle/main.pm
+++ b/products/sle/main.pm
@@ -620,6 +620,7 @@ sub load_virt_feature_tests {
     }
     loadtest "virt_autotest/xen_guest_irqbalance" if get_var("ENABLE_XEN_GUEST_IRQBALANCE");
     loadtest "virt_autotest/sriov_network_card_pci_passthrough" if get_var("ENABLE_SRIOV_NETWORK_CARD_PCI_PASSTHROUGH");
+    loadtest "virt_autotest/vgpu" if get_var("VGPU_TEST");
     if (get_var('ENABLE_HOTPLUGGING')) {
         loadtest 'virtualization/universal/hotplugging_guest_preparation';
         loadtest 'virtualization/universal/hotplugging_network_interfaces';
diff --git a/tests/virt_autotest/install_package.pm b/tests/virt_autotest/install_package.pm
index 59b4f4f2be58..7fb2879e00d9 100644
--- a/tests/virt_autotest/install_package.pm
+++ b/tests/virt_autotest/install_package.pm
@@ -15,7 +15,7 @@ use Utils::Architectures;
 use version_utils qw(is_sle);
 use virt_utils;
 use utils;
-use Utils::Backends 'is_remote_backend';
+use Utils::Backends qw(use_ssh_serial_console is_remote_backend);
 use virt_autotest::utils qw(is_xen_host subscribe_extensions_and_modules);
 
 sub install_package {
@@ -36,39 +36,6 @@ sub install_package {
         zypper_call("--no-gpg-checks ar -f '$qa_server_repo' server-repo");
     }
 
-    #workaround for dependency on xmlstarlet for qa_lib_virtauto on sles11sp4 and sles12sp1
-    #workaround for dependency on bridge-utils for qa_lib_virtauto on sles15sp0
-    my $repo_0_to_install = get_var("REPO_0_TO_INSTALL", '');
-    my $dependency_repo = '';
-    my $dependency_rpms = '';
-    if ($repo_0_to_install =~ /SLES-11-SP4/m) {
-        $dependency_repo = 'http://download.suse.de/ibs/SUSE:/SLE-11:/Update/standard/';
-        $dependency_rpms = 'xmlstarlet';
-    }
-    elsif ($repo_0_to_install =~ /SLE-12-SP1/m) {
-        $dependency_repo = 'http://download.suse.de/ibs/SUSE:/SLE-12:/Update/standard/';
-        $dependency_rpms = 'xmlstarlet';
-    }
-    elsif ($repo_0_to_install =~ /SLE-15-Installer/m) {
-        $dependency_repo = 'http://download.suse.de/ibs/SUSE:/SLE-15:/GA/standard/';
-        $dependency_rpms = 'bridge-utils';
-    }
-
-    if ($dependency_repo) {
-        if (is_s390x) {
-            lpar_cmd("zypper --non-interactive --no-gpg-checks ar -f ${dependency_repo} dependency_repo");
-            lpar_cmd("zypper --non-interactive --gpg-auto-import-keys ref");
-            lpar_cmd("zypper --non-interactive in $dependency_rpms");
-            lpar_cmd("zypper --non-interactive rr dependency_repo");
-        }
-        else {
-            zypper_call("--no-gpg-checks ar -f ${dependency_repo} dependency_repo");
-            zypper_call("--gpg-auto-import-keys ref", 180);
-            zypper_call("in $dependency_rpms");
-            zypper_call("rr dependency_repo");
-        }
-    }
-
     #Install KVM role patterns for aarch64 virtualization host
     if (is_remote_backend && is_aarch64) {
         zypper_call("--gpg-auto-import-keys ref", timeout => 180);
@@ -119,6 +86,11 @@ sub install_package {
 }
 
 sub run {
+    # Only for x86_64
+    if (is_x86_64) {
+        select_console 'sol', await_console => 0;
+        use_ssh_serial_console;
+    }
     install_package;
 }
 
diff --git a/tests/virt_autotest/login_console.pm b/tests/virt_autotest/login_console.pm
index 0b1d197613a6..ce10a6171714 100644
--- a/tests/virt_autotest/login_console.pm
+++ b/tests/virt_autotest/login_console.pm
@@ -261,8 +261,8 @@ sub login_to_console {
 
     # Set ssh console timeout for virt tests on ipmi backend machines
     # it will make ssh serial console alive even with long time command
-    # For TW hosts, sshd configurations have been created in its autoyast profiles
-    if (is_remote_backend and is_x86_64 and get_var('VIRT_AUTOTEST', '')) {
+    # For SLE15 and TW autoyast installation, sshd configurations have been created in its autoyast profiles
+    if (is_remote_backend and is_x86_64 and get_var('VIRT_AUTOTEST') and !(is_sle('15+') and get_var('AUTOYAST'))) {
         if (is_sle) {
             set_ssh_console_timeout_before_use;
         }
diff --git a/tests/virt_autotest/vgpu.pm b/tests/virt_autotest/vgpu.pm
new file mode 100644
index 000000000000..942b0ffe02e4
--- /dev/null
+++ b/tests/virt_autotest/vgpu.pm
@@ -0,0 +1,356 @@
+# SUSE's openQA tests
+# Copyright 2024 SUSE LLC
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Summary: A test to pass an NVIDIA vGPU to guest via libvirt. Only KVM hosts are supported.
+# Test environment: one AMPERE GPU card, such A10, A30 etc., with SR-IOV capability and/or MIG features in host machine;
+#                   A UEFI guest vm is defined in the host, and they are ssh accessible from host.
+# Test flow:
+#    - install the NVIDIA vGPU manager on host and reboot
+#    - create a GPU Instance and a Compute Instance if MIG feature is available
+#    - create a vGPU device in host
+#    - assign the vGPU device to guest vm
+#    - install the vGPU driver in guest vm
+#    - detach vGPU from guest and remove the vGPU
+# Maintainer: qe-virt@suse.de, Julie CAO <JCao@suse.com>
+
+use base "virt_feature_test_base";
+use strict;
+use warnings;
+use utils;
+use testapi;
+use virt_autotest::common;
+use version_utils qw(is_sle);
+use virt_autotest::utils;
+use Utils::Backends qw(use_ssh_serial_console is_remote_backend);
+use ipmi_backend_utils qw(reconnect_when_ssh_console_broken);
+
+our $log_dir = "/tmp/vgpu";
+our $vm_xml_save_dir = "/tmp/download_vm_xml";
+
+sub run_test {
+    my $self = shift;
+
+    # Use sol & ssh console explixitly in case test module run on an installed host
+    # thus no login_console.pm has been required any more
+    select_console 'sol', await_console => 0;
+    use_ssh_serial_console;
+
+    # Test Prerequisite is fulfilled
+    return unless is_host_ready_for_vgpu_test();
+
+    my $gpu_device = get_gpu_device();
+    die "No NVIDIA AMPERE GPU card on the machine!" if $gpu_device eq '';
+
+    # Clean up test logs
+    script_run "[ -d $log_dir ] && rm -rf $log_dir; mkdir -p $log_dir";
+
+    # Install NVIDIA vGPU manager
+    my $vgpu_manager = get_required_var("VGPU_MANAGER_URL");
+    install_vgpu_manager_and_reboot($vgpu_manager);
+
+    # Enable SR-IOV VFs
+    enable_sriov($gpu_device);
+
+    return if check_var('VGPU_TEST', 'short');
+
+    # Create a Compute Instance(CI) for GPU with MIG mode supported, such A30/A100
+    # given MIG mode is enabled. will handle non MIG mode when zen2 is available
+    if (get_var('GPU_MIG_MODE')) {
+        enable_mig_mode();
+        # Create a GI(GPU Instance)
+        my ($gi_id, $gpu_id) = create_gpu_instance();
+        # Create a CI(Compute Instance)
+        my $ci_id = create_compute_instance($gi_id, $gpu_id);
+    }
+
+    # Create a vGPU device
+    my $vgpu = create_vgpu($gpu_device, get_var('GPU_MIG_MODE'));
+
+    save_original_guest_xmls();
+    my $vgpu_grid_driver = get_required_var("VGPU_GRID_DRIVER_URL");
+    foreach my $guest (keys %virt_autotest::common::guests) {
+        record_info("Guest $guest");
+
+        # Assign vGPU to guest
+        my $gpu_slot_in_guest = "0x0a";
+        assign_vgpu_to_guest($vgpu, $guest, $gpu_slot_in_guest);
+
+        # Install vGPU grid driver in guest
+        install_vgpu_grid_driver($guest, $vgpu_grid_driver);
+
+        # Detach vGPU from guest
+        detach_vgpu_from_guest($gpu_slot_in_guest, $guest);
+        check_guest_health($guest);
+    }
+
+    # Remove vGPU device
+    remove_vgpu($vgpu);
+
+    # Disable GPU SR-IOV
+    assert_script_run("/usr/lib/nvidia/sriov-manage -d $gpu_device");
+    record_info("SRIOV disabled", $gpu_device);
+
+    # Upload vgpu related logs
+    upload_virt_logs($log_dir, "logs");
+
+    # Redefine guest from their original configuration files
+    restore_original_guests();
+}
+
+sub is_host_ready_for_vgpu_test {
+    if (is_sle('<15-SP3') or !is_kvm_host) {
+        record_info("Host not supported!", "NVIDIA vGPU is only supported on KVM hosts with SLE15SP3 and greater!", result => 'softfail');
+        return 0;
+    }
+    #check VT-d is supported in Intel x86_64 machines
+    if (script_run("grep -m 1 Intel /proc/cpuinfo") == 0) {
+        assert_script_run "dmesg | grep -E \"DMAR:.*IOMMU enabled\"";
+    }
+    script_run("lsmod | grep -e vfio -e nvidia -e mdev");
+    return 1;
+}
+
+sub get_gpu_device {
+    #NVIDIA AMPERE GPU
+    my $gpu_devices = script_output("lspci | grep NVIDIA | cut -d ' ' -f1");
+    foreach (split("\n", $gpu_devices)) {
+        return $_ if script_run("lspci -v -s $_ | grep 'SR-IOV'") == 0;
+    }
+}
+
+sub install_vgpu_manager_and_reboot {
+    my ${dirver_url} = shift;
+    die "The vGPU driver URL requires to be in 'http://...' format!" unless ${dirver_url} =~ /http:\/\/.*\/(.*\.run)/;
+    my ${driver_file} = $1;
+
+    # Check if vGPU manager has been already loaded
+    if (script_run("nvidia-smi") == 0) {
+        my ${driver_version} = script_output("nvidia-smi -q | grep 'Driver Version' | grep -oE \"[0-9.]+\"");
+        if (${driver_file} =~ /${driver_version}/) {
+            record_info("Warning", "vGPU manager ${driver_version} has already been loaded!", result => 'softfail');
+            return;
+        }
+    }
+    # Install vGPU manager
+    download_script(${driver_file}, script_url => ${dirver_url});
+    zypper_call "in kernel-default-devel gcc";
+    assert_script_run("modprobe vfio_pci_core") if is_sle('>=15-SP5');
+    assert_script_run("./${driver_file} -s");
+    record_info("vGPU manager is installed successfully.");
+
+    # Reboot host(it took 180+ seconds for ph052 to bootup with calltrace)
+    enter_cmd("grub2-once 0; reboot");
+    record_info("Host rebooting ...");
+    wait_for_host_reboot;
+    check_host_health;
+
+    # Verify vGPU manager installed successfully
+    assert_script_run("lspci -d 10de: -k");
+    assert_script_run("lsmod | grep nvidia");
+    assert_script_run("nvidia-smi");
+    script_run("ls -ld /sys/class/m*");
+}
+
+sub enable_sriov {
+    my $gpu = shift;
+    assert_script_run("/usr/lib/nvidia/sriov-manage -e $gpu");
+    script_run("dmesg | tail");
+    assert_script_run("ls -l /sys/bus/pci/devices/0000:$gpu/ | grep virtfn");
+    assert_script_run("lspci | grep NVIDIA");
+    record_info("SR-IOV enabled");
+}
+
+sub enable_mig_mode {
+    # Enable MIG mode
+    assert_script_run("nvidia-smi -mig 1");
+    # Verify MIG mode is enabled
+    assert_script_run("nvidia-smi -i 0 --query-gpu=pci.bus_id,mig.mode.current --format=csv | grep 'Enabled'");
+    record_info("Mig Mode is enalbed!");
+}
+
+sub create_gpu_instance {
+
+    # Create a GI by randomly picking up a supported type which has free instances by the GPU device model
+    # Only those profiles on which there are available instances are candidates
+    script_run("nvidia-smi mig -lgip");
+    my $avail_gi_profile_cmd = "nvidia-smi mig -lgip | grep MIG | grep -v '0/' | grep -v '+me'";
+    die "No available GI profiles!" unless script_run($avail_gi_profile_cmd) == 0;
+    my @gi_profile_ids = split '\n', script_output("$avail_gi_profile_cmd | awk '{ print \$5 }'");
+    my $gi_profile = $gi_profile_ids[int(rand($#gi_profile_ids + 1))];
+    my ($gi_id, $gpu_id);
+    if (script_output("nvidia-smi mig -cgi $gi_profile") =~ /Successfully created GPU instance ID\s+(\d*) on GPU\s+(\d*)/) {
+        ($gi_id, $gpu_id) = ($1, $2);
+        assert_script_run("nvidia-smi mig -lgi");
+    }
+    else {
+        die "Fail to create a GPU Instance!";
+    }
+    record_info("GI created", "GPU_ID: \$gpu_id, GI_ID: \$gi_id");
+
+    return ($gi_id, $gpu_id);
+}
+
+sub create_compute_instance {
+    my $gi_id = shift;
+
+    # Create a CI to fully use a GI
+    my $ci_id;
+    if (script_output("nvidia-smi mig -cci -gi $gi_id") =~ /Successfully created compute instance ID\s+(\d*).*GPU instance ID\s+$gi_id/) {
+        $ci_id = $1;
+        assert_script_run("nvidia-smi mig -lci");
+        script_run("nvidia-smi");
+    }
+    else {
+        die "Fail to create a Compute Instance on GPU ID $gi_id";
+    }
+    record_info("CI created", "GI_ID: \$gi_id, CI_ID: \$ci_id");
+    return $ci_id;
+}
+
+sub create_vgpu {
+    my ($gpu, $mig_mode) = @_;
+
+    # Find available vgpu types
+    my $vf_count = ${mig_mode} ? '8' : '32';
+    assert_script_run("cd /sys/bus/pci/devices/0000:$gpu/virtfn" . int(rand($vf_count)) . "/mdev_supported_types");
+    assert_script_run('for i in *; do echo $i $(cat $i/name) available instance: $(cat $i/avail*); done');
+    my $vgpu_type = ${mig_mode} ? "A.*-.*-.*C(ME)?" : "A.*-.*[CQ]";
+    # We have NVIDIA A10 GPU card with SR-IOV feature and A30 with MIG mode
+    # They behave a little differently in creating a vGPU device
+    # Find the available vGPU instance
+    my @avail_types = split /\n/, script_output("for i in *; do [ `cat \$i/avail*` -ne 0 ] && sed -n '/ ${vgpu_type}\$/p' \$i/name; done | cut -d '/' -f1", proceed_on_failure => 1);
+    die "No available vGPU types for GPU $gpu!" if @{avail_types} == 0;
+
+    # Choose a ramdom vgpu type and create a vGPU
+    my $vgpu_type_name = ${avail_types [int(rand($#avail_types + 1))]};
+    my $vgpu_type_id = script_output("grep -l '${vgpu_type_name}' */name | cut -d '/' -f1");
+    my $vgpu_id = script_output('uuidgen');
+    assert_script_run("echo ${vgpu_id} > ${vgpu_type_id}/create");
+
+    # Verify if vGPU created successfully
+    record_info("vGPU created", "$vgpu_type_name " . script_output("lsmdev"));
+    assert_script_run("dmesg | tail");
+    assert_script_run("nvidia-smi vgpu -q");
+
+    return ${vgpu_id};
+}
+
+sub assign_vgpu_to_guest {
+    my ($uuid, $vm, $slot) = @_;
+
+    # The UEFI guest created in virt test is ok for vGPU test
+    die "vGPU only works on UEFI guests!" unless $vm =~ /uefi/i;
+    assert_script_run("virsh shutdown $vm") unless script_output("virsh domstate $vm") eq "shut off";
+    # Save guest xml to /tmp/vm.xml, undefine the current one and define with the new xml
+    my $vm_xml_file = "/tmp/$vm.xml";
+    assert_script_run "virsh dumpxml --inactive $vm > $vm_xml_file";
+    assert_script_run "virsh undefine $vm";
+
+    # Add the vgpu device section to guest configuration file
+    die "PCI slot '0x0a' has already been used" if script_run("grep \"slot='$slot'\" $vm_xml_file") == 0;
+    my $vgpu_xml_section = "<hostdev mode='subsystem' type='mdev' model='vfio-pci' display='off'>\\\n  <source>\\\n    <address uuid='$uuid'/>\\\n  </source>\\\n  <address type='pci' domain='0x0000' bus='0x00' slot='$slot' function='0x0'/>\\\n</hostdev>";
+
+    assert_script_run("sed -i \"/<devices>/a\\\\${vgpu_xml_section}\" $vm_xml_file");
+    upload_logs($vm_xml_file);
+    assert_script_run "virsh define $vm_xml_file";
+    assert_script_run "virsh start $vm";
+    wait_guest_online($vm);
+    assert_script_run "ssh root\@$vm 'lspci | grep NVIDIA'";
+    record_info("vGPU attached to $vm", script_output("ssh root\@$vm 'lspci | grep NVIDIA'"));
+}
+
+sub install_vgpu_grid_driver {
+    my ($vm, $driver_url) = @_;
+
+    # Download drivers from fileserver
+    die "The vGPU driver URL requires to be in 'http://...' format!" unless $driver_url =~ /http:\/\/.*\/(.*\.run)/;
+    my ${driver_file} = $1;
+
+    # Check if the driver has been already installed
+    if (script_run("ssh root\@$vm 'nvidia-smi'") == 0) {
+        my $grid_version = script_output("ssh root\@$vm 'nvidia-smi -q | grep \"Driver Version\" | grep -oE \"[0-9.]+\"'");
+        if (${driver_file} =~ /$grid_version/) {
+            record_info("Warning", "vGPU grid driver $grid_version has already been loaded!", result => 'softfail');
+            return;
+        }
+    }
+
+    # Install dependencies seperately. It is easier to locate problem this way.
+    download_script(${driver_file}, script_url => ${driver_url}, machine => $vm);
+    assert_script_run "ssh root\@$vm 'zypper -n in kernel-default-devel'";
+    assert_script_run "ssh root\@$vm 'zypper -n in libglvnd-devel'";
+    # Install vGPU grid drivers without manual interactions
+    assert_script_run("ssh root\@$vm './${driver_file} -s'");
+    assert_script_run("ssh root\@$vm 'nvidia-smi'");
+    record_info("vGPU Grid driver is installed successfully in $vm");
+    # Verify
+    assert_script_run("ssh root\@$vm 'lsmod | grep -i nvidia'");
+}
+
+sub detach_vgpu_from_guest {
+    my ($slot, $vm) = @_;
+    script_run("ssh root\@$vm 'poweroff'");
+    script_retry("virsh domstate $vm | grep 'shut off'", timeout => 60, delay => 5, retry => 3, die => 0);
+    assert_script_run("virt-xml $vm --remove-device --hostdev type=mdev,address.slot=$slot");
+    assert_script_run("virsh start $vm");
+    record_info("vGPU has been removed successfully from $vm");
+    wait_guest_online($vm);
+}
+
+sub remove_vgpu {
+    my $uuid = shift;
+    assert_script_run("cd `find /sys/devices -type d -name $uuid`");
+    assert_script_run("echo '1' > remove");
+    script_run('cd ../mdev_supported_types/; for i in *; do echo "$i" $(cat $i/name) available: $(cat $i/avail*); done');
+    die "Fatal: vGPU device $uuid is still alive!" if script_run("lsmdev | grep $uuid") == 0;
+    record_info("vGPU removed", "$uuid has been removed from the host");
+}
+
+sub save_vgpu_device_status_logs {
+    my $vm = shift;
+
+    #vm configuration file
+    enter_cmd("virsh dumpxml $vm > $log_dir/${vm}.xml");
+
+    my $log_file = "log.txt";
+    #logging device information in guest
+    if (script_run("virsh domstate $vm | grep running") == 0) {
+        script_run "echo `date` > $log_file";
+        script_run "echo '***** Status & logs inside $vm *****' >> $log_file";
+        my $debug_script = "vgpu_guest_logging.sh";
+        download_script($debug_script, machine => $vm, proceed_on_failure => 1);
+        script_run("timeout 25 ssh root\@$vm \"~/$debug_script\" >> $log_file 2>&1");
+        # Save nvidia driver installer logs
+        script_run("rsync root\@$vm:/var/log/nvidia-installer.log ${log_dir}/${vm}_nvidia-installer.log");
+        script_run "mv $log_file $log_dir/${vm}_vgpu_device_status.txt";
+    }
+}
+
+sub post_fail_hook {
+    my $self = shift;
+    diag("Module vgpu post fail hook starts.");
+
+    # Make sure sol console is ok
+    enter_cmd "echo DONE > /dev/$serialdev";
+    reconnect_when_ssh_console_broken unless defined(wait_serial('DONE', timeout => 30));
+
+    my $log_file = "$log_dir/host_vgpu_device_status.txt";
+    print_cmd_output_to_file("virsh list --all", $log_file);
+    print_cmd_output_to_file("nvidia-smi", $log_file);
+    my $cmd = "journalctl --cursor-file /tmp/cursor.txt | grep -r -e 'kernel:' -e gpu -e nvidia | grep -v bash_history";
+    script_run("echo '' >> $log_file");
+    script_run("$cmd >> $log_file");
+    script_run("cp /var/log/nvidia-installer.log $log_dir");
+    save_vgpu_device_status_logs($_) foreach (keys %virt_autotest::common::guests);
+    upload_virt_logs($log_dir, "vgpu");
+    $self->SUPER::post_fail_hook;
+    restore_original_guests() unless check_var('VGPU_TEST', 'short');
+}
+
+sub test_flags {
+    #continue subsequent test in the case test restored
+    return {fatal => 0};
+}
+
+1;