Fix SDAF deployer cleanup failures

TEAM-9647 - [SDAF] SDAF_PRD_deploy_HanaSR failed to do deployer cleanup when failed on: configure_deployer & deploy_workload_zone

lib/sles4sap/console_redirection.pm

@@ -12,7 +12,7 @@ use testapi;
 use Carp qw(croak);
 use Exporter qw(import);
 use Regexp::Common qw(net);
-use serial_terminal qw(select_serial_terminal);
+use serial_terminal qw(select_serial_terminal serial_term_prompt);
 
 =head1 SYNOPSIS
 
@@ -175,6 +175,11 @@ from serial console. VM ID is collected by opening 'log-console' which is not re
 sub check_serial_redirection {
     # Do not select serial console if it is already done. This avoids log pollution and speeds up process.
     if (is_serial_terminal()) {
+        # Teminate the process when scirpts timed out
+        unless (wait_serial(serial_term_prompt())) {
+            type_string('', terminate_with => 'ETX');
+            type_string("\n");
+        }
         select_serial_terminal();
         set_serial_term_prompt();
     }

lib/sles4sap/sap_deployment_automation_framework/deployment.pm

@@ -707,16 +707,27 @@ sub sdaf_execute_remover {
         '--type', $type_parameter,
         '--auto-approve');
 
-    # capture command output into log file
-    my $output_log_file = log_dir() . "/cleanup_$args{deployment_type}.txt";
-    $remover_cmd = log_command_output(command => $remover_cmd, log_file => $output_log_file);
-
-    # SDAF must be executed from the profile directory, otherwise it will fail
-    assert_script_run("cd " . $tfvars_path);
-    record_info('SDAF destroy', "Executing SDAF remover:\n$remover_cmd");
-    # Keep the timeout high, definitely above 1H. Azure tends to be slow.
-    my $rc = script_run($remover_cmd, timeout => 7200);
-    upload_logs($output_log_file, log_name => $output_log_file);
+    my $rc;
+    my $output_log_file = log_dir() . "/cleanup_$args{deployment_type}_attempt.txt";
+    my $attempt_no = 1;
+    while ($attempt_no <= 3) {
+        record_info("Attempt #$attempt_no");
+        # Capture command output into log file
+        $output_log_file =~ s/attempt/attempt-$attempt_no/;
+        $remover_cmd = log_command_output(command => $remover_cmd, log_file => $output_log_file);
+
+        # SDAF must be executed from the profile directory, otherwise it will fail
+        assert_script_run("cd " . $tfvars_path);
+        record_info('SDAF destroy', "Executing SDAF remover:\n$remover_cmd");
+        # Keep the timeout high, definitely above 1H. Azure tends to be slow.
+        $rc = script_run($remover_cmd, timeout => 7200);
+        upload_logs($output_log_file, log_name => $output_log_file);
+
+        last unless $rc;
+        sleep 30;
+        record_info("SDAF destroy retry $attempt_no", "destroy of '$args{deployment_type}' exited with RC '$rc', retrying ...");
+        $attempt_no++;
+    }
 
     # Do not kill the test, only return RC. There are still files to be cleaned up on deployer VM side.
     return $rc;

lib/sles4sap/sap_deployment_automation_framework/deployment_connector.pm

@@ -140,8 +140,8 @@ sub get_deployer_vm_name {
         "--query \"\[?tags.deployment_id == '$args{deployment_id}'].name\"",
         '--output json'
     );
-
-    my @vm_list = @{decode_json(script_output($az_cmd))};
+    # for testing purpose, please delete "timeout => 1" later
+    my @vm_list = @{decode_json(script_output("sleep 10; $az_cmd", timeout => 1))};
     diag((caller(0))[3] . " - VMs found: " . join(', ', @vm_list));
     die "Multiple VMs with same IDs found. Each VM must have unique ID!\n
     Following VMs found tagged with: deployment_id=$args{deployment_id}"