Increasing ncn shutdown timeout and hard power off handle

IM:CRAYSAT-1513
Reviewer:Ryan

sat/cli/bootsys/mgmt_power.py

@@ -36,6 +36,7 @@
 
 from sat.cli.bootsys.ipmi_console import IPMIConsoleLogger, ConsoleLoggingError
 from sat.cli.bootsys.util import get_and_verify_ncn_groups, get_ssh_client, FatalBootsysError
+from sat.cli.bootsys.util import hard_power_off
 from sat.waiting import GroupWaiter, WaitingFailure
 from sat.config import get_config_value
 from sat.util import BeginEndLogger, get_username_and_password_interactively, prompt_continue
@@ -278,16 +279,37 @@ def do_mgmt_shutdown_power(ssh_client, username, password, excluded_ncns, ncn_sh
         with IPMIConsoleLogger(other_ncns, username, password):
             LOGGER.info(f'Sending shutdown command to other NCNs: {", ".join(other_ncns)}')
             start_shutdown(other_ncns, ssh_client)
+
             LOGGER.info(f'Waiting up to {ncn_shutdown_timeout} seconds for other NCNs to '
                         f'reach powered off state according to ipmitool: {", ".join(other_ncns)}.')
-            finish_shutdown(other_ncns, username, password,
-                            ncn_shutdown_timeout, ipmi_timeout)
-            LOGGER.info('Shutdown and power off of all other NCNs complete.')
+            ipmi_waiter = IPMIPowerStateWaiter(other_ncns, 'off', ncn_shutdown_timeout, username, password)
+            pending_hosts = ipmi_waiter.wait_for_completion()
+
+            if pending_hosts:
+                prompt_msg = ('Some NCNs have not reached the "off" state. Do you want to proceed '
+                              'with forcibly powering off these NCNs? (yes/no): ')
+                if hard_power_off(prompt_msg):
+                    LOGGER.warning('Forcibly powering off nodes: %s', ', '.join(pending_hosts))
+
+                    # Confirm all nodes have actually turned off.
+                    failed_hosts = IPMIPowerStateWaiter(pending_hosts, 'off', ipmi_timeout, username, password,
+                                                        send_command=True).wait_for_completion()
+
+                    if failed_hosts:
+                        LOGGER.error('The following nodes failed to reach powered '
+                                     'off state: %s', ', '.join(failed_hosts))
+                        raise SystemExit(1)
+                else:
+                    LOGGER.info('User chose not to proceed with forcible shutdown. Exiting gracefully.')
+                    raise SystemExit(0)
+            else:
+                LOGGER.info('Shutdown and power off of all other NCNs complete.')
     except ConsoleLoggingError as err:
-        LOGGER.error(f'Aborting shutdown of NCNs due failure to set up NCN console logging: {err}')
+        LOGGER.error(f'Aborting shutdown of NCNs due to failure to set up NCN console logging: {err}')
         raise SystemExit(1)
 
 
+
 def do_power_off_ncns(args):
     """Power off NCNs while monitoring consoles with ipmitool.
 

sat/cli/bootsys/parser.py

@@ -55,7 +55,7 @@
                 'compute and application nodes have completed their BOS shutdown.'),
     TimeoutSpec('bos-boot', ['boot', 'reboot'], 900,
                 'compute and application nodes have completed their BOS boot.'),
-    TimeoutSpec('ncn-shutdown', ['shutdown'], 300,
+    TimeoutSpec('ncn-shutdown', ['shutdown'], 900,
                 'management NCNs have completed a graceful shutdown and have reached '
                 'the powered off state according to IPMI.'),
 ]

sat/cli/bootsys/util.py

@@ -229,3 +229,18 @@ def get_ssh_client(host_keys=None):
     ssh_client.set_missing_host_key_policy(WarningPolicy)
 
     return ssh_client
+
+def hard_power_off(prompt):
+    """Prompt the user for a yes or no response.
+
+    Args:
+        prompt (str): The prompt message to display.
+
+    Returns:
+        str: 'yes' or 'no' based on user input.
+    """
+    while True:
+        response = input(f'{prompt} (yes/no): ').strip().lower()
+        if response in {'yes', 'no'}:
+            return response
+        print('Invalid response. Please enter "yes" or "no".')