phd: check exit status of commands, have tests expect success/failure

phd-tests/framework/src/test_vm/mod.rs

@@ -18,7 +18,7 @@ use crate::{
     Framework,
 };
 
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, bail, Context, Result};
 use camino::Utf8PathBuf;
 use core::result::Result as StdResult;
 use propolis_client::{
@@ -142,6 +142,107 @@ enum VmState {
     Ensured { serial: SerialConsole },
 }
 
+/// Description of the acceptable status codes from executing a command in a
+/// [`TestVm::run_shell_command`].
+// This could reasonably have a `Status(u16)` variant to check specific non-zero
+// statuses, but specific codes are not terribly portable! In the few cases we
+// can expect a specific status for errors, those specific codes change between
+// f.ex illumos and Linux guests.
+enum StatusCheck {
+    Ok,
+    NotOk,
+}
+
+pub struct ShellOutputExecutor<'ctx> {
+    vm: &'ctx TestVm,
+    cmd: &'ctx str,
+    status_check: Option<StatusCheck>,
+}
+
+impl<'a> ShellOutputExecutor<'a> {
+    pub fn ignore_status(mut self) -> ShellOutputExecutor<'a> {
+        self.status_check = None;
+        self
+    }
+
+    pub fn check_ok(mut self) -> ShellOutputExecutor<'a> {
+        self.status_check = Some(StatusCheck::Ok);
+        self
+    }
+
+    pub fn check_err(mut self) -> ShellOutputExecutor<'a> {
+        self.status_check = Some(StatusCheck::NotOk);
+        self
+    }
+}
+use futures::FutureExt;
+
+impl<'a> std::future::IntoFuture for ShellOutputExecutor<'a> {
+    type Output = Result<String>;
+    type IntoFuture = futures::future::BoxFuture<'a, Result<String>>;
+
+    fn into_future(self) -> Self::IntoFuture {
+        Box::pin(async move {
+            // Allow the guest OS to transform the input command into a
+            // guest-specific command sequence. This accounts for the guest's
+            // shell type (which affects e.g. affects how it displays multi-line
+            // commands) and serial console buffering discipline.
+            let command_sequence =
+                self.vm.guest_os.shell_command_sequence(self.cmd);
+            self.vm.run_command_sequence(command_sequence).await?;
+
+            // `shell_command_sequence` promises that the generated command
+            // sequence clears buffer of everything up to and including the
+            // input command before actually issuing the final '\n' that issues
+            // the command.  This ensures that the buffer contents returned by
+            // this call contain only the command's output.
+            let output = self
+                .vm
+                .wait_for_serial_output(
+                    self.vm.guest_os.get_shell_prompt(),
+                    Duration::from_secs(300),
+                )
+                .await?;
+
+            // Trim any leading newlines inserted when the command was issued
+            // and any trailing whitespace that isn't actually part of the
+            // command output. Any other embedded whitespace is the caller's
+            // problem.
+            let output = output.trim().to_string();
+
+            if let Some(check) = self.status_check {
+                let status_command_sequence =
+                    self.vm.guest_os.shell_command_sequence("echo $?");
+                self.vm.run_command_sequence(status_command_sequence).await?;
+                let status = self
+                    .vm
+                    .wait_for_serial_output(
+                        self.vm.guest_os.get_shell_prompt(),
+                        Duration::from_secs(300),
+                    )
+                    .await?;
+                let status = status.trim().parse::<u16>()?;
+
+                match check {
+                    StatusCheck::Ok => {
+                        if status != 0 {
+                            bail!("expected status 0, got {}", status);
+                        }
+                    }
+                    StatusCheck::NotOk => {
+                        if status == 0 {
+                            bail!("expected non-zero status, got {}", status);
+                        }
+                    }
+                }
+            }
+
+            Ok(output)
+        })
+        .boxed()
+    }
+}
+
 /// A virtual machine running in a Propolis server. Test cases create these VMs
 /// using the `factory::VmFactory` embedded in their test contexts.
 ///
@@ -868,30 +969,23 @@ impl TestVm {
     /// waits for another shell prompt to appear using
     /// [`Self::wait_for_serial_output`] and returns any text that was buffered
     /// to the serial console after the command was sent.
-    pub async fn run_shell_command(&self, cmd: &str) -> Result<String> {
-        // Allow the guest OS to transform the input command into a
-        // guest-specific command sequence. This accounts for the guest's shell
-        // type (which affects e.g. affects how it displays multi-line commands)
-        // and serial console buffering discipline.
-        let command_sequence = self.guest_os.shell_command_sequence(cmd);
-        self.run_command_sequence(command_sequence).await?;
-
-        // `shell_command_sequence` promises that the generated command sequence
-        // clears buffer of everything up to and including the input command
-        // before actually issuing the final '\n' that issues the command.
-        // This ensures that the buffer contents returned by this call contain
-        // only the command's output.
-        let out = self
-            .wait_for_serial_output(
-                self.guest_os.get_shell_prompt(),
-                Duration::from_secs(300),
-            )
-            .await?;
-
-        // Trim any leading newlines inserted when the command was issued and
-        // any trailing whitespace that isn't actually part of the command
-        // output. Any other embedded whitespace is the caller's problem.
-        Ok(out.trim().to_string())
+    ///
+    /// After running the shell command, sends `echo $?` to query and return the
+    /// command's return status as well.
+    ///
+    /// This will return an error if the command returns a non-zero status by
+    /// default; to ignore the status or expect a non-zero as a positive
+    /// condition, see [`ShellOutputExecutor::ignore_status`] or
+    /// [`ShellOutputExecutor::check_err`].
+    pub fn run_shell_command<'a>(
+        &'a self,
+        cmd: &'a str,
+    ) -> ShellOutputExecutor<'a> {
+        ShellOutputExecutor {
+            vm: self,
+            cmd,
+            status_check: Some(StatusCheck::Ok),
+        }
     }
 
     pub async fn graceful_reboot(&self) -> Result<()> {

phd-tests/tests/src/boot_order.rs

@@ -20,6 +20,11 @@ use efi_utils::{
     EDK2_FIRMWARE_VOL_GUID, EDK2_UI_APP_GUID,
 };
 
+// NOTE: This function differs from `run_shell_command` in that it implicitly
+// ignores the status of executed commands. When
+// https://github.com/oxidecomputer/propolis/issues/773 is fixed and this is
+// deleted, callers of this function may need to be updated to call
+// `.ignore_status` or `.check_err`
 pub(crate) async fn run_long_command(
     vm: &phd_framework::TestVm,
     cmd: &str,
@@ -31,7 +36,7 @@ pub(crate) async fn run_long_command(
     // I haven't gone and debugged that; instead, chunk the input command up
     // into segments short enough to not wrap when input, put them all in a
     // file, then run the file.
-    vm.run_shell_command("rm cmd").await?;
+    vm.run_shell_command("rm cmd").ignore_status().await?;
     let mut offset = 0;
     // Escape any internal `\`. This isn't comprehensive escaping (doesn't
     // handle \n, for example)..
@@ -48,7 +53,9 @@ pub(crate) async fn run_long_command(
 
         vm.run_shell_command(&format!("echo -n \'{}\' >>cmd", chunk)).await?;
     }
-    vm.run_shell_command(". cmd").await
+    // `ignore_status` because it's a bit cumbersome to wrap this whole thing in
+    // a way that checks statuses,
+    vm.run_shell_command(". cmd").ignore_status().await
 }
 
 // This test checks that with a specified boot order, the guest boots whichever
@@ -284,7 +291,8 @@ async fn guest_can_adjust_boot_order(ctx: &Framework) {
 
     // If the guest doesn't have an EFI partition then there's no way for boot
     // order preferences to be persisted.
-    let mountline = vm.run_shell_command("mount | grep efivarfs").await?;
+    let mountline =
+        vm.run_shell_command("mount | grep efivarfs").ignore_status().await?;
 
     if !mountline.starts_with("efivarfs on ") {
         warn!(

phd-tests/tests/src/crucible/migrate.rs

@@ -19,7 +19,8 @@ async fn smoke_test(ctx: &Framework) {
     source.launch().await?;
     source.wait_to_boot().await?;
 
-    let lsout = source.run_shell_command("ls foo.bar 2> /dev/null").await?;
+    let lsout =
+        source.run_shell_command("ls foo.bar 2> /dev/null").check_err().await?;
     assert_eq!(lsout, "");
     source.run_shell_command("touch ./foo.bar").await?;
     source.run_shell_command("sync ./foo.bar").await?;

phd-tests/tests/src/crucible/smoke.rs

@@ -37,6 +37,7 @@ async fn guest_reboot_test(ctx: &Framework) {
     vm.launch().await?;
     vm.wait_to_boot().await?;
 
+    // XXX: use graceful_reboot() now.
     // Don't use `run_shell_command` because the guest won't echo another prompt
     // after this.
     vm.send_serial_str("reboot\n").await?;
@@ -64,7 +65,8 @@ async fn shutdown_persistence_test(ctx: &Framework) {
 
     // Verify that the test file doesn't exist yet, then touch it, flush it, and
     // shut down the VM.
-    let lsout = vm.run_shell_command("ls foo.bar 2> /dev/null").await?;
+    let lsout =
+        vm.run_shell_command("ls foo.bar 2> /dev/null").check_err().await?;
     assert_eq!(lsout, "");
     vm.run_shell_command("touch ./foo.bar").await?;
     vm.run_shell_command("sync ./foo.bar").await?;

phd-tests/tests/src/disk.rs

@@ -37,7 +37,7 @@ async fn in_memory_backend_smoke_test(ctx: &Framework) {
     // try to check that the disk is located there and fail the test early if
     // it's not. If the by-path directory is missing, put up a warning and hope
     // for the best.
-    let dev_disk = vm.run_shell_command("ls /dev/disk").await?;
+    let dev_disk = vm.run_shell_command("ls /dev/disk").ignore_status().await?;
     if dev_disk.contains("by-path") {
         let ls = vm.run_shell_command("ls -la /dev/disk/by-path").await?;
         info!(%ls, "guest disk device paths");

phd-tests/tests/src/hw.rs

@@ -17,15 +17,23 @@ async fn lspci_lifecycle_test(ctx: &Framework) {
     vm.launch().await?;
     vm.wait_to_boot().await?;
 
-    let lspci = vm.run_shell_command(LSPCI).await?;
-    let lshw = vm.run_shell_command(LSHW).await?;
+    // XXX: do not `ignore_status()` on these commands! They fail for any number
+    // of reasons on different guests:
+    // * sudo may not exist (some Alpine)
+    // * lshw may not exist (Debian)
+    // * we may not input a sudo password (Ubuntu)
+
+    let lspci = vm.run_shell_command(LSPCI).ignore_status().await?;
+    let lshw = vm.run_shell_command(LSHW).ignore_status().await?;
     ctx.lifecycle_test(vm, &[Action::StopAndStart], move |vm| {
         let lspci = lspci.clone();
         let lshw = lshw.clone();
         Box::pin(async move {
-            let new_lspci = vm.run_shell_command(LSPCI).await.unwrap();
+            let new_lspci =
+                vm.run_shell_command(LSPCI).ignore_status().await.unwrap();
             assert_eq!(new_lspci, lspci);
-            let new_lshw = vm.run_shell_command(LSHW).await.unwrap();
+            let new_lshw =
+                vm.run_shell_command(LSHW).ignore_status().await.unwrap();
             assert_eq!(new_lshw, lshw);
         })
     })

phd-tests/tests/src/migrate.rs

@@ -55,7 +55,11 @@ mod from_base {
             spawn_base_vm(ctx, "migration_from_base_and_back").await?;
         source.launch().await?;
         source.wait_to_boot().await?;
-        let lsout = source.run_shell_command("ls foo.bar 2> /dev/null").await?;
+        // `ls` with no results exits non-zero, so expect an error here.
+        let lsout = source
+            .run_shell_command("ls foo.bar 2> /dev/null")
+            .check_err()
+            .await?;
         assert_eq!(lsout, "");
 
         // create an empty file on the source VM.
@@ -74,8 +78,9 @@ mod from_base {
                     // the file should still exist on the target VM after migration.
                     let lsout = target
                         .run_shell_command("ls foo.bar")
+                        .ignore_status()
                         .await
-                        .expect("`ls foo.bar` should succeed");
+                        .expect("can try to run `ls foo.bar`");
                     assert_eq!(lsout, "foo.bar");
                 })
             },
@@ -257,7 +262,9 @@ mod running_process {
         ))
         .await?;
         vm.run_shell_command("chmod +x dirt.sh").await?;
-        let run_dirt = vm.run_shell_command("./dirt.sh").await?;
+        // When dirt.sh suspends itself, the parent shell will report a non-zero
+        // status (one example is 148: 128 + SIGTSTP aka 20 on Linux).
+        let run_dirt = vm.run_shell_command("./dirt.sh").check_err().await?;
         assert!(run_dirt.contains("made dirt"), "dirt.sh failed: {run_dirt:?}");
         assert!(
             run_dirt.contains("Stopped"),
@@ -346,7 +353,8 @@ async fn multiple_migrations(ctx: &Framework) {
 async fn run_smoke_test(ctx: &Framework, mut source: TestVm) -> Result<()> {
     source.launch().await?;
     source.wait_to_boot().await?;
-    let lsout = source.run_shell_command("ls foo.bar 2> /dev/null").await?;
+    let lsout =
+        source.run_shell_command("ls foo.bar 2> /dev/null").check_err().await?;
     assert_eq!(lsout, "");
 
     // create an empty file on the source VM.
@@ -361,8 +369,9 @@ async fn run_smoke_test(ctx: &Framework, mut source: TestVm) -> Result<()> {
                 // the file should still exist on the target VM after migration.
                 let lsout = target
                     .run_shell_command("ls foo.bar")
+                    .ignore_status()
                     .await
-                    .expect("`ls foo.bar` should succeed after migration");
+                    .expect("can try to run `ls foo.bar`");
                 assert_eq!(lsout, "foo.bar");
             })
         },