add crucible to propolis-standalone config toml

README.md

@@ -225,6 +225,89 @@ you've done the following:
 - optionally, run `setup-alpine` to configure the VM (including setting a root
   password)
 
+## Crucible Storage With propolis-standalone
+
+propolis-standalone supports defining crucible-backed storage devices in the
+TOML config. It is somewhat inconvenient to do this without scripting, because
+`generation` must monotonically increase with each successive connection to the
+Downstairs datastore. So if you use this, you need to somehow monotonically bump
+up that number in the TOML file before re-launching the VM, unless you're also
+creating a new Downstairs region from scratch.
+
+All the crucible configuration options are crucible-specific, so future changes
+to crucible may result in changes to the config options here as well. Consult
+the [oxidecomputer/crucible](https://github.com/oxidecomputer/crucible) codebase
+if you need low level details on what certain options actually do.
+
+Here's an example config. Read the comments for parameter-specific details:
+
+```toml
+[block_dev.some_datastore]
+type = "crucible"
+
+# === REQUIRED OPTIONS ===
+# these MUST match the region configuration downstairs
+block_size = 512
+blocks_per_extent = 262144
+extent_count = 32
+
+# Array of the SocketAddrs of the Downstairs instances. There must be three
+# of these, or propolis-standalone will panic.
+targets = [
+  "127.0.0.1:3810",
+  "127.0.0.1:3820",
+  "127.0.0.1:3830",
+]
+
+# Generation number used when connecting to Downstairs. This must
+# monotonically increase with each successive connection to the Downstairs,
+# which means that you need to bump this number every time you restart
+# your VM. Kind of annoying, maybe we can get a better way to pass it in.
+# Anyway, if you don't want to read-modify-write this value, a hack you
+# could do is set this to the current number of seconds since the epoch.
+# This'll always work, except for if the system time goes backwards, which
+# it can definitely do! So, you know. Be careful.
+generation = 1
+# === END REQUIRED OPTIONS ===
+
+
+# === OPTIONAL OPTIONS ===
+# This should be a UUID. It can be anything, really. When unset, defaults
+# to a random UUIDv4
+# upstairs_id = "e4396bd0-ede1-48d7-ac14-3d2094dfba5b"
+
+# When true, some random amount of IO requests will synthetically "fail".
+# This is useful when testing IO behavior under Bad Conditions.
+# Defaults to false.
+# lossy = false
+
+# the Upstairs (propolis-side) component of crucible currently regularly
+# dispatches flushes to act as IO barriers. By default this happens once every 5
+# seconds, but you can adjust it with this option.
+# flush_timeout = <number>
+
+# Base64'd encryption key used to encrypt data at rest. Keys are 256 bits.
+# Note that the region must have already been created with encryption
+# enabled for this to work. That may change later though.
+# encryption_key = ""
+
+# These three values are pem files for TLS encryption of data between
+# propolis and the downstairs.
+# cert_pem = ""
+# key_pem = ""
+# root_cert_pem = ""
+
+# Specifies the SocketAddr of the Upstairs crucible control interface. When
+# ommitted, the control interface won't be started. The control interface is an
+# HTTP server that exposes commands to take snapshots, simulate faults, and
+# retrieve runtime debug information.
+# control_addr = ""
+
+# When true, the device will be read-only. Defaults to false
+# read_only = false
+# === END OPTIONAL OPTIONS ===
+```
+
 ## License
 
 Unless otherwise noted, all components are licensed under the [Mozilla Public

bin/propolis-standalone/Cargo.toml

@@ -19,15 +19,17 @@ toml.workspace = true
 tokio = { workspace = true, features = ["io-util", "rt-multi-thread"] }
 serde = { workspace = true, features = ["derive"] }
 propolis.workspace = true
+crucible-client-types = { workspace = true, optional = true }
 propolis-standalone-config = { workspace = true }
 erased-serde.workspace = true
 serde_json.workspace = true
 slog.workspace = true
 slog-async.workspace = true
 slog-term.workspace = true
 num_enum.workspace = true
+uuid.workspace = true
 
 [features]
 default = ["dtrace-probes"]
 dtrace-probes = ["propolis/dtrace-probes"]
-crucible = ["propolis/crucible", "propolis/oximeter"]
+crucible = ["propolis/crucible-full", "propolis/oximeter", "crucible-client-types"]

bin/propolis-standalone/src/config.rs

@@ -23,13 +23,11 @@ pub fn block_backend(
         "file" => {
             let path = be.options.get("path").unwrap().as_str().unwrap();
 
-            let readonly: bool = || -> Option<bool> {
-                match be.options.get("readonly") {
-                    Some(toml::Value::Boolean(read_only)) => Some(*read_only),
-                    Some(toml::Value::String(v)) => v.parse().ok(),
-                    _ => None,
-                }
-            }()
+            let readonly = (match be.options.get("readonly") {
+                Some(toml::Value::Boolean(read_only)) => Some(*read_only),
+                Some(toml::Value::String(v)) => v.parse().ok(),
+                _ => None,
+            })
             .unwrap_or(false);
 
             let be = block::FileBackend::create(
@@ -43,6 +41,10 @@ pub fn block_backend(
             let creg = ChildRegister::new(&be, Some(path.to_string()));
             (be, creg)
         }
+        #[cfg(feature = "crucible")]
+        "crucible" => create_crucible_backend(log, be),
+        #[cfg(not(feature = "crucible"))]
+        "crucible" => panic!("crucible device specified in VM config, but propolis-standalone was not built with crucible support. rebuild propolis-standalone with the `crucible` feature enabled, or remove all crucible devices from your VM config"),
         _ => {
             panic!("unrecognized block dev type {}!", be.bdtype);
         }
@@ -71,3 +73,122 @@ pub fn parse_bdf(v: &str) -> Option<Bdf> {
         None
     }
 }
+
+#[cfg(feature = "crucible")]
+fn create_crucible_backend(
+    log: &slog::Logger,
+    be: &propolis_standalone_config::BlockDevice,
+) -> (Arc<dyn block::Backend>, ChildRegister) {
+    use slog::info;
+    use uuid::Uuid;
+    info!(
+        log,
+        "Building a crucible VolumeConstructionRequest from options {:?}",
+        be.options
+    );
+
+    // No defaults on here because we really shouldn't try and guess
+    // what block size the downstairs is using. A lot of things
+    // default to 512, but it's best not to assume it'll always be
+    // that way.
+    let block_size =
+        be.options.get("block_size").unwrap().as_integer().unwrap() as u64;
+
+    let blocks_per_extent =
+        be.options.get("blocks_per_extent").unwrap().as_integer().unwrap()
+            as u64;
+
+    let extent_count =
+        be.options.get("extent_count").unwrap().as_integer().unwrap() as u32;
+
+    // Parse a UUID, or generate a random one if none is specified.
+    // Reasonable in something primarily used for testing like
+    // propolis-standalone, but you wouldn't want to do this in
+    // prod.
+    let uuid = be
+        .options
+        .get("upstairs_id")
+        .map(|x| Uuid::parse_str(x.as_str().unwrap()).unwrap())
+        .unwrap_or_else(Uuid::new_v4);
+
+    // The actual addresses of the three downstairs we're going to connect to.
+    let targets: Vec<_> = be
+        .options
+        .get("targets")
+        .unwrap()
+        .as_array()
+        .unwrap()
+        .iter()
+        .map(|target_val| target_val.as_str().unwrap().parse().unwrap())
+        .collect();
+    // There is currently no universe where an amount of Downstairs
+    // other than 3 is valid.
+    assert_eq!(targets.len(), 3);
+
+    let lossy =
+        be.options.get("lossy").map(|x| x.as_bool().unwrap()).unwrap_or(false);
+
+    let flush_timeout =
+        be.options.get("flush_timeout").map(|x| x.as_integer().unwrap() as u32);
+
+    let key = be
+        .options
+        .get("encryption_key")
+        .map(|x| x.as_str().unwrap().to_string());
+
+    let cert_pem =
+        be.options.get("cert_pem").map(|x| x.as_str().unwrap().to_string());
+
+    let key_pem =
+        be.options.get("key_pem").map(|x| x.as_str().unwrap().to_string());
+
+    let root_cert_pem = be
+        .options
+        .get("root_cert_pem")
+        .map(|x| x.as_str().unwrap().to_string());
+
+    let control_addr = be
+        .options
+        .get("control_addr")
+        .map(|target_val| target_val.as_str().unwrap().parse().unwrap());
+
+    let read_only = be
+        .options
+        .get("read_only")
+        .map(|x| x.as_bool().unwrap())
+        .unwrap_or(false);
+
+    // This needs to increase monotonically with each successive
+    // connection to the downstairs. As a hack, you can set it to
+    // the current system time, and this will usually give us a newer
+    // generation than the last connection. NEVER do this in prod
+    // EVER.
+    let generation =
+        be.options.get("generation").unwrap().as_integer().unwrap() as u64;
+
+    let req = crucible_client_types::VolumeConstructionRequest::Region {
+        block_size,
+        blocks_per_extent,
+        extent_count,
+        opts: crucible_client_types::CrucibleOpts {
+            id: uuid,
+            target: targets,
+            lossy,
+            flush_timeout,
+            key,
+            cert_pem,
+            key_pem,
+            root_cert_pem,
+            control: control_addr,
+            read_only,
+        },
+        gen: generation,
+    };
+    info!(log, "Creating Crucible disk from request {:?}", req);
+    // QUESTION: is producer_registry: None correct here?
+    let be =
+        block::CrucibleBackend::create(req.clone(), read_only, None).unwrap();
+    let creg =
+        ChildRegister::new(&be, Some(be.get_uuid().unwrap().to_string()));
+    (be, creg)
+}