feat: initial filesystem virt (#4)

Cargo.toml

@@ -14,6 +14,7 @@ name = "wasi-virt"
 [workspace]
 members = [
   "virtual-adapter",
+  "tests/components/file-read",
   "tests/components/get-env"
 ]
 
@@ -30,7 +31,9 @@ clap = { version = "4", features = ["derive"] }
 serde = { version = "1", features = ["derive"] }
 toml = "0.7"
 walrus = "0.20.1"
-wit-component = "0.11.0"
+wasm-metadata = { git = "https://github.com/bytecodealliance/wasm-tools" }
+wasm-opt = "0.113.0"
+wit-component = { git = "https://github.com/bytecodealliance/wasm-tools" }
 
 [build-dependencies]
 anyhow = "1"
@@ -40,10 +43,10 @@ anyhow = "1"
 async-std = { version = "1", features = ["attributes"] }
 cap-std = "1.0.12"
 heck =  { version = "0.4" }
-wasm-compose = { git = "https://github.com/bytecodealliance/wasm-tools", branch = "main" }
+wasm-compose = { git = "https://github.com/bytecodealliance/wasm-tools" }
 wasmtime = { version = "10.0.1", features = ["component-model"] }
 wasmtime-wasi =  "10.0.1"
 
 [workspace.dependencies]
 anyhow = "1"
-wit-bindgen = "0.8.0"
+wit-bindgen = { git = "https://github.com/bytecodealliance/wit-bindgen" }

README.md

@@ -17,20 +17,58 @@ The virtualized component can be composed into a WASI Preview2 component with `w
 Subsystem support:
 
 - [x] Environment virtualization
-- [ ] Filesystem virtualization
+- [x] Filesystem virtualization
+- [ ] Stdio
+- [ ] Sockets
+- [ ] Clocks
+- [ ] [Your suggestion here](https://github.com/bytecodealliance/WASI-Virt/issues/new)
 
-### Example
+While current virtualization support is limited, the goal for this project is to support a wide range of WASI virtualization use cases.
+
+### Explainer
+
+When wanting to run WebAssembly Components depending on WASI APIs in other environments it can provide
+a point of friction having to port WASI interop to every target platform.
+
+In addition having full unrestricted access to core operating system APIs is a security concern.
+
+WASI Virt allows taking a component that depends on WASI APIs and using a virtualized adapter to convert
+it into a component that no longer depends on those WASI APIs, or conditionally only depends on them in 
+a configurable way.
+
+For example, consider converting an application to a WebAssembly Component that assumes it can load
+read some files from the filesystem, but never needs to write.
+
+Using WASI Virt, those specific file paths can be mounted and virtualized into the component itself as 
+a post-compile operation, while banning the final component from being able to access the host's filesystem at
+all. The inner program still imports a wasi filesystem, but the filesystem implementation is provided by another component, rather than in the host environment. The composition of these two components no longer has a
+filesystem import, so it can be run in hosts (or other components) which do not provide a filesystem API.
+
+### Basic Usage
 
 ```rs
 use std::fs;
-use wasi_virt::WasiVirt;
+use wasi_virt::{WasiVirt, FsEntry};
 
 fn main() {
     let virt_component_bytes = WasiVirt::new()
         // provide an allow list of host env vars
         .env_host_allow(&["PUBLIC_ENV_VAR"])
         // provide custom env overrides
         .env_overrides(&[("SOME", "ENV"), ("VAR", "OVERRIDES")])
+        // mount and virtualize a local directory recursively
+        .fs_preopen("/dir", FsEntry::Virtualize("/local/dir"))
+        // create a virtual directory containing some virtual files
+        .fs_preopen("/another-dir", FsEntry::Dir(BTreeMap::from([
+          // create a virtual file from the given UTF8 source
+          ("file.txt", FsEntry::Source("Hello world")),
+          // create a virtual file read from a local file at
+          // virtualization time
+          ("another.wasm", FsEntry::Virtualize("/local/another.wasm"))
+          // create a virtual file which reads from a given file
+          // path at runtime using the runtime host filesystem API
+          ("host.txt", FsEntry::RuntimeFile("/runtime/host/path.txt"))
+        ])))
         .create()
         .unwrap();
     fs::write("virt.component.wasm", virt_component_bytes).unwrap();
@@ -43,6 +81,8 @@ With the created `virt.component.wasm` component, this can now be composed into
 wasm-tools compose mycomponent.wasm -d virt.component.wasm -o out.component.wasm
 ```
 
+When configuring a virtualization that does not fall back to the host, imports to the subsystem will be entirely stripped from the component.
+
 ## CLI
 
 A CLI is also provided in this crate supporting:
@@ -51,22 +91,41 @@ A CLI is also provided in this crate supporting:
 wasi-virt config.toml -o virt.wasm
 ```
 
+### Configuration
+
 With the configuration file format:
 
 ```
+### Environment Virtualization
 [env]
-# Support all env vars on the final host (apart from the overrides)
-# Set to "none" to entirely encapsulate the host env
-host = "all"
-# Always ensures that this env var and value is set
+### Set environment variable values:
 overrides = [["CUSTOM", "VAL"]]
-```
-
-Allow lists and deny lists can also be provided via:
-
-```
-[env.host]
-allow = ["ENV_KEY"] # Or Deny = ...
+### Enable environment vars for the host:
+host = "all"
+### Alternatively create an allow list:
+# [env.host]
+# allow = ["ENV_KEY"]
+### or deny list:
+# [env.host]
+# deny = ["ENV_KEY"]
+
+### FS Virtualization
+
+### Create a virtual directory with file.txt from
+### the provided inline UTF8 string, and with another.wasm
+### inlined into the virtual adapter from the local filesystem
+### path at virtualization time:
+[fs.preopens."/".dir]
+"file.txt" = { source = "inner contents" }
+"another.wasm" = { virtualize = "/local/path/to/another.wasm" }
+
+### Mount a local directory as a virtualized directory:
+[fs.preopens."/dir"]
+virtualize = "/local/path"
+
+### Mount a passthrough runtime host directory:
+[fs.preopens."/runtime-host"]
+runtime = "/runtime/path"
 ```
 
 # License

build.rs

@@ -1,5 +1,6 @@
 use anyhow::{anyhow, Result};
-use std::{env, process::Command};
+use std::env;
+use std::process::Command;
 
 fn cmd(arg: &str) -> Result<()> {
     let mut cmd = if cfg!(target_os = "windows") {
@@ -23,15 +24,22 @@ fn cmd(arg: &str) -> Result<()> {
 }
 
 fn main() -> Result<()> {
-    if env::var("BUILDING_VIRT").is_err() {
-        env::set_var("BUILDING_VIRT", "1");
-        cmd("cargo +nightly build -p virtual-adapter --target wasm32-wasi --release -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort")?;
-        cmd("cp target/wasm32-wasi/release/virtual_adapter.wasm lib/")?;
+    if env::var("BUILDING_VIRT").is_ok() {
+        return Ok(());
     }
+    env::set_var("BUILDING_VIRT", "1");
+
+    // build the main virtual adapter
+    cmd("cargo +nightly build -p virtual-adapter --target wasm32-wasi --release -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort")?;
+    cmd("cp target/wasm32-wasi/release/virtual_adapter.wasm lib/")?;
 
     println!("cargo:rerun-if-changed=Cargo.toml");
+    println!("cargo:rerun-if-changed=Cargo.lock");
     println!("cargo:rerun-if-changed=virtual-adapter/Cargo.toml");
     println!("cargo:rerun-if-changed=virtual-adapter/src/lib.rs");
+    println!("cargo:rerun-if-changed=virtual-adapter/src/fs.rs");
+    println!("cargo:rerun-if-changed=virtual-adapter/src/env.rs");
+    println!("cargo:rerun-if-changed=wit/virt.wit");
     println!("cargo:rerun-if-changed=build.rs");
     Ok(())
 }

src/bin/wasi-virt.rs

@@ -16,17 +16,7 @@ use wasi_virt::{create_virt, VirtOpts};
 struct Args {
     /// Virtualization TOML configuration
     ///
-    /// Example configuration:
-    ///  
-    ///   [env]
-    ///   host = "All" # or "None"
-    ///   overrides = [["CUSTOM", "VAL"]]
-    ///
-    /// Alternatively, allow or deny env keys for the host can be configured via:
-    ///
-    ///   [env.host]
-    ///   Allow = ["ENV_KEY"] # Or Deny = ...
-    ///
+    /// As defined in [`VirtOpts`]
     #[arg(short, long, verbatim_doc_comment)]
     config: String,
 
@@ -42,7 +32,14 @@ fn main() -> Result<()> {
 
     let virt_component = create_virt(&virt_cfg)?;
 
-    fs::write(args.out, virt_component)?;
+    if virt_component.virtual_files.len() > 0 {
+        println!("Virtualized files from local filesystem:\n");
+        for (virtual_path, original_path) in virt_component.virtual_files {
+            println!("  - {virtual_path} : {original_path}");
+        }
+    }
+
+    fs::write(args.out, virt_component.adapter)?;
 
     Ok(())
 }