diff --git a/Cargo.lock b/Cargo.lock index 3c0392b..b0c82fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "async-channel" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf46fee83e5ccffc220104713af3292ff9bc7c64c7de289f66dae8e38d826833" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" dependencies = [ "concurrent-queue", "event-listener", @@ -128,7 +128,7 @@ dependencies = [ "async-lock", "async-task", "concurrent-queue", - "fastrand", + "fastrand 1.9.0", "futures-lite", "slab", ] @@ -218,7 +218,7 @@ checksum = "a564d521dd56509c4c47480d00b80ee55f7e385ae48db5744c67ad50c92d2ebf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -279,7 +279,7 @@ dependencies = [ "async-lock", "async-task", "atomic-waker", - "fastrand", + "fastrand 1.9.0", "futures-lite", "log", ] @@ -376,9 +376,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.3.11" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" +checksum = "3eab9e8ceb9afdade1ab3f0fd8dbce5b1b2f468ad653baf10e771781b2b67b73" dependencies = [ "clap_builder", "clap_derive", @@ -387,9 +387,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.11" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" +checksum = "9f2763db829349bf00cfc06251268865ed4363b93a943174f638daf3ecdba2cd" dependencies = [ "anstream", "anstyle", @@ -399,14 +399,14 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.2" +version = "4.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" dependencies = [ "heck 0.4.1", "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -415,6 +415,16 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + [[package]] name = "colorchoice" version = "1.0.0" @@ -552,7 +562,7 @@ dependencies = [ "itertools", "log", "smallvec", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", "wasmtime-types", ] @@ -618,6 +628,50 @@ dependencies = [ "typenum", ] +[[package]] +name = "cxx" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f68e12e817cb19eaab81aaec582b4052d07debd3c3c6b083b9d361db47c7dc9d" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e789217e4ab7cf8cc9ce82253180a9fe331f35f5d339f0ccfe0270b39433f397" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.25", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78a19f4c80fd9ab6c882286fa865e92e07688f4387370a209508014ead8751d0" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fcfa71f66c8563c4fa9dd2bb68368d50267856f831ac5d85367e0805f9606c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.25", +] + [[package]] name = "debugid" version = "0.8.0" @@ -708,9 +762,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88bffebc5d80432c9b140ee17875ff173a8ab62faad5b257da912bd2f6c1c0a1" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" @@ -754,6 +808,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fastrand" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" + [[package]] name = "fd-lock" version = "4.0.0" @@ -761,7 +821,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b0377f1edc77dbd1118507bc7a66e4ab64d2b90c66f90726dc801e73a8c68f9" dependencies = [ "cfg-if", - "rustix 0.38.3", + "rustix 0.38.4", "windows-sys", ] @@ -775,6 +835,14 @@ dependencies = [ "log", ] +[[package]] +name = "file-read" +version = "0.1.0" +dependencies = [ + "anyhow", + "wit-bindgen", +] + [[package]] name = "fixedbitset" version = "0.4.2" @@ -797,7 +865,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d167b646a876ba8fda6b50ac645cfd96242553cbaf0ca4fccaa39afcbf0801f" dependencies = [ "io-lifetimes 1.0.11", - "rustix 0.38.3", + "rustix 0.38.4", "windows-sys", ] @@ -828,7 +896,7 @@ version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" dependencies = [ - "fastrand", + "fastrand 1.9.0", "futures-core", "futures-io", "memchr", @@ -1060,7 +1128,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.3", + "rustix 0.38.4", "windows-sys", ] @@ -1139,10 +1207,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" [[package]] -name = "linked-hash-map" -version = "0.5.6" +name = "link-cplusplus" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] [[package]] name = "linux-raw-sys" @@ -1311,9 +1382,9 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b368fba921b0dce7e60f5e04ec15e565b3303972b42bcfde1d0713b881959eb" +checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" dependencies = [ "unicode-ident", ] @@ -1338,6 +1409,17 @@ dependencies = [ "unicase", ] +[[package]] +name = "pulldown-cmark" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a1a2f1f0a7ecff9c31abbe177637be0e97a0aef46cf8738ece09327985d998" +dependencies = [ + "bitflags 1.3.2", + "memchr", + "unicase", +] + [[package]] name = "quote" version = "1.0.29" @@ -1408,6 +1490,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "redox_users" version = "0.4.3" @@ -1415,15 +1506,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" dependencies = [ "getrandom", - "redox_syscall", + "redox_syscall 0.2.16", "thiserror", ] [[package]] name = "regalloc2" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12513beb38dd35aab3ac5f5b89fd0330159a0dc21d5309d75073011bbc8032b0" +checksum = "5b4dcbd3a2ae7fb94b5813fa0e957c6ab51bf5d0a8ee1b69e0c2d0f1e6eb8485" dependencies = [ "hashbrown 0.13.2", "log", @@ -1446,9 +1537,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9aaecc05d5c4b5f7da074b9a0d1a0867e71fd36e7fc0482d8bcfe8e8fc56290" +checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" dependencies = [ "aho-corasick", "memchr", @@ -1457,9 +1548,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab07dc67230e4a4718e70fd5c20055a4334b121f1f9db8fe63ef39ce9b8c846" +checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" [[package]] name = "rustc-demangle" @@ -1491,9 +1582,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac5ffa1efe7548069688cd7028f32591853cd7b5b756d41bcffd2353e4fc75b4" +checksum = "0a962918ea88d644592894bc6dc55acc6c0956488adcebbfb6e273506b7fd6e5" dependencies = [ "bitflags 2.3.3", "errno", @@ -1502,6 +1593,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + [[package]] name = "ryu" version = "1.0.14" @@ -1514,6 +1611,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + [[package]] name = "semver" version = "1.0.17" @@ -1522,29 +1625,29 @@ checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed" [[package]] name = "serde" -version = "1.0.167" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7daf513456463b42aa1d94cff7e0c24d682b429f020b9afa4f5ba5c40a22b237" +checksum = "30e27d1e4fd7659406c492fd6cfaf2066ba8773de45ca75e855590f856dc34a9" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.167" +version = "1.0.171" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b69b106b68bc8054f0e974e70d19984040f8a5cf9215ca82626ea4853f82c4b9" +checksum = "389894603bd18c46fa56231694f8d827779c0951a667087194cf9de94ed24682" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] name = "serde_json" -version = "1.0.100" +version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1e14e89be7aa4c4b78bdbdc9eb5bf8517829a600ae8eaa39a6e1d960b5185c" +checksum = "b5062a995d481b2308b6064e9af76011f2921c35f97b0468811ed9f6cd91dfed" dependencies = [ "itoa", "ryu", @@ -1562,14 +1665,15 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.8.26" +version = "0.9.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" +checksum = "452e67b9c20c37fa79df53201dc03839651086ed9bbe92b3ca585ca9fdaa7d85" dependencies = [ - "indexmap 1.9.3", + "indexmap 2.0.0", + "itoa", "ryu", "serde", - "yaml-rust", + "unsafe-libyaml", ] [[package]] @@ -1623,6 +1727,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "spdx" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" +dependencies = [ + "smallvec", +] + [[package]] name = "sptr" version = "0.3.2" @@ -1641,6 +1754,25 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strum" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" + +[[package]] +name = "strum_macros" +version = "0.24.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 1.0.109", +] + [[package]] name = "syn" version = "1.0.109" @@ -1654,9 +1786,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.23" +version = "2.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fb7d6d8281a51045d62b8eb3a7d1ce347b76f312af50cd3dc0af39c87c1737" +checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" dependencies = [ "proc-macro2", "quote", @@ -1674,16 +1806,29 @@ dependencies = [ "cap-std", "fd-lock", "io-lifetimes 2.0.2", - "rustix 0.38.3", + "rustix 0.38.4", "windows-sys", "winx 0.36.1", ] [[package]] name = "target-lexicon" -version = "0.12.8" +version = "0.12.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b1c7f239eb94671427157bd93b3694320f3668d4e1eff08c7285366fd777fac" +checksum = "df8e77cb757a61f51b947ec4a7e3646efd825b73561db1c232a8ccb639e611a0" + +[[package]] +name = "tempfile" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5486094ee78b2e5038a6382ed7645bc084dc2ec433426ca4c3cb61e2007b8998" +dependencies = [ + "cfg-if", + "fastrand 2.0.0", + "redox_syscall 0.3.5", + "rustix 0.38.4", + "windows-sys", +] [[package]] name = "termcolor" @@ -1711,7 +1856,7 @@ checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -1761,9 +1906,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.19.12" +version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c500344a19072298cd05a7224b3c0c629348b78692bf48466c5238656e315a78" +checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ "indexmap 2.0.0", "serde", @@ -1793,7 +1938,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", ] [[package]] @@ -1859,6 +2004,12 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" +[[package]] +name = "unsafe-libyaml" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1865806a559042e51ab5414598446a5871b561d21b6764f2eabb0dd481d880a6" + [[package]] name = "url" version = "2.4.0" @@ -1920,7 +2071,7 @@ dependencies = [ "leb128", "log", "walrus-macro", - "wasm-encoder 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-encoder 0.29.0", "wasmparser 0.80.2", ] @@ -1999,9 +2150,11 @@ dependencies = [ "toml 0.7.6", "walrus", "wasm-compose", + "wasm-metadata 0.9.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wasm-opt", "wasmtime", "wasmtime-wasi", - "wit-component", + "wit-component 0.12.0 (git+https://github.com/bytecodealliance/wasm-tools)", ] [[package]] @@ -2025,7 +2178,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", "wasm-bindgen-shared", ] @@ -2059,7 +2212,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.23", + "syn 2.0.25", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2072,8 +2225,8 @@ checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" [[package]] name = "wasm-compose" -version = "0.2.17" -source = "git+https://github.com/bytecodealliance/wasm-tools?branch=main#3379199d0d58da323740f7fdaa4618b459851b5d" +version = "0.3.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" dependencies = [ "anyhow", "heck 0.4.1", @@ -2083,9 +2236,9 @@ dependencies = [ "serde", "serde_yaml", "smallvec", - "wasm-encoder 0.29.0 (git+https://github.com/bytecodealliance/wasm-tools?branch=main)", - "wasmparser 0.107.0 (git+https://github.com/bytecodealliance/wasm-tools?branch=main)", - "wat 1.0.66 (git+https://github.com/bytecodealliance/wasm-tools?branch=main)", + "wasm-encoder 0.30.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wasmparser 0.108.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wat 1.0.67 (git+https://github.com/bytecodealliance/wasm-tools)", ] [[package]] @@ -2099,23 +2252,88 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.29.0" -source = "git+https://github.com/bytecodealliance/wasm-tools?branch=main#3379199d0d58da323740f7fdaa4618b459851b5d" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2f8e9778e04cbf44f58acc301372577375a666b966c50b03ef46144f80436a8" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-encoder" +version = "0.30.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" dependencies = [ "leb128", ] [[package]] name = "wasm-metadata" -version = "0.8.0" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e5156581ff4a302405c44ca7c85347563ca431d15f1a773f12c9c7b9a6cdc9" +checksum = "d51db59397fc650b5f2fc778e4a5c4456cd856bed7fc1ec15f8d3e28229dc463" dependencies = [ "anyhow", - "indexmap 1.9.3", + "indexmap 2.0.0", "serde", - "wasm-encoder 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json", + "spdx", + "wasm-encoder 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.108.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wasm-metadata" +version = "0.9.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" +dependencies = [ + "anyhow", + "indexmap 2.0.0", + "serde", + "serde_json", + "spdx", + "wasm-encoder 0.30.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wasmparser 0.108.0 (git+https://github.com/bytecodealliance/wasm-tools)", +] + +[[package]] +name = "wasm-opt" +version = "0.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65a2799e08026234b07b44da6363703974e75be21430cef00756bbc438c8ff8a" +dependencies = [ + "anyhow", + "libc", + "strum", + "strum_macros", + "tempfile", + "thiserror", + "wasm-opt-cxx-sys", + "wasm-opt-sys", +] + +[[package]] +name = "wasm-opt-cxx-sys" +version = "0.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d26f86d1132245e8bcea8fac7f02b10fb885b6696799969c94d7d3c14db5e1" +dependencies = [ + "anyhow", + "cxx", + "cxx-build", + "wasm-opt-sys", +] + +[[package]] +name = "wasm-opt-sys" +version = "0.113.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "497d069cd3420cdd52154a320b901114a20946878e2de62c670f9d906e472370" +dependencies = [ + "anyhow", + "cc", + "cxx", + "cxx-build", ] [[package]] @@ -2136,8 +2354,18 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.107.0" -source = "git+https://github.com/bytecodealliance/wasm-tools?branch=main#3379199d0d58da323740f7fdaa4618b459851b5d" +version = "0.108.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c956109dcb41436a39391139d9b6e2d0a5e0b158e1293ef352ec977e5e36c5" +dependencies = [ + "indexmap 2.0.0", + "semver", +] + +[[package]] +name = "wasmparser" +version = "0.108.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" dependencies = [ "indexmap 2.0.0", "semver", @@ -2145,12 +2373,12 @@ dependencies = [ [[package]] name = "wasmprinter" -version = "0.2.59" +version = "0.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc960b30b84abca377768f3c62cff3a1c74db8c0f6759ed581827da0bd3a3fed" +checksum = "b76cb909fe3d9b0de58cee1f4072247e680ff5cc1558ccad2790a9de14a23993" dependencies = [ "anyhow", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.108.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -2177,7 +2405,7 @@ dependencies = [ "serde", "serde_json", "target-lexicon", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", "wasmtime-cache", "wasmtime-component-macro", "wasmtime-component-util", @@ -2187,7 +2415,7 @@ dependencies = [ "wasmtime-jit", "wasmtime-runtime", "wasmtime-winch", - "wat 1.0.66 (registry+https://github.com/rust-lang/crates.io-index)", + "wat 1.0.67 (registry+https://github.com/rust-lang/crates.io-index)", "windows-sys", ] @@ -2232,7 +2460,7 @@ dependencies = [ "syn 1.0.109", "wasmtime-component-util", "wasmtime-wit-bindgen", - "wit-parser", + "wit-parser 0.8.0", ] [[package]] @@ -2259,7 +2487,7 @@ dependencies = [ "object", "target-lexicon", "thiserror", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", "wasmtime-cranelift-shared", "wasmtime-environ", ] @@ -2295,8 +2523,8 @@ dependencies = [ "serde", "target-lexicon", "thiserror", - "wasm-encoder 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-encoder 0.29.0", + "wasmparser 0.107.0", "wasmprinter", "wasmtime-component-util", "wasmtime-types", @@ -2399,7 +2627,7 @@ dependencies = [ "cranelift-entity", "serde", "thiserror", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", ] [[package]] @@ -2440,7 +2668,7 @@ dependencies = [ "gimli 0.27.3", "object", "target-lexicon", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", "wasmtime-cranelift-shared", "wasmtime-environ", "winch-codegen", @@ -2454,7 +2682,7 @@ checksum = "d3334b0466a4d340de345cda83474d1d2c429770c3d667877971407672bc618a" dependencies = [ "anyhow", "heck 0.4.1", - "wit-parser", + "wit-parser 0.8.0", ] [[package]] @@ -2468,42 +2696,42 @@ dependencies = [ [[package]] name = "wast" -version = "60.0.0" +version = "61.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd06cc744b536e30387e72a48fdd492105b9c938bb4f415c39c616a7a0a697ad" +checksum = "dc6b347851b52fd500657d301155c79e8c67595501d179cef87b6f04ebd25ac4" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-encoder 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wast" -version = "60.0.0" -source = "git+https://github.com/bytecodealliance/wasm-tools?branch=main#3379199d0d58da323740f7fdaa4618b459851b5d" +version = "61.0.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.29.0 (git+https://github.com/bytecodealliance/wasm-tools?branch=main)", + "wasm-encoder 0.30.0 (git+https://github.com/bytecodealliance/wasm-tools)", ] [[package]] name = "wat" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5abe520f0ab205366e9ac7d3e6b2fc71de44e32a2b58f2ec871b6b575bdcea3b" +checksum = "459e764d27c3ab7beba1ebd617cc025c7e76dea6e7c5ce3189989a970aea3491" dependencies = [ - "wast 60.0.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wast 61.0.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wat" -version = "1.0.66" -source = "git+https://github.com/bytecodealliance/wasm-tools?branch=main#3379199d0d58da323740f7fdaa4618b459851b5d" +version = "1.0.67" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" dependencies = [ - "wast 60.0.0 (git+https://github.com/bytecodealliance/wasm-tools?branch=main)", + "wast 61.0.0 (git+https://github.com/bytecodealliance/wasm-tools)", ] [[package]] @@ -2601,7 +2829,7 @@ dependencies = [ "regalloc2", "smallvec", "target-lexicon", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.107.0", "wasmtime-environ", ] @@ -2673,9 +2901,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "winnow" -version = "0.4.8" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9482fe6ceabdf32f3966bfdd350ba69256a97c30253dc616fe0005af24f164e" +checksum = "81fac9742fd1ad1bd9643b991319f72dd031016d44b77039a26977eb667141e7" dependencies = [ "memchr", ] @@ -2704,8 +2932,7 @@ dependencies = [ [[package]] name = "wit-bindgen" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "392d16e9e46cc7ca98125bc288dd5e4db469efe8323d3e0dac815ca7f2398522" +source = "git+https://github.com/bytecodealliance/wit-bindgen#4e2dcd443132d3e8d1234d6f74d3dd95f0ff09a6" dependencies = [ "bitflags 2.3.3", "wit-bindgen-rust-macro", @@ -2714,32 +2941,29 @@ dependencies = [ [[package]] name = "wit-bindgen-core" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d422d36cbd78caa0e18c3371628447807c66ee72466b69865ea7e33682598158" +source = "git+https://github.com/bytecodealliance/wit-bindgen#4e2dcd443132d3e8d1234d6f74d3dd95f0ff09a6" dependencies = [ "anyhow", - "wit-component", - "wit-parser", + "wit-component 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wit-parser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wit-bindgen-rust" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b76db68264f5d2089dc4652581236d8e75c5b89338de6187716215fd0e68ba3" +source = "git+https://github.com/bytecodealliance/wit-bindgen#4e2dcd443132d3e8d1234d6f74d3dd95f0ff09a6" dependencies = [ "heck 0.4.1", - "wasm-metadata", + "wasm-metadata 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "wit-bindgen-core", "wit-bindgen-rust-lib", - "wit-component", + "wit-component 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wit-bindgen-rust-lib" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c50f334bc08b0903a43387f6eea6ef6aa9eb2a085729f1677b29992ecef20ba" +source = "git+https://github.com/bytecodealliance/wit-bindgen#4e2dcd443132d3e8d1234d6f74d3dd95f0ff09a6" dependencies = [ "heck 0.4.1", "wit-bindgen-core", @@ -2748,31 +2972,46 @@ dependencies = [ [[package]] name = "wit-bindgen-rust-macro" version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced38a5e174940c6a41ae587babeadfd2e2c2dc32f3b6488bcdca0e8922cf3f3" +source = "git+https://github.com/bytecodealliance/wit-bindgen#4e2dcd443132d3e8d1234d6f74d3dd95f0ff09a6" dependencies = [ "anyhow", "proc-macro2", - "syn 2.0.23", + "syn 2.0.25", "wit-bindgen-core", "wit-bindgen-rust", - "wit-component", + "wit-bindgen-rust-lib", + "wit-component 0.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "wit-component" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbd4c7f8f400327c482c88571f373844b7889e61460650d650fc5881bb3575c" +checksum = "253bd426c532f1cae8c633c517c63719920535f3a7fada3589de40c5b734e393" dependencies = [ "anyhow", "bitflags 1.3.2", - "indexmap 1.9.3", + "indexmap 2.0.0", "log", - "wasm-encoder 0.29.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wasm-metadata", - "wasmparser 0.107.0 (registry+https://github.com/rust-lang/crates.io-index)", - "wit-parser", + "wasm-encoder 0.30.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasm-metadata 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wasmparser 0.108.0 (registry+https://github.com/rust-lang/crates.io-index)", + "wit-parser 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "wit-component" +version = "0.12.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" +dependencies = [ + "anyhow", + "bitflags 2.3.3", + "indexmap 2.0.0", + "log", + "wasm-encoder 0.30.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wasm-metadata 0.9.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wasmparser 0.108.0 (git+https://github.com/bytecodealliance/wasm-tools)", + "wit-parser 0.9.0 (git+https://github.com/bytecodealliance/wasm-tools)", ] [[package]] @@ -2785,31 +3024,53 @@ dependencies = [ "id-arena", "indexmap 1.9.3", "log", - "pulldown-cmark", + "pulldown-cmark 0.8.0", "semver", "unicode-xid", "url", ] [[package]] -name = "witx" -version = "0.9.1" +name = "wit-parser" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" +checksum = "82f2afd756820d516d4973f67a739ca5529cc872d80114be17d4bba79375981c" dependencies = [ "anyhow", + "id-arena", + "indexmap 2.0.0", "log", - "thiserror", - "wast 35.0.2", + "pulldown-cmark 0.9.3", + "semver", + "unicode-xid", + "url", +] + +[[package]] +name = "wit-parser" +version = "0.9.0" +source = "git+https://github.com/bytecodealliance/wasm-tools#3948ae92972785b252f3c76af601ac83ee3ac3da" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.0.0", + "log", + "pulldown-cmark 0.9.3", + "semver", + "unicode-xid", + "url", ] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "witx" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "e366f27a5cabcddb2706a78296a40b8fcc451e1a6aba2fc1d94b4a01bdaaef4b" dependencies = [ - "linked-hash-map", + "anyhow", + "log", + "thiserror", + "wast 35.0.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index b170e5e..a834f7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,6 +14,7 @@ name = "wasi-virt" [workspace] members = [ "virtual-adapter", + "tests/components/file-read", "tests/components/get-env" ] @@ -30,7 +31,9 @@ clap = { version = "4", features = ["derive"] } serde = { version = "1", features = ["derive"] } toml = "0.7" walrus = "0.20.1" -wit-component = "0.11.0" +wasm-metadata = { git = "https://github.com/bytecodealliance/wasm-tools" } +wasm-opt = "0.113.0" +wit-component = { git = "https://github.com/bytecodealliance/wasm-tools" } [build-dependencies] anyhow = "1" @@ -40,10 +43,10 @@ anyhow = "1" async-std = { version = "1", features = ["attributes"] } cap-std = "1.0.12" heck = { version = "0.4" } -wasm-compose = { git = "https://github.com/bytecodealliance/wasm-tools", branch = "main" } +wasm-compose = { git = "https://github.com/bytecodealliance/wasm-tools" } wasmtime = { version = "10.0.1", features = ["component-model"] } wasmtime-wasi = "10.0.1" [workspace.dependencies] anyhow = "1" -wit-bindgen = "0.8.0" +wit-bindgen = { git = "https://github.com/bytecodealliance/wit-bindgen" } diff --git a/README.md b/README.md index 6b0c8f5..9ed8ca4 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,38 @@ The virtualized component can be composed into a WASI Preview2 component with `w Subsystem support: - [x] Environment virtualization -- [ ] Filesystem virtualization +- [x] Filesystem virtualization +- [ ] Stdio +- [ ] Sockets +- [ ] Clocks +- [ ] [Your suggestion here](https://github.com/bytecodealliance/WASI-Virt/issues/new) -### Example +While current virtualization support is limited, the goal for this project is to support a wide range of WASI virtualization use cases. + +### Explainer + +When wanting to run WebAssembly Components depending on WASI APIs in other environments it can provide +a point of friction having to port WASI interop to every target platform. + +In addition having full unrestricted access to core operating system APIs is a security concern. + +WASI Virt allows taking a component that depends on WASI APIs and using a virtualized adapter to convert +it into a component that no longer depends on those WASI APIs, or conditionally only depends on them in +a configurable way. + +For example, consider converting an application to a WebAssembly Component that assumes it can load +read some files from the filesystem, but never needs to write. + +Using WASI Virt, those specific file paths can be mounted and virtualized into the component itself as +a post-compile operation, while banning the final component from being able to access the host's filesystem at +all. The inner program still imports a wasi filesystem, but the filesystem implementation is provided by another component, rather than in the host environment. The composition of these two components no longer has a +filesystem import, so it can be run in hosts (or other components) which do not provide a filesystem API. + +### Basic Usage ```rs use std::fs; -use wasi_virt::WasiVirt; +use wasi_virt::{WasiVirt, FsEntry}; fn main() { let virt_component_bytes = WasiVirt::new() @@ -31,6 +56,19 @@ fn main() { .env_host_allow(&["PUBLIC_ENV_VAR"]) // provide custom env overrides .env_overrides(&[("SOME", "ENV"), ("VAR", "OVERRIDES")]) + // mount and virtualize a local directory recursively + .fs_preopen("/dir", FsEntry::Virtualize("/local/dir")) + // create a virtual directory containing some virtual files + .fs_preopen("/another-dir", FsEntry::Dir(BTreeMap::from([ + // create a virtual file from the given UTF8 source + ("file.txt", FsEntry::Source("Hello world")), + // create a virtual file read from a local file at + // virtualization time + ("another.wasm", FsEntry::Virtualize("/local/another.wasm")) + // create a virtual file which reads from a given file + // path at runtime using the runtime host filesystem API + ("host.txt", FsEntry::RuntimeFile("/runtime/host/path.txt")) + ]))) .create() .unwrap(); fs::write("virt.component.wasm", virt_component_bytes).unwrap(); @@ -43,6 +81,8 @@ With the created `virt.component.wasm` component, this can now be composed into wasm-tools compose mycomponent.wasm -d virt.component.wasm -o out.component.wasm ``` +When configuring a virtualization that does not fall back to the host, imports to the subsystem will be entirely stripped from the component. + ## CLI A CLI is also provided in this crate supporting: @@ -51,22 +91,41 @@ A CLI is also provided in this crate supporting: wasi-virt config.toml -o virt.wasm ``` +### Configuration + With the configuration file format: ``` +### Environment Virtualization [env] -# Support all env vars on the final host (apart from the overrides) -# Set to "none" to entirely encapsulate the host env -host = "all" -# Always ensures that this env var and value is set +### Set environment variable values: overrides = [["CUSTOM", "VAL"]] -``` - -Allow lists and deny lists can also be provided via: - -``` -[env.host] -allow = ["ENV_KEY"] # Or Deny = ... +### Enable environment vars for the host: +host = "all" +### Alternatively create an allow list: +# [env.host] +# allow = ["ENV_KEY"] +### or deny list: +# [env.host] +# deny = ["ENV_KEY"] + +### FS Virtualization + +### Create a virtual directory with file.txt from +### the provided inline UTF8 string, and with another.wasm +### inlined into the virtual adapter from the local filesystem +### path at virtualization time: +[fs.preopens."/".dir] +"file.txt" = { source = "inner contents" } +"another.wasm" = { virtualize = "/local/path/to/another.wasm" } + +### Mount a local directory as a virtualized directory: +[fs.preopens."/dir"] +virtualize = "/local/path" + +### Mount a passthrough runtime host directory: +[fs.preopens."/runtime-host"] +runtime = "/runtime/path" ``` # License diff --git a/build.rs b/build.rs index daef9d2..ce2c7a1 100644 --- a/build.rs +++ b/build.rs @@ -1,5 +1,6 @@ use anyhow::{anyhow, Result}; -use std::{env, process::Command}; +use std::env; +use std::process::Command; fn cmd(arg: &str) -> Result<()> { let mut cmd = if cfg!(target_os = "windows") { @@ -23,15 +24,22 @@ fn cmd(arg: &str) -> Result<()> { } fn main() -> Result<()> { - if env::var("BUILDING_VIRT").is_err() { - env::set_var("BUILDING_VIRT", "1"); - cmd("cargo +nightly build -p virtual-adapter --target wasm32-wasi --release -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort")?; - cmd("cp target/wasm32-wasi/release/virtual_adapter.wasm lib/")?; + if env::var("BUILDING_VIRT").is_ok() { + return Ok(()); } + env::set_var("BUILDING_VIRT", "1"); + + // build the main virtual adapter + cmd("cargo +nightly build -p virtual-adapter --target wasm32-wasi --release -Z build-std=std,panic_abort -Z build-std-features=panic_immediate_abort")?; + cmd("cp target/wasm32-wasi/release/virtual_adapter.wasm lib/")?; println!("cargo:rerun-if-changed=Cargo.toml"); + println!("cargo:rerun-if-changed=Cargo.lock"); println!("cargo:rerun-if-changed=virtual-adapter/Cargo.toml"); println!("cargo:rerun-if-changed=virtual-adapter/src/lib.rs"); + println!("cargo:rerun-if-changed=virtual-adapter/src/fs.rs"); + println!("cargo:rerun-if-changed=virtual-adapter/src/env.rs"); + println!("cargo:rerun-if-changed=wit/virt.wit"); println!("cargo:rerun-if-changed=build.rs"); Ok(()) } diff --git a/lib/virtual_adapter.wasm b/lib/virtual_adapter.wasm index 44ad6be..781ad34 100755 Binary files a/lib/virtual_adapter.wasm and b/lib/virtual_adapter.wasm differ diff --git a/lib/wasi_snapshot_preview1.command.wasm b/lib/wasi_snapshot_preview1.command.wasm deleted file mode 100644 index 1c26541..0000000 Binary files a/lib/wasi_snapshot_preview1.command.wasm and /dev/null differ diff --git a/src/bin/wasi-virt.rs b/src/bin/wasi-virt.rs index bc3e6ef..75f093f 100644 --- a/src/bin/wasi-virt.rs +++ b/src/bin/wasi-virt.rs @@ -16,17 +16,7 @@ use wasi_virt::{create_virt, VirtOpts}; struct Args { /// Virtualization TOML configuration /// - /// Example configuration: - /// - /// [env] - /// host = "All" # or "None" - /// overrides = [["CUSTOM", "VAL"]] - /// - /// Alternatively, allow or deny env keys for the host can be configured via: - /// - /// [env.host] - /// Allow = ["ENV_KEY"] # Or Deny = ... - /// + /// As defined in [`VirtOpts`] #[arg(short, long, verbatim_doc_comment)] config: String, @@ -42,7 +32,14 @@ fn main() -> Result<()> { let virt_component = create_virt(&virt_cfg)?; - fs::write(args.out, virt_component)?; + if virt_component.virtual_files.len() > 0 { + println!("Virtualized files from local filesystem:\n"); + for (virtual_path, original_path) in virt_component.virtual_files { + println!(" - {virtual_path} : {original_path}"); + } + } + + fs::write(args.out, virt_component.adapter)?; Ok(()) } diff --git a/src/data.rs b/src/data.rs new file mode 100644 index 0000000..ee54588 --- /dev/null +++ b/src/data.rs @@ -0,0 +1,219 @@ +use crate::walrus_ops::{ + bump_stack_global, get_exported_func, get_memory_id, remove_exported_func, +}; +use anyhow::{bail, Result}; +use std::collections::HashMap; +use walrus::{ + ActiveData, ActiveDataLocation, DataKind, ElementKind, FunctionBuilder, FunctionId, + FunctionKind, InitExpr, Module, ValType, +}; + +/// Data section +/// Because data is stack-allocated we create a corresponding byte vector as large +/// as the stack, zero fill it then populate it backwards from the +/// stack pointer. The final stack pointer will then be against the smaller stack. +/// This way, returned pointers are correct from the start, directly +/// corresponding to offsets in the slice, and can be known without having to +/// separately perform relocations. +/// We could alternatively do a smaller allocation then progressively grow, +/// while supporting reverse population, but this alloc seems fine for now. +pub(crate) struct Data { + stack_start: usize, + stack_ptr: usize, + strings: HashMap, + bytes: Vec, + passive_segments: Vec>, +} + +pub(crate) trait WasmEncode +where + Self: Sized, +{ + fn align() -> usize; + fn size() -> usize; + fn encode(&self, bytes: &mut [u8]); +} + +impl WasmEncode for u32 { + fn align() -> usize { + 4 + } + + fn size() -> usize { + 4 + } + + fn encode(&self, bytes: &mut [u8]) { + bytes[0..4].copy_from_slice(&self.to_le_bytes()); + } +} + +impl Data { + pub fn new(stack_start: usize) -> Self { + let mut bytes = Vec::new(); + bytes.resize(stack_start, 0); + Data { + strings: HashMap::new(), + stack_start, + stack_ptr: stack_start, + bytes, + passive_segments: Vec::new(), + } + } + + pub fn passive_bytes(&mut self, bytes: &[u8]) -> u32 { + let passive_idx = self.passive_segments.len(); + self.passive_segments.push(bytes.to_vec()); + passive_idx as u32 + } + + fn stack_alloc<'a>(&'a mut self, data_len: usize, align: usize) -> Result<&'a mut [u8]> { + if data_len + align > self.stack_ptr { + bail!("Out of stack space for file virtualization, use passive segments by decreasing the passive cutoff instead"); + } + let mut new_stack_ptr = self.stack_ptr - data_len; + if new_stack_ptr % align != 0 { + let padding = align - (self.stack_ptr % align); + new_stack_ptr -= padding; + } + self.stack_ptr = new_stack_ptr; + Ok(&mut self.bytes[new_stack_ptr..new_stack_ptr + data_len]) + } + + pub fn stack_bytes(&mut self, bytes: &[u8]) -> Result { + self.stack_alloc(bytes.len(), 1)?.copy_from_slice(bytes); + Ok(self.stack_ptr as u32) + } + + /// Allocate some bytes into the data section, return the pointer + /// Note this is only safe for T being repr(C) / packed + pub fn write_slice(&mut self, data: &[T]) -> Result { + let size = T::size(); + let bytes = self.stack_alloc(data.len() * size, T::align())?; + let mut cursor = 0; + for item in data { + item.encode(&mut bytes[cursor..cursor + size]); + cursor += size; + } + Ok(self.stack_ptr as u32) + } + + /// Allocate a static string and return its pointer + /// If the string already exists, return the existing pointer + pub fn string(&mut self, str: &str) -> Result { + if let Some(&ptr) = self.strings.get(str) { + return Ok(ptr); + } + // 1 for null termination + // because of zero fill we are already null-terminated + let len = str.as_bytes().len(); + let bytes = self.stack_alloc(len + 1, 1)?; + bytes[0..len].copy_from_slice(str.as_bytes()); + bytes[len] = 0; + self.strings.insert(str.to_string(), self.stack_ptr as u32); + Ok(self.stack_ptr as u32) + } + pub fn finish(mut self, module: &mut Module) -> Result<()> { + // stack embedding + let memory = get_memory_id(module)?; + let rem = (self.stack_start - self.stack_ptr) % 8; + if rem != 0 { + self.stack_ptr -= 8 - rem; + } + bump_stack_global(module, (self.stack_start - self.stack_ptr) as i32)?; + module.data.add( + DataKind::Active(ActiveData { + memory, + location: ActiveDataLocation::Absolute(self.stack_ptr as u32), + }), + self.bytes.as_slice()[self.stack_ptr..self.stack_start].to_vec(), + ); + + // passive segment embedding + // we create one function for each passive segment, due to + if self.passive_segments.len() > 0 { + let alloc_fid = get_exported_func(module, "cabi_realloc")?; + + let offset_local = module.locals.add(ValType::I32); + let len_local = module.locals.add(ValType::I32); + let ptr_local = module.locals.add(ValType::I32); + + let mut passive_fids: Vec> = Vec::new(); + for passive_segment in self.passive_segments { + let passive_id = module.data.add(DataKind::Passive, passive_segment); + + // construct the passive segment allocation function + let mut builder = FunctionBuilder::new( + &mut module.types, + &[ValType::I32, ValType::I32], + &[ValType::I32], + ); + builder + .func_body() + // cabi_realloc args + .i32_const(0) + .i32_const(0) + .i32_const(4) + // Last realloc arg is byte length to allocate + .local_get(len_local) + // mem init arg 0 - destination address + .call(alloc_fid) + .local_tee(ptr_local) + // mem init arg 1 - source segment offset + .local_get(offset_local) + // mem init arg 2 - size of initialization + .local_get(len_local) + .memory_init(memory, passive_id) + // return the allocated pointer + .local_get(ptr_local); + + passive_fids.push(Some( + module + .funcs + .add_local(builder.local_func(vec![offset_local, len_local])), + )); + } + + let passive_tid = module.tables.add_local( + passive_fids.len() as u32, + Some(passive_fids.len() as u32), + ValType::Funcref, + ); + module.elements.add( + ElementKind::Active { + table: passive_tid, + offset: InitExpr::Value(walrus::ir::Value::I32(0)), + }, + ValType::Funcref, + passive_fids, + ); + + // main passive call function + let passive_fn_alloc_type = module + .types + .add(&[ValType::I32, ValType::I32], &[ValType::I32]); + let passive_idx = module.locals.add(ValType::I32); + let mut builder = FunctionBuilder::new( + &mut module.types, + &[ValType::I32, ValType::I32, ValType::I32], + &[ValType::I32], + ); + builder + .func_body() + .local_get(offset_local) + .local_get(len_local) + .local_get(passive_idx) + .call_indirect(passive_fn_alloc_type, passive_tid); + + // update the existing passive_alloc function export with the new function body + let passive_alloc_fid = get_exported_func(module, "passive_alloc")?; + let passive_alloc_func = module.funcs.get_mut(passive_alloc_fid); + passive_alloc_func.kind = + FunctionKind::Local(builder.local_func(vec![passive_idx, offset_local, len_local])); + } + + remove_exported_func(module, "passive_alloc")?; + + Ok(()) + } +} diff --git a/src/lib.rs b/src/lib.rs index 87a1466..08c61ef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,33 @@ -use anyhow::Result; -use env::{create_env_virt, VirtEnv}; +use anyhow::{Context, Result}; use serde::Deserialize; +use std::env; +use std::fs; +use std::time::SystemTime; +use virt_env::{create_env_virt, strip_env_virt, VirtEnv}; +use virt_fs::{create_fs_virt, strip_fs_virt, VirtFs}; +use wasm_metadata::Producers; +use wasm_opt::Feature; +use wasm_opt::OptimizationOptions; +use wit_component::metadata; use wit_component::ComponentEncoder; +use wit_component::StringEncoding; -mod env; +mod data; +mod virt_env; +mod virt_fs; mod walrus_ops; +pub type VirtualFiles = virt_fs::VirtualFiles; + #[derive(Deserialize, Debug, Default, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct VirtOpts { /// Environment virtualization - env: Option, + pub env: Option, + /// Filesystem virtualization + pub fs: Option, + /// Disable wasm-opt run if desired + pub wasm_opt: Option, } #[derive(Debug, Default, Clone)] @@ -17,30 +35,118 @@ pub struct WasiVirt { virt_opts: VirtOpts, } +pub struct VirtResult { + pub adapter: Vec, + pub virtual_files: VirtualFiles, +} + impl WasiVirt { pub fn new() -> Self { Self::default() } - pub fn create(&self) -> Result> { + pub fn create(&self) -> Result { create_virt(&self.virt_opts) } } -pub fn create_virt<'a>(opts: &VirtOpts) -> Result> { +pub fn create_virt<'a>(opts: &VirtOpts) -> Result { let virt_adapter = include_bytes!("../lib/virtual_adapter.wasm"); let config = walrus::ModuleConfig::new(); let mut module = config.parse(virt_adapter)?; + module.name = Some("wasi_virt".into()); - // env virtualization injection if let Some(env) = &opts.env { create_env_virt(&mut module, env)?; + } else { + strip_env_virt(&mut module)?; + } + let virtual_files = if let Some(fs) = &opts.fs { + create_fs_virt(&mut module, fs)? + } else { + strip_fs_virt(&mut module)?; + Default::default() + }; + + // decode the component custom section to strip out the unused world exports + // before reencoding. + let mut component_section = module + .customs + .remove_raw("component-type:virtual-adapter") + .context("Unable to find component section")?; + + let (_, mut bindgen) = metadata::decode(virt_adapter)?; + let (_, pkg_id) = bindgen + .resolve + .package_names + .iter() + .find(|(name, _)| name.namespace == "local") + .unwrap(); + + let base_world = bindgen + .resolve + .select_world(*pkg_id, Some("virtual-base"))?; + + let env_world = bindgen.resolve.select_world(*pkg_id, Some("virtual-env"))?; + let fs_world = bindgen.resolve.select_world(*pkg_id, Some("virtual-fs"))?; + + if opts.env.is_some() { + bindgen.resolve.merge_worlds(env_world, base_world)?; + } + if opts.fs.is_some() { + bindgen.resolve.merge_worlds(fs_world, base_world)?; } - let bytes = module.emit_wasm(); + let mut producers = Producers::default(); + producers.add("processed-by", "wasi-virt", env!("CARGO_PKG_VERSION")); + + component_section.data = metadata::encode( + &bindgen.resolve, + base_world, + StringEncoding::UTF8, + Some(&producers), + )?; + + module.customs.add(component_section); + + let mut bytes = module.emit_wasm(); + + // because we rely on dead code ellimination to remove unnecessary adapter code + // we save into a temporary file and run wasm-opt before returning + // this can be disabled with wasm_opt: false + if opts.wasm_opt.unwrap_or(true) { + let dir = env::temp_dir(); + let tmp_input = dir.join(format!("virt.core.input.{}.wasm", timestamp())); + let tmp_output = dir.join(format!("virt.core.output.{}.wasm", timestamp())); + fs::write(&tmp_input, bytes) + .with_context(|| "Unable to write temporary file for wasm-opt call on adapter")?; + OptimizationOptions::new_optimize_for_size_aggressively() + .enable_feature(Feature::ReferenceTypes) + .run(&tmp_input, &tmp_output) + .with_context(|| "Unable to apply wasm-opt optimization to virt. This can be disabled with wasm_opt: false.") + .or_else(|e| { + fs::remove_file(&tmp_input)?; + Err(e) + })?; + bytes = fs::read(&tmp_output)?; + fs::remove_file(&tmp_input)?; + fs::remove_file(&tmp_output)?; + } // now adapt the virtualized component let encoder = ComponentEncoder::default().validate(true).module(&bytes)?; - encoder.encode() + let encoded = encoder.encode()?; + + Ok(VirtResult { + adapter: encoded, + virtual_files, + }) +} + +fn timestamp() -> u64 { + match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => n.as_secs(), + Err(_) => panic!(), + } } diff --git a/src/env.rs b/src/virt_env.rs similarity index 86% rename from src/env.rs rename to src/virt_env.rs index 02bcf8d..17ceb91 100644 --- a/src/env.rs +++ b/src/virt_env.rs @@ -1,25 +1,30 @@ +use crate::{ + walrus_ops::{ + bump_stack_global, get_active_data_segment, get_memory_id, remove_exported_func, + stub_imported_func, + }, + WasiVirt, +}; use anyhow::{bail, Context, Result}; use serde::Deserialize; use walrus::{ ir::Value, ActiveData, ActiveDataLocation, DataKind, ExportItem, GlobalKind, InitExpr, Module, }; -use crate::{ - walrus_ops::{bump_stack_global, get_active_data_segment, stub_imported_func}, - WasiVirt, -}; - #[derive(Deserialize, Debug, Clone, Default)] +#[serde(deny_unknown_fields)] pub struct VirtEnv { /// Set specific environment variable overrides - overrides: Vec<(String, String)>, + #[serde(default)] + pub overrides: Vec<(String, String)>, /// Define how to embed into the host environment /// (Pass-through / encapsulate / allow / deny) - host: HostEnv, + #[serde(default)] + pub host: HostEnv, } #[derive(Deserialize, Debug, Clone, Default)] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] pub enum HostEnv { /// Apart from the overrides, pass through all environment /// variables from the host @@ -93,17 +98,11 @@ pub fn create_env_virt<'a>(module: &'a mut Module, env: &VirtEnv) -> Result<()> // If host env is disabled, remove its import entirely // replacing it with a stub panic if matches!(env.host, HostEnv::None) { - stub_imported_func(module, "wasi:cli-base/environment", "get-environment")?; + stub_env_virt(module)?; // we do arguments as well because virt assumes reactors for now... - stub_imported_func(module, "wasi:cli-base/environment", "get-arguments")?; } - let memory = module - .memories - .iter() - .nth(0) - .context("Adapter does not export a memory")? - .id(); + let memory = get_memory_id(module)?; // prepare the field data list vector for writing // strings must be sorted as binary searches are used against this data @@ -146,6 +145,10 @@ pub fn create_env_virt<'a>(module: &'a mut Module, env: &VirtEnv) -> Result<()> } } + if field_data_bytes.len() % 8 != 0 { + field_data_bytes.resize(field_data_bytes.len() + 4, 0); + } + let field_data_addr = if field_data_bytes.len() > 0 { // Offset the stack global by the static field data length let field_data_addr = bump_stack_global(module, field_data_bytes.len() as i32)?; @@ -165,7 +168,7 @@ pub fn create_env_virt<'a>(module: &'a mut Module, env: &VirtEnv) -> Result<()> // In the existing static data segment, update the static data options. // - // From virtual-adapter/src/lib.js: + // From virtual-adapter/src/env.rs: // // #[repr(C)] // pub struct Env { @@ -216,3 +219,16 @@ pub fn create_env_virt<'a>(module: &'a mut Module, env: &VirtEnv) -> Result<()> Ok(()) } + +fn stub_env_virt(module: &mut Module) -> Result<()> { + stub_imported_func(module, "wasi:cli-base/environment", "get-arguments", true)?; + stub_imported_func(module, "wasi:cli-base/environment", "get-environment", true)?; + Ok(()) +} + +pub(crate) fn strip_env_virt(module: &mut Module) -> Result<()> { + stub_env_virt(module)?; + remove_exported_func(module, "wasi:cli-base/environment#get-arguments")?; + remove_exported_func(module, "wasi:cli-base/environment#get-environment")?; + Ok(()) +} diff --git a/src/virt_fs.rs b/src/virt_fs.rs new file mode 100644 index 0000000..0bc5076 --- /dev/null +++ b/src/virt_fs.rs @@ -0,0 +1,616 @@ +use std::fmt; +use std::{collections::BTreeMap, fs}; + +use anyhow::{bail, Context, Result}; +use serde::Deserialize; +use walrus::{ir::Value, ExportItem, GlobalKind, InitExpr, Module}; + +use crate::{ + data::{Data, WasmEncode}, + walrus_ops::{ + get_active_data_segment, get_stack_global, remove_exported_func, stub_imported_func, + }, + WasiVirt, +}; + +pub type VirtualFiles = BTreeMap; + +#[derive(Deserialize, Debug, Default, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub struct VirtFs { + /// Filesystem state to virtualize + pub preopens: BTreeMap, + /// A cutoff size in bytes, above which + /// files will be treated as passive segments. + /// Per-file control may also be provided. + pub passive_cutoff: Option, +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +pub enum FsEntry { + /// symlink absolute or relative file path on the virtual filesystem + Symlink(String), + /// host path at virtualization time + Virtualize(String), + /// host path st runtime + RuntimeDir(String), + RuntimeFile(String), + /// Virtual file + File(Vec), + /// String (UTF8) file source convenience + Source(String), + /// Virtual directory + Dir(VirtDir), +} + +#[derive(Deserialize, Debug, Clone)] +#[serde(deny_unknown_fields)] +pub struct VirtFile { + pub bytes: Option>, + pub source: Option, +} + +type VirtDir = BTreeMap; + +impl WasiVirt { + fn get_or_create_fs(&mut self) -> &mut VirtFs { + self.virt_opts.fs.get_or_insert_with(Default::default) + } + + pub fn preopen(mut self, name: String, preopen: FsEntry) -> Self { + let fs = self.get_or_create_fs(); + fs.preopens.insert(name, preopen); + self + } + + pub fn passive_cutoff(mut self, passive_cutoff: usize) -> Self { + let fs = self.get_or_create_fs(); + fs.passive_cutoff = Some(passive_cutoff); + self + } +} + +#[derive(Debug)] +struct StaticIndexEntry { + name: u32, + ty: StaticIndexType, + data: StaticFileData, +} + +impl WasmEncode for StaticIndexEntry { + fn align() -> usize { + 4 + } + fn size() -> usize { + 16 + } + fn encode(&self, bytes: &mut [u8]) { + self.name.encode(&mut bytes[0..4]); + self.ty.encode(&mut bytes[4..8]); + self.data.encode(&mut bytes[8..16]); + } +} + +#[allow(dead_code)] +#[derive(Debug, Clone, Copy)] +#[repr(u32)] +enum StaticIndexType { + ActiveFile, + PassiveFile, + Dir, + RuntimeHostDir, + RuntimeHostFile, +} + +impl WasmEncode for StaticIndexType { + fn align() -> usize { + 4 + } + fn size() -> usize { + 4 + } + fn encode(&self, bytes: &mut [u8]) { + bytes[0..4].copy_from_slice(&(*self as u32).to_le_bytes()); + } +} + +union StaticFileData { + /// Active memory data pointer for ActiveFile + active: (u32, u32), + + /// Passive memory element index and len for PassiveFile + passive: (u32, u32), + + /// Host path string for HostDir / HostFile + host_path: u32, + + /// Pointer and child entry count for Dir + dir: (u32, u32), +} + +impl WasmEncode for StaticFileData { + fn align() -> usize { + 4 + } + fn size() -> usize { + 8 + } + fn encode(&self, bytes: &mut [u8]) { + bytes[0..4].copy_from_slice(&unsafe { self.dir.0.to_le_bytes() }); + bytes[4..8].copy_from_slice(&unsafe { self.dir.1.to_le_bytes() }); + } +} + +impl fmt::Debug for StaticFileData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&format!( + "STATIC [{:?}, {:?}]", + unsafe { self.dir.0 }, + unsafe { self.dir.1 } + ))?; + Ok(()) + } +} + +impl FsEntry { + fn visit_pre_mut<'a, Visitor>(&'a mut self, base_path: &str, visit: &mut Visitor) -> Result<()> + where + Visitor: FnMut(&mut FsEntry, &str, &str) -> Result<()>, + { + visit(self, base_path, "")?; + self.visit_pre_mut_inner(visit, base_path) + } + + fn visit_pre_mut_inner<'a, Visitor>( + &'a mut self, + visit: &mut Visitor, + base_path: &str, + ) -> Result<()> + where + Visitor: FnMut(&mut FsEntry, &str, &str) -> Result<()>, + { + if let FsEntry::Dir(dir) = self { + for (name, sub_entry) in dir.iter_mut() { + visit(sub_entry, name, base_path)?; + } + for (name, sub_entry) in dir.iter_mut() { + let path = format!("{base_path}{name}"); + sub_entry.visit_pre_mut_inner(visit, &path)?; + } + } + Ok(()) + } + + pub fn visit_pre<'a, Visitor>(&'a self, base_path: &str, visit: &mut Visitor) -> Result<()> + where + Visitor: FnMut(&FsEntry, &str, &str, usize) -> Result<()>, + { + visit(self, base_path, "", 0)?; + self.visit_pre_inner(visit, base_path) + } + + fn visit_pre_inner<'a, Visitor>(&'a self, visit: &mut Visitor, base_path: &str) -> Result<()> + where + Visitor: FnMut(&FsEntry, &str, &str, usize) -> Result<()>, + { + match self { + FsEntry::Dir(dir) => { + let len = dir.iter().len(); + for (idx, (name, sub_entry)) in dir.iter().enumerate() { + visit(sub_entry, name, base_path, len - idx - 1)?; + } + for (name, sub_entry) in dir { + let path = format!("{base_path}{name}"); + sub_entry.visit_pre_inner(visit, &path)?; + } + } + _ => {} + } + Ok(()) + } +} + +pub fn create_fs_virt<'a>(module: &'a mut Module, fs: &VirtFs) -> Result { + let mut virtual_files = BTreeMap::new(); + + // First we iterate the options and fill in all HostDir and HostFile entries + // With inline directory and file entries + let mut fs = fs.clone(); + for (name, entry) in fs.preopens.iter_mut() { + entry.visit_pre_mut(name, &mut |entry, name, path| { + match entry { + FsEntry::Source(source) => { + *entry = FsEntry::File(source.as_bytes().to_vec()) + }, + FsEntry::Virtualize(host_path) => { + // read a directory or file path from the host + let metadata = fs::metadata(&host_path)?; + if metadata.is_dir() { + let mut entries: BTreeMap = BTreeMap::new(); + for entry in fs::read_dir(&host_path)? { + let entry = entry?; + let file_name = entry.file_name(); + let file_name_str = file_name.to_str().unwrap(); + let mut full_path = host_path.clone(); + full_path.push('/'); + full_path.push_str(file_name_str); + virtual_files.insert(format!("{path}{name}/{file_name_str}"), full_path.to_string()); + entries.insert(file_name_str.into(), FsEntry::Virtualize(full_path)); + } + *entry = FsEntry::Dir(entries); + } else { + if !metadata.is_file() { + bail!("Only files and directories are currently supported for host paths to virtualize"); + } + let bytes = fs::read(&host_path)?; + *entry = FsEntry::File(bytes) + } + } + FsEntry::File(_) | FsEntry::RuntimeFile(_) | FsEntry::RuntimeDir(_) | FsEntry::Symlink(_) | FsEntry::Dir(_) => {} + } + Ok(()) + })?; + } + + // Create the data section bytes + let mut data_section = Data::new(get_stack_global(module)? as usize); + let mut host_passthrough = false; + + // Next we linearize the pre-order directory graph as the static file data + // Using a pre-order traversal + // Each parent node is formed along with its child length and deep subgraph + // length. + let mut static_fs_data: Vec = Vec::new(); + let mut preopen_indices: Vec = Vec::new(); + for (name, entry) in &fs.preopens { + preopen_indices.push(static_fs_data.len() as u32); + entry.visit_pre(name, &mut |entry, name, _path, remaining_siblings| { + let name_str_ptr = data_section.string(name)?; + let (ty, data) = match &entry { + // removed during previous step + FsEntry::Virtualize(_) | FsEntry::Source(_) => unreachable!(), + FsEntry::Symlink(_) => todo!("symlink support"), + FsEntry::RuntimeFile(path) => { + host_passthrough = true; + let str = data_section.string(path)?; + ( + StaticIndexType::RuntimeHostFile, + StaticFileData { host_path: str }, + ) + } + FsEntry::RuntimeDir(path) => { + host_passthrough = true; + let str = data_section.string(path)?; + ( + StaticIndexType::RuntimeHostDir, + StaticFileData { host_path: str }, + ) + } + FsEntry::Dir(dir) => { + let child_cnt = dir.len() as u32; + // children will be visited next in preorder and contiguously + // therefore the child index in the static fs data is known + // to be the next index + let start_idx = static_fs_data.len() as u32 + 1; + let child_idx = start_idx + remaining_siblings as u32; + ( + StaticIndexType::Dir, + StaticFileData { + dir: (child_idx, child_cnt), + }, + ) + } + FsEntry::File(bytes) => { + let byte_len = bytes.len(); + if byte_len > fs.passive_cutoff.unwrap_or(1024) as usize { + let passive_idx = data_section.passive_bytes(bytes); + ( + StaticIndexType::PassiveFile, + StaticFileData { + passive: (passive_idx, bytes.len() as u32), + }, + ) + } else { + let ptr = data_section.stack_bytes(bytes)?; + ( + StaticIndexType::ActiveFile, + StaticFileData { + active: (ptr, bytes.len() as u32), + }, + ) + } + } + }; + static_fs_data.push(StaticIndexEntry { + name: name_str_ptr as u32, + ty, + data, + }); + Ok(()) + })?; + } + + // now write the linearized static index entry section into the data + let static_index_addr = data_section.write_slice(static_fs_data.as_slice())?; + + let memory = module.memories.iter().nth(0).unwrap().id(); + + let fs_ptr_addr = { + let fs_ptr_export = module + .exports + .iter() + .find(|expt| expt.name.as_str() == "fs") + .context("Adapter 'fs' is not exported")?; + let ExportItem::Global(fs_ptr_global) = fs_ptr_export.item else { + bail!("Adapter 'fs' not a global"); + }; + let GlobalKind::Local(InitExpr::Value(Value::I32(fs_ptr_addr))) = + &module.globals.get(fs_ptr_global).kind + else { + bail!("Adapter 'fs' not a local I32 global value"); + }; + *fs_ptr_addr as u32 + }; + + // If host fs is disabled, remove its imports entirely + // replacing it with a stub panic + if !host_passthrough { + stub_fs_virt(module)?; + } + + let (data, data_offset) = get_active_data_segment(module, memory, fs_ptr_addr)?; + + let preopen_addr = data_section.write_slice(preopen_indices.as_slice())?; + + const FS_STATIC_LEN: usize = 16; + if data.value.len() < data_offset + FS_STATIC_LEN { + let padding = 4 - (data_offset + FS_STATIC_LEN) % 4; + data.value.resize(data_offset + FS_STATIC_LEN + padding, 0); + } + + let bytes = data.value.as_mut_slice(); + + // In the existing static data segment, update the static data options. + // + // From virtual-adapter/src/fs.rs: + // + // #[repr(C)] + // pub static mut fs: Fs = Fs { + // preopen_cnt: 0, // [byte 0] + // preopens: 0 as *const usize, // [byte 4] + // static_index_cnt: 0, // [byte 8] + // static_index: 0 as *const StaticIndexEntry, // [byte 12] + // host_passthrough: false, // [byte 16] + // }; + bytes[data_offset..data_offset + 4].copy_from_slice(&(fs.preopens.len() as u32).to_le_bytes()); + bytes[data_offset + 4..data_offset + 8].copy_from_slice(&(preopen_addr as u32).to_le_bytes()); + bytes[data_offset + 8..data_offset + 12] + .copy_from_slice(&(static_fs_data.len() as u32).to_le_bytes()); + bytes[data_offset + 12..data_offset + 16] + .copy_from_slice(&(static_index_addr as u32).to_le_bytes()); + if host_passthrough { + bytes[data_offset + 16..data_offset + 20].copy_from_slice(&(1 as u32).to_le_bytes()); + } + + data_section.finish(module)?; + + // return the processed virtualized filesystem + Ok(virtual_files) +} + +fn stub_fs_virt(module: &mut Module) -> Result<()> { + stub_imported_func(module, "wasi:cli-base/preopens", "get-directories", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "read_via_stream", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "write_via_stream", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "append_via_stream", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "advise", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "sync-data", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "get-flags", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "get-type", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "set-size", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "set-times", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "read", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "write", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "read-directory", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "sync", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "create-directory-at", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "stat", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "stat-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "set-times-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "link-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "open-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "readlink-at", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "remove-directory-at", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "rename-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "symlink-at", false)?; + stub_imported_func(module, "wasi:filesystem/filesystem", "access-at", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "unlink-file-at", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "change-file-permissions-at", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "change-directory-permissions-at", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "lock-shared", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "lock-exclusive", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "try-lock-shared", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "try-lock-exclusive", + false, + )?; + stub_imported_func(module, "wasi:filesystem/filesystem", "unlock", false)?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "drop-descriptor", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "read-directory-entry", + false, + )?; + stub_imported_func( + module, + "wasi:filesystem/filesystem", + "drop-directory-entry-stream", + false, + )?; + + stub_imported_func( + module, + "wasi:io/streams", + "drop-directory-entry-stream", + false, + )?; + stub_imported_func(module, "wasi:io/streams", "read", false)?; + stub_imported_func(module, "wasi:io/streams", "blocking-read", false)?; + stub_imported_func(module, "wasi:io/streams", "skip", false)?; + stub_imported_func(module, "wasi:io/streams", "blocking-skip", false)?; + stub_imported_func( + module, + "wasi:io/streams", + "subscribe-to-input-stream", + false, + )?; + stub_imported_func(module, "wasi:io/streams", "drop-input-stream", false)?; + stub_imported_func(module, "wasi:io/streams", "write", false)?; + stub_imported_func(module, "wasi:io/streams", "blocking-write", false)?; + stub_imported_func(module, "wasi:io/streams", "write-zeroes", false)?; + stub_imported_func(module, "wasi:io/streams", "blocking-write-zeroes", false)?; + stub_imported_func(module, "wasi:io/streams", "splice", false)?; + stub_imported_func(module, "wasi:io/streams", "blocking-splice", false)?; + stub_imported_func(module, "wasi:io/streams", "forward", false)?; + stub_imported_func( + module, + "wasi:io/streams", + "subscribe-to-output-stream", + false, + )?; + stub_imported_func(module, "wasi:io/streams", "drop-output-stream", false)?; + Ok(()) +} + +pub(crate) fn strip_fs_virt(module: &mut Module) -> Result<()> { + stub_fs_virt(module)?; + + remove_exported_func(module, "wasi:cli-base/preopens#get-directories")?; + + remove_exported_func(module, "wasi:filesystem/filesystem#read-via-stream")?; + remove_exported_func(module, "wasi:filesystem/filesystem#write-via-stream")?; + remove_exported_func(module, "wasi:filesystem/filesystem#append-via-stream")?; + remove_exported_func(module, "wasi:filesystem/filesystem#advise")?; + remove_exported_func(module, "wasi:filesystem/filesystem#sync-data")?; + remove_exported_func(module, "wasi:filesystem/filesystem#get-flags")?; + remove_exported_func(module, "wasi:filesystem/filesystem#get-type")?; + remove_exported_func(module, "wasi:filesystem/filesystem#set-size")?; + remove_exported_func(module, "wasi:filesystem/filesystem#set-times")?; + remove_exported_func(module, "wasi:filesystem/filesystem#read")?; + remove_exported_func(module, "wasi:filesystem/filesystem#write")?; + remove_exported_func(module, "wasi:filesystem/filesystem#read-directory")?; + remove_exported_func(module, "wasi:filesystem/filesystem#sync")?; + remove_exported_func(module, "wasi:filesystem/filesystem#create-directory-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#stat")?; + remove_exported_func(module, "wasi:filesystem/filesystem#stat-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#set-times-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#link-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#open-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#readlink-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#remove-directory-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#rename-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#symlink-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#access-at")?; + remove_exported_func(module, "wasi:filesystem/filesystem#unlink-file-at")?; + remove_exported_func( + module, + "wasi:filesystem/filesystem#change-file-permissions-at", + )?; + remove_exported_func( + module, + "wasi:filesystem/filesystem#change-directory-permissions-at", + )?; + remove_exported_func(module, "wasi:filesystem/filesystem#lock-shared")?; + remove_exported_func(module, "wasi:filesystem/filesystem#lock-exclusive")?; + remove_exported_func(module, "wasi:filesystem/filesystem#try-lock-shared")?; + remove_exported_func(module, "wasi:filesystem/filesystem#try-lock-exclusive")?; + remove_exported_func(module, "wasi:filesystem/filesystem#unlock")?; + remove_exported_func(module, "wasi:filesystem/filesystem#drop-descriptor")?; + remove_exported_func(module, "wasi:filesystem/filesystem#read-directory-entry")?; + remove_exported_func( + module, + "wasi:filesystem/filesystem#drop-directory-entry-stream", + )?; + + remove_exported_func(module, "wasi:io/streams#read")?; + remove_exported_func(module, "wasi:io/streams#blocking-read")?; + remove_exported_func(module, "wasi:io/streams#skip")?; + remove_exported_func(module, "wasi:io/streams#blocking-skip")?; + remove_exported_func(module, "wasi:io/streams#subscribe-to-input-stream")?; + remove_exported_func(module, "wasi:io/streams#drop-input-stream")?; + remove_exported_func(module, "wasi:io/streams#write")?; + remove_exported_func(module, "wasi:io/streams#blocking-write")?; + remove_exported_func(module, "wasi:io/streams#write-zeroes")?; + remove_exported_func(module, "wasi:io/streams#blocking-write-zeroes")?; + remove_exported_func(module, "wasi:io/streams#splice")?; + remove_exported_func(module, "wasi:io/streams#blocking-splice")?; + remove_exported_func(module, "wasi:io/streams#forward")?; + remove_exported_func(module, "wasi:io/streams#subscribe-to-output-stream")?; + remove_exported_func(module, "wasi:io/streams#drop-output-stream")?; + + Ok(()) +} diff --git a/src/walrus_ops.rs b/src/walrus_ops.rs index 0e8f159..d232d65 100644 --- a/src/walrus_ops.rs +++ b/src/walrus_ops.rs @@ -1,30 +1,42 @@ use anyhow::{bail, Context, Result}; use walrus::{ - ir::Value, ActiveData, ActiveDataLocation, Data, DataKind, Function, FunctionBuilder, - FunctionKind, GlobalKind, ImportKind, ImportedFunction, InitExpr, MemoryId, Module, + ir::Value, ActiveData, ActiveDataLocation, Data, DataKind, ExportItem, Function, + FunctionBuilder, FunctionId, FunctionKind, GlobalKind, ImportKind, ImportedFunction, InitExpr, + MemoryId, Module, }; +pub(crate) fn get_active_data_start(data: &Data, mem: MemoryId) -> Result { + let DataKind::Active(active_data) = &data.kind else { + bail!("Adapter data section is not active"); + }; + if active_data.memory != mem { + bail!("Adapter data memory is not the expected memory id"); + } + let ActiveDataLocation::Absolute(loc) = &active_data.location else { + bail!("Adapter data memory is not absolutely offset"); + }; + Ok(*loc) +} + pub(crate) fn get_active_data_segment( module: &mut Module, mem: MemoryId, addr: u32, ) -> Result<(&mut Data, usize)> { - let data = module - .data - .iter() - .find(|&data| { - let DataKind::Active(active_data) = &data.kind else { - return false; - }; - if active_data.memory != mem { - return false; + let mut found_data: Option<&Data> = None; + for data in module.data.iter() { + let data_addr = get_active_data_start(data, mem)?; + if data_addr <= addr { + let best_match = match found_data { + Some(found_data) => data_addr > get_active_data_start(found_data, mem)?, + None => true, }; - let ActiveDataLocation::Absolute(loc) = &active_data.location else { - return false; - }; - *loc <= addr && *loc + data.value.len() as u32 > addr - }) - .context("Unable to find data section for env ptr")?; + if best_match { + found_data = Some(data); + } + } + } + let data = found_data.context("Unable to find data section for ptr")?; let DataKind::Active(ActiveData { location: ActiveDataLocation::Absolute(loc), .. @@ -37,7 +49,33 @@ pub(crate) fn get_active_data_segment( Ok((module.data.get_mut(data_id), offset)) } +pub(crate) fn get_memory_id(module: &Module) -> Result { + let mut mem_iter = module.memories.iter(); + let memory = mem_iter.next().context("Module does not export a memory")?; + if mem_iter.next().is_some() { + bail!("Multiple memories unsupported") + } + Ok(memory.id()) +} + +pub(crate) fn get_stack_global(module: &Module) -> Result { + let stack_global_id = module + .globals + .iter() + .find(|&global| global.name.as_deref() == Some("__stack_pointer")) + .context("Unable to find __stack_pointer global name")? + .id(); + let stack_global = module.globals.get(stack_global_id); + let GlobalKind::Local(InitExpr::Value(Value::I32(stack_value))) = &stack_global.kind else { + bail!("Stack global is not a constant I32"); + }; + Ok(*stack_value as u32) +} + pub(crate) fn bump_stack_global(module: &mut Module, offset: i32) -> Result { + if offset % 8 != 0 { + bail!("Stack global must be bumped by 8 byte alignment, offset of {offset} provided"); + } let stack_global_id = module .globals .iter() @@ -59,16 +97,38 @@ pub(crate) fn bump_stack_global(module: &mut Module, offset: i32) -> Result Ok(new_stack_value as u32) } +pub(crate) fn get_exported_func(module: &mut Module, name: &str) -> Result { + let exported_fn = module + .exports + .iter() + .find(|expt| expt.name == name) + .with_context(|| format!("Unable to find export '{name}'"))?; + let ExportItem::Function(fid) = exported_fn.item else { + bail!("{name} not a function"); + }; + Ok(fid) +} + pub(crate) fn stub_imported_func( module: &mut Module, import_module: &str, import_name: &str, + throw_if_not_found: bool, ) -> Result<()> { - let imported_fn = module + let imported_fn = match module .imports .iter() .find(|impt| impt.module == import_module && impt.name == import_name) - .unwrap(); + { + Some(found) => found, + None => { + if throw_if_not_found { + bail!("Unable to find import {import_module}#{import_name} to stub"); + } else { + return Ok(()); + } + } + }; let ImportKind::Function(fid) = imported_fn.kind else { bail!("Unable to stub import {import_module}#{import_name}, as it is not an imported function"); @@ -97,3 +157,15 @@ pub(crate) fn stub_imported_func( Ok(()) } + +pub(crate) fn remove_exported_func(module: &mut Module, export_name: &str) -> Result<()> { + let exported_fn = module + .exports + .iter() + .find(|expt| expt.name == export_name) + .with_context(|| format!("Unable to find export {export_name}"))?; + + module.exports.delete(exported_fn.id()); + + Ok(()) +} diff --git a/tests/cases/env-allow.toml b/tests/cases/env-allow.toml index 0e0bab1..68e43fa 100644 --- a/tests/cases/env-allow.toml +++ b/tests/cases/env-allow.toml @@ -1,13 +1,13 @@ component = "get-env" -[host_env] +[host-env] PRIVATE_TOKEN = "PRIVATE" PUBLIC_VAR = "VAL" -[virt_opts.env] +[virt-opts.env] overrides = [["CUSTOM", "VAL"]] -[virt_opts.env.host] +[virt-opts.env.host] allow = ["PUBLIC_VAR"] [expect] diff --git a/tests/cases/env-deny.toml b/tests/cases/env-deny.toml index 1890153..fecfe6b 100644 --- a/tests/cases/env-deny.toml +++ b/tests/cases/env-deny.toml @@ -1,13 +1,13 @@ component = "get-env" -[host_env] +[host-env] PRIVATE_TOKEN = "PRIVATE" PUBLIC_VAR = "VAL" -[virt_opts.env] +[virt-opts.env] overrides = [["CUSTOM", "VAL"]] -[virt_opts.env.host] +[virt-opts.env.host] deny = ["PRIVATE_TOKEN"] [expect] diff --git a/tests/cases/env-none-overrides.toml b/tests/cases/env-none-overrides.toml index 28e8a66..da9496f 100644 --- a/tests/cases/env-none-overrides.toml +++ b/tests/cases/env-none-overrides.toml @@ -1,9 +1,9 @@ component = "get-env" -[host_env] +[host-env] CUSTOM = "TEST" -[virt_opts.env] +[virt-opts.env] host = "none" overrides = [["ENV_OVERRIDE", "Value"]] diff --git a/tests/cases/env-none.toml b/tests/cases/env-none.toml index c303343..cf45e10 100644 --- a/tests/cases/env-none.toml +++ b/tests/cases/env-none.toml @@ -1,9 +1,9 @@ component = "get-env" -[host_env] +[host-env] CUSTOM = "TEST" -[virt_opts.env] +[virt-opts.env] host = "none" overrides = [] diff --git a/tests/cases/env-passthrough.toml b/tests/cases/env-passthrough.toml index 6839e64..372def1 100644 --- a/tests/cases/env-passthrough.toml +++ b/tests/cases/env-passthrough.toml @@ -1,9 +1,9 @@ component = "get-env" -[host_env] +[host-env] CUSTOM = "ENV" -[virt_opts.env] +[virt-opts.env] host = "all" overrides = [] diff --git a/tests/cases/fs-dir-read.toml b/tests/cases/fs-dir-read.toml new file mode 100644 index 0000000..3516532 --- /dev/null +++ b/tests/cases/fs-dir-read.toml @@ -0,0 +1,11 @@ +component = "file-read" + +host-fs-path = "/mydir" + +[virt-opts.fs.preopens."/".dir."mydir".dir] +"file1.txt" = { source = "inner contents1" } +"file2.txt" = { source = "inner contents2" } +"echo.txt" = { source = "inner contents echo" } + +[expect] +file-read = "echo.txtfile1.txtfile2.txt" diff --git a/tests/cases/fs-file-read.toml b/tests/cases/fs-file-read.toml new file mode 100644 index 0000000..e13bc13 --- /dev/null +++ b/tests/cases/fs-file-read.toml @@ -0,0 +1,9 @@ +component = "file-read" + +host-fs-path = "/file.txt" + +[virt-opts.fs.preopens."/".dir] +"file.txt" = { source = "contents" } + +[expect] +file-read = "contents" diff --git a/tests/cases/fs-host-read.toml b/tests/cases/fs-host-read.toml new file mode 100644 index 0000000..f92104a --- /dev/null +++ b/tests/cases/fs-host-read.toml @@ -0,0 +1,230 @@ +component = "file-read" + +host-fs-path = "/file.txt" + +[virt-opts.fs.preopens."/".dir] +"file.txt" = { runtime-file = "/LICENSE" } + +[expect] +file-read = """ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +""" \ No newline at end of file diff --git a/tests/cases/fs-inner-read.toml b/tests/cases/fs-inner-read.toml new file mode 100644 index 0000000..68b3f63 --- /dev/null +++ b/tests/cases/fs-inner-read.toml @@ -0,0 +1,9 @@ +component = "file-read" + +host-fs-path = "/mydir/file.txt" + +[virt-opts.fs.preopens."/".dir."mydir".dir] +"file.txt" = { source = "inner contents" } + +[expect] +file-read = "inner contents" diff --git a/tests/cases/fs-nested-dir-read.toml b/tests/cases/fs-nested-dir-read.toml new file mode 100644 index 0000000..1e0fe65 --- /dev/null +++ b/tests/cases/fs-nested-dir-read.toml @@ -0,0 +1,43 @@ +component = "file-read" + +host-fs-path = "/clocks/monotonic-clock.wit" + +[virt-opts.fs.preopens."/"] +virtualize = "./wit/deps" + +[expect] +file-read = '''package wasi:clocks + +/// WASI Monotonic Clock is a clock API intended to let users measure elapsed +/// time. +/// +/// It is intended to be portable at least between Unix-family platforms and +/// Windows. +/// +/// A monotonic clock is a clock which has an unspecified initial value, and +/// successive reads of the clock will produce non-decreasing values. +/// +/// It is intended for measuring elapsed time. +interface monotonic-clock { + use wasi:poll/poll.{pollable} + + /// A timestamp in nanoseconds. + type instant = u64 + + /// Read the current value of the clock. + /// + /// The clock is monotonic, therefore calling this function repeatedly will + /// produce a sequence of non-decreasing values. + now: func() -> instant + + /// Query the resolution of the clock. + resolution: func() -> instant + + /// Create a `pollable` which will resolve once the specified time has been + /// reached. + subscribe: func( + when: instant, + absolute: bool + ) -> pollable +} +''' diff --git a/tests/cases/fs-passive-file-read.toml b/tests/cases/fs-passive-file-read.toml new file mode 100644 index 0000000..a20b811 --- /dev/null +++ b/tests/cases/fs-passive-file-read.toml @@ -0,0 +1,23 @@ +component = "file-read" + +host-fs-path = "/env-none.toml" + +[virt-opts.fs] +passive-cutoff = 10 + +[virt-opts.fs.preopens."/"] +virtualize = "./tests/cases" + +[expect] +file-read = '''component = "get-env" + +[host-env] +CUSTOM = "TEST" + +[virt-opts.env] +host = "none" +overrides = [] + +[expect] +env = [] +''' diff --git a/tests/cases/fs-virt-dir-read.toml b/tests/cases/fs-virt-dir-read.toml new file mode 100644 index 0000000..5805d38 --- /dev/null +++ b/tests/cases/fs-virt-dir-read.toml @@ -0,0 +1,20 @@ +component = "file-read" + +host-fs-path = "/env-none.toml" + +[virt-opts.fs.preopens."/"] +virtualize = "./tests/cases" + +[expect] +file-read = '''component = "get-env" + +[host-env] +CUSTOM = "TEST" + +[virt-opts.env] +host = "none" +overrides = [] + +[expect] +env = [] +''' diff --git a/tests/components/file-read/Cargo.toml b/tests/components/file-read/Cargo.toml new file mode 100644 index 0000000..c741ca9 --- /dev/null +++ b/tests/components/file-read/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "file-read" +version = "0.1.0" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +anyhow = { workspace = true } +wit-bindgen = { workspace = true } diff --git a/tests/components/file-read/src/lib.rs b/tests/components/file-read/src/lib.rs new file mode 100644 index 0000000..0c1043b --- /dev/null +++ b/tests/components/file-read/src/lib.rs @@ -0,0 +1,55 @@ +use std::fs; + +wit_bindgen::generate!({ + path: "../../../wit", + world: "virt-test" +}); + +struct VirtTestImpl; + +export_virt_test!(VirtTestImpl); + +impl VirtTest for VirtTestImpl { + fn test_get_env() -> Vec<(String, String)> { + Vec::new() + } + fn test_file_read(path: String) -> String { + let meta = match fs::metadata(&path) { + Ok(meta) => meta, + Err(err) => { + return format!("ERR: reading metadata {path}\n{:?}", err); + } + }; + if meta.is_file() { + match fs::read_to_string(&path) { + Ok(source) => source, + Err(err) => format!("ERR: {:?}", err), + } + } else if meta.is_dir() { + let dir = match fs::read_dir(&path) { + Ok(dir) => dir, + Err(err) => { + return format!("ERR: reading dir {path}\n{:?}", err); + } + }; + let mut files = String::new(); + for file in dir { + let file = match file { + Ok(file) => file, + Err(err) => { + return format!("ERR: reading dir entry\n{:?}", err); + } + }; + files.push_str(match file.file_name().to_str() { + Some(name) => name, + None => { + return format!("ERR: invalid filename string '{:?}'", file.file_name()); + } + }); + } + files + } else { + "ERR: Not a file or dir".into() + } + } +} diff --git a/tests/components/get-env/src/lib.rs b/tests/components/get-env/src/lib.rs index 1f0d614..0591d72 100644 --- a/tests/components/get-env/src/lib.rs +++ b/tests/components/get-env/src/lib.rs @@ -1,8 +1,8 @@ use std::env; wit_bindgen::generate!({ - path: "../../../wit", - world: "virt-test" + path: "../../../wit", + world: "virt-test" }); struct VirtTestImpl; @@ -13,4 +13,7 @@ impl VirtTest for VirtTestImpl { fn test_get_env() -> Vec<(String, String)> { env::vars().collect() } + fn test_file_read(_path: String) -> String { + unimplemented!(); + } } diff --git a/tests/virt.rs b/tests/virt.rs index d67cc7e..2a1e13e 100644 --- a/tests/virt.rs +++ b/tests/virt.rs @@ -1,4 +1,5 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context, Result}; +use cap_std::ambient_authority; use heck::ToSnakeCase; use serde::Deserialize; use std::collections::BTreeMap; @@ -10,7 +11,10 @@ use wasmtime::{ component::{Component, Linker}, Config, Engine, Store, WasmBacktraceDetails, }; -use wasmtime_wasi::preview2::{wasi as wasi_preview2, Table, WasiCtx, WasiCtxBuilder, WasiView}; +use wasmtime_wasi::preview2::{ + wasi as wasi_preview2, DirPerms, FilePerms, Table, WasiCtx, WasiCtxBuilder, WasiView, +}; +use wasmtime_wasi::Dir; use wit_component::ComponentEncoder; wasmtime::component::bindgen!({ @@ -41,14 +45,18 @@ fn cmd(arg: &str) -> Result<()> { } #[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] struct TestExpectation { env: Option>, + file_read: Option, } #[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] struct TestCase { component: String, host_env: Option>, + host_fs_path: Option, virt_opts: Option, expect: TestExpectation, } @@ -63,8 +71,14 @@ async fn virt_test() -> Result<()> { let test_case_file_name = test_case.file_name().to_string_lossy().to_string(); let test_case_name = test_case_file_name.strip_suffix(".toml").unwrap(); + // Filtering... + // if test_case_name != "fs-host-read" { + // continue; + // } + // load the test case JSON data - let test: TestCase = toml::from_str(&fs::read_to_string(&test_case_path)?)?; + let test: TestCase = toml::from_str(&fs::read_to_string(&test_case_path)?) + .context(format!("Error reading test case {:?}", test_case_path))?; let component_name = &test.component; @@ -93,8 +107,10 @@ async fn virt_test() -> Result<()> { let mut virt_component_path = generated_path.join(test_case_name); virt_component_path.set_extension("virt.wasm"); let virt_opts = test.virt_opts.clone().unwrap_or_default(); - let virt_component = create_virt(&virt_opts)?; - fs::write(&virt_component_path, virt_component)?; + let virt_component = create_virt(&virt_opts) + .with_context(|| format!("Error creating virtual adapter for {:?}", test_case_path))?; + + fs::write(&virt_component_path, virt_component.adapter)?; // compose the test component with the defined test virtualization let component_bytes = ComponentComposer::new( @@ -113,7 +129,12 @@ async fn virt_test() -> Result<()> { } // execute the composed virtualized component test function - let mut builder = WasiCtxBuilder::new().inherit_stdio(); + let mut builder = WasiCtxBuilder::new().inherit_stdio().push_preopened_dir( + Dir::open_ambient_dir(".", ambient_authority())?, + DirPerms::READ, + FilePerms::READ, + "/", + ); if let Some(host_env) = &test.host_env { let env: Vec<(String, String)> = host_env .iter() @@ -157,7 +178,7 @@ async fn virt_test() -> Result<()> { // simple logger for debugging let mut log_builder = linker.instance("console")?; log_builder.func_wrap("log", |_store, params: (String,)| { - println!("LOG: {}", params.0); + eprintln!("LOG: {}", params.0); Ok(()) })?; @@ -168,9 +189,9 @@ async fn virt_test() -> Result<()> { VirtTest::instantiate_async(&mut store, &component, &linker).await?; // env var expectation check - if let Some(env) = &test.expect.env { + if let Some(expect_env) = &test.expect.env { let env_vars = instance.call_test_get_env(&mut store).await?; - if !env_vars.eq(env) { + if !env_vars.eq(expect_env) { return Err(anyhow!( "Unexpected env vars testing {:?}: @@ -179,12 +200,34 @@ async fn virt_test() -> Result<()> { {:?}", test_case_path, - env, + expect_env, env_vars, test )); } } + + // fs read expectation check + if let Some(expect_file_read) = &test.expect.file_read { + let file_read = instance + .call_test_file_read(&mut store, test.host_fs_path.as_ref().unwrap()) + .await?; + if !file_read.eq(expect_file_read) { + return Err(anyhow!( + "Unexpected file read result testing {:?}: + + \x1b[1mExpected:\x1b[0m {:?} + \x1b[1mActual:\x1b[0m {:?} + + {:?}", + test_case_path, + expect_file_read, + file_read, + test + )); + } + } + println!("\x1b[1;32m√\x1b[0m {:?}", test_case_path); } Ok(()) diff --git a/virtual-adapter/src/env.rs b/virtual-adapter/src/env.rs new file mode 100644 index 0000000..677ab54 --- /dev/null +++ b/virtual-adapter/src/env.rs @@ -0,0 +1,88 @@ +use crate::exports::wasi::cli_base::environment::Environment; +use crate::wasi::cli_base::environment; +use crate::VirtAdapter; + +#[repr(C)] +pub struct Env { + /// Whether to fallback to the host env + /// [byte 0] + host_fallback: bool, + /// Whether we are providing an allow list or a deny list + /// on the fallback lookups + /// [byte 1] + host_fallback_allow: bool, + /// How many host fields are defined in the data pointer + /// [byte 4] + host_field_cnt: u32, + /// Host many host fields are defined to be allow or deny + /// (these are concatenated at the end of the data with empty values) + /// [byte 8] + host_allow_or_deny_cnt: u32, + /// Byte data of u32 byte len followed by string bytes + /// up to the lengths previously provided. + /// [byte 12] + host_field_data: *const u8, +} + +#[no_mangle] +pub static mut env: Env = Env { + host_fallback: true, + host_fallback_allow: false, + host_field_cnt: 0, + host_allow_or_deny_cnt: 0, + host_field_data: 0 as *const u8, +}; + +fn read_data_str(offset: &mut isize) -> &'static str { + let data: *const u8 = unsafe { env.host_field_data.offset(*offset) }; + let byte_len = unsafe { (data as *const u32).read() } as usize; + *offset += 4; + let data: *const u8 = unsafe { env.host_field_data.offset(*offset) }; + let str_data = unsafe { std::slice::from_raw_parts(data, byte_len) }; + *offset += byte_len as isize; + let rem = *offset % 4; + if rem > 0 { + *offset += 4 - rem; + } + unsafe { core::str::from_utf8_unchecked(str_data) } +} + +impl Environment for VirtAdapter { + fn get_environment() -> Vec<(String, String)> { + let mut environment = Vec::new(); + let mut data_offset: isize = 0; + for _ in 0..unsafe { env.host_field_cnt } { + let env_key = read_data_str(&mut data_offset); + let env_val = read_data_str(&mut data_offset); + environment.push((env_key.to_string(), env_val.to_string())); + } + let override_len = environment.len(); + // fallback ASSUMES that all data is alphabetically ordered + if unsafe { env.host_fallback } { + let mut allow_or_deny = Vec::new(); + for _ in 0..unsafe { env.host_allow_or_deny_cnt } { + let allow_or_deny_key = read_data_str(&mut data_offset); + allow_or_deny.push(allow_or_deny_key); + } + + let is_allow_list = unsafe { env.host_fallback_allow }; + for (key, value) in environment::get_environment() { + if environment[0..override_len] + .binary_search_by_key(&&key, |(s, _)| s) + .is_ok() + { + continue; + } + let in_list = allow_or_deny.binary_search(&key.as_ref()).is_ok(); + if is_allow_list && in_list || !is_allow_list && !in_list { + environment.push((key, value)); + } + } + } + environment + } + + fn get_arguments() -> Vec { + environment::get_arguments() + } +} diff --git a/virtual-adapter/src/fs.rs b/virtual-adapter/src/fs.rs new file mode 100644 index 0000000..44a33b1 --- /dev/null +++ b/virtual-adapter/src/fs.rs @@ -0,0 +1,752 @@ +use crate::exports::wasi::cli_base::preopens::Preopens; +use crate::exports::wasi::filesystem::filesystem::{ + AccessType, Advice, Datetime, DescriptorFlags, DescriptorStat, DescriptorType, DirectoryEntry, + ErrorCode, Filesystem, Modes, NewTimestamp, OpenFlags, PathFlags, +}; +use crate::exports::wasi::io::streams::{StreamError, Streams}; +use crate::wasi::cli_base::preopens; +use crate::wasi::filesystem::filesystem; +// use crate::wasi::io::streams; + +// for debugging +// use crate::console; +// use std::fmt; + +use crate::VirtAdapter; +use std::alloc::Layout; +use std::cmp; +use std::collections::BTreeMap; +use std::ffi::CStr; +use std::slice; + +// static fs config +#[repr(C)] +pub struct Fs { + preopen_cnt: usize, + preopens: *const usize, + static_index_cnt: usize, + static_index: *const StaticIndexEntry, + host_passthrough: bool, +} + +impl Fs { + fn preopens() -> Vec<&'static StaticIndexEntry> { + let preopen_offsets = unsafe { slice::from_raw_parts(fs.preopens, fs.preopen_cnt) }; + let static_index = Fs::static_index(); + preopen_offsets + .iter() + .map(|&idx| &static_index[idx]) + .collect() + } + fn static_index() -> &'static [StaticIndexEntry] { + unsafe { slice::from_raw_parts(fs.static_index, fs.static_index_cnt) } + } +} + +// #[derive(Debug)] +struct Descriptor { + // the static entry referenced by this descriptor + entry: *const StaticIndexEntry, + // the descriptor index of this descriptor + fd: u32, + // if a host entry, the underlying host descriptor + // (if any) + host_fd: Option, +} + +impl Descriptor { + fn entry(&self) -> &StaticIndexEntry { + unsafe { self.entry.as_ref() }.unwrap() + } + + fn drop(&self) { + unsafe { + STATE.descriptor_table.remove(&self.fd); + } + if let Some(host_fd) = self.host_fd { + filesystem::drop_descriptor(host_fd); + } + } + + fn get_bytes<'a>(&mut self, offset: u64, len: u64) -> Result<(Vec, bool), ErrorCode> { + let entry = self.entry(); + match entry.ty { + StaticIndexType::ActiveFile => { + if offset as usize == unsafe { entry.data.active.1 } { + return Ok((vec![], true)); + } + if offset as usize > unsafe { entry.data.active.1 } { + return Err(ErrorCode::InvalidSeek); + } + let read_ptr = unsafe { entry.data.active.0.add(offset as usize) }; + let read_len = cmp::min( + unsafe { entry.data.active.1 } - offset as usize, + len as usize, + ); + let bytes = unsafe { slice::from_raw_parts(read_ptr, read_len) }; + Ok((bytes.to_vec(), read_len < len as usize)) + } + StaticIndexType::PassiveFile => { + if offset as usize == unsafe { entry.data.passive.1 } { + return Ok((vec![], true)); + } + if offset as usize > unsafe { entry.data.passive.1 } { + return Err(ErrorCode::InvalidSeek); + } + let read_len = cmp::min( + unsafe { entry.data.passive.1 } - offset as usize, + len as usize, + ); + let data = passive_alloc( + unsafe { entry.data.passive.0 }, + offset as u32, + read_len as u32, + ); + let bytes = unsafe { slice::from_raw_parts(data, read_len) }; + let vec = bytes.to_vec(); + unsafe { std::alloc::dealloc(data, Layout::from_size_align(1, 4).unwrap()) }; + Ok((vec, read_len < len as usize)) + } + StaticIndexType::Dir => todo!(), + StaticIndexType::RuntimeDir => todo!(), + StaticIndexType::RuntimeFile => { + if let Some(host_fd) = self.host_fd { + return filesystem::read(host_fd, len, offset).map_err(err_map); + } + + let path = unsafe { CStr::from_ptr(entry.data.runtime_path) }; + let path = path.to_str().unwrap(); + + let Some((preopen_fd, subpath)) = FsState::get_host_preopen(path) else { + return Err(ErrorCode::NoEntry); + }; + let host_fd = filesystem::open_at( + preopen_fd, + filesystem::PathFlags::empty(), + subpath, + filesystem::OpenFlags::empty(), + filesystem::DescriptorFlags::READ, + filesystem::Modes::READABLE, + ) + .map_err(err_map)?; + + self.host_fd = Some(host_fd); + filesystem::read(host_fd, len, offset).map_err(err_map) + } + } + } +} + +fn err_map(e: filesystem::ErrorCode) -> ErrorCode { + match e { + filesystem::ErrorCode::Access => ErrorCode::Access, + filesystem::ErrorCode::WouldBlock => ErrorCode::WouldBlock, + filesystem::ErrorCode::Already => ErrorCode::Already, + filesystem::ErrorCode::BadDescriptor => ErrorCode::BadDescriptor, + filesystem::ErrorCode::Busy => ErrorCode::Busy, + filesystem::ErrorCode::Deadlock => ErrorCode::Deadlock, + filesystem::ErrorCode::Quota => ErrorCode::Quota, + filesystem::ErrorCode::Exist => ErrorCode::Exist, + filesystem::ErrorCode::FileTooLarge => ErrorCode::FileTooLarge, + filesystem::ErrorCode::IllegalByteSequence => ErrorCode::IllegalByteSequence, + filesystem::ErrorCode::InProgress => ErrorCode::InProgress, + filesystem::ErrorCode::Interrupted => ErrorCode::Interrupted, + filesystem::ErrorCode::Invalid => ErrorCode::Invalid, + filesystem::ErrorCode::Io => ErrorCode::Io, + filesystem::ErrorCode::IsDirectory => ErrorCode::IsDirectory, + filesystem::ErrorCode::Loop => ErrorCode::Loop, + filesystem::ErrorCode::TooManyLinks => ErrorCode::TooManyLinks, + filesystem::ErrorCode::MessageSize => ErrorCode::MessageSize, + filesystem::ErrorCode::NameTooLong => ErrorCode::NameTooLong, + filesystem::ErrorCode::NoDevice => ErrorCode::NoDevice, + filesystem::ErrorCode::NoEntry => ErrorCode::NoEntry, + filesystem::ErrorCode::NoLock => ErrorCode::NoLock, + filesystem::ErrorCode::InsufficientMemory => ErrorCode::InsufficientMemory, + filesystem::ErrorCode::InsufficientSpace => ErrorCode::InsufficientSpace, + filesystem::ErrorCode::NotDirectory => ErrorCode::NotDirectory, + filesystem::ErrorCode::NotEmpty => ErrorCode::NotEmpty, + filesystem::ErrorCode::NotRecoverable => ErrorCode::NotRecoverable, + filesystem::ErrorCode::Unsupported => ErrorCode::Unsupported, + filesystem::ErrorCode::NoTty => ErrorCode::NoTty, + filesystem::ErrorCode::NoSuchDevice => ErrorCode::NoSuchDevice, + filesystem::ErrorCode::Overflow => ErrorCode::Overflow, + filesystem::ErrorCode::NotPermitted => ErrorCode::NotPermitted, + filesystem::ErrorCode::Pipe => ErrorCode::Pipe, + filesystem::ErrorCode::ReadOnly => ErrorCode::ReadOnly, + filesystem::ErrorCode::InvalidSeek => ErrorCode::InvalidSeek, + filesystem::ErrorCode::TextFileBusy => ErrorCode::TextFileBusy, + filesystem::ErrorCode::CrossDevice => ErrorCode::CrossDevice, + } +} + +impl StaticIndexEntry { + // fn idx(&self) -> usize { + // let static_index_start = unsafe { fs.static_index }; + // let cur_index_start = self as *const StaticIndexEntry; + // unsafe { cur_index_start.sub_ptr(static_index_start) } + // } + fn name(&self) -> &'static str { + let c_str = unsafe { CStr::from_ptr((*self).name) }; + c_str.to_str().unwrap() + } + fn ty(&self) -> DescriptorType { + match self.ty { + StaticIndexType::ActiveFile + | StaticIndexType::PassiveFile + | StaticIndexType::RuntimeFile => DescriptorType::RegularFile, + StaticIndexType::Dir | StaticIndexType::RuntimeDir => DescriptorType::Directory, + } + } + fn size(&self) -> Result { + match self.ty { + StaticIndexType::ActiveFile => Ok(unsafe { self.data.active.1 } as u64), + StaticIndexType::PassiveFile => Ok(unsafe { self.data.passive.1 } as u64), + StaticIndexType::Dir | StaticIndexType::RuntimeDir => Ok(0), + StaticIndexType::RuntimeFile => { + let path = unsafe { CStr::from_ptr(self.data.runtime_path) }; + let path = path.to_str().unwrap(); + let Some((fd, subpath)) = FsState::get_host_preopen(path) else { + return Err(ErrorCode::NoEntry); + }; + let stat = filesystem::stat_at(fd, filesystem::PathFlags::empty(), subpath) + .map_err(err_map)?; + Ok(stat.size) + } + } + } + fn child_list(&self) -> Result<&'static [StaticIndexEntry], ErrorCode> { + if !matches!(self.ty(), DescriptorType::Directory) { + return Err(ErrorCode::NotDirectory); + } + let (child_list_idx, child_list_len) = unsafe { (*self).data.dir }; + let static_index = Fs::static_index(); + Ok(&static_index[child_list_idx..child_list_idx + child_list_len]) + } + fn dir_lookup(&self, path: &str) -> Result<&StaticIndexEntry, ErrorCode> { + assert!(path.len() > 0); + let (first_part, rem) = match path.find('/') { + Some(idx) => (&path[0..idx], &path[idx + 1..]), + None => (path, ""), + }; + let child_list = self.child_list()?; + if let Ok(child_idx) = child_list.binary_search_by(|entry| entry.name().cmp(first_part)) { + let child = &child_list[child_idx]; + if rem.len() > 0 { + child.dir_lookup(rem) + } else { + Ok(child) + } + } else { + Err(ErrorCode::NoEntry) + } + } +} + +// #[derive(Debug)] +#[repr(C)] +struct StaticIndexEntry { + name: *const i8, + ty: StaticIndexType, + data: StaticFileData, +} + +#[repr(C)] +union StaticFileData { + /// Active memory data pointer for ActiveFile + active: (*const u8, usize), + /// Passive memory element index and len for PassiveFile + passive: (u32, usize), + /// Host path string for HostDir / HostFile + runtime_path: *const i8, + // Index and child entry count for Dir + dir: (usize, usize), +} + +// impl fmt::Debug for StaticFileData { +// fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { +// f.write_str(&format!( +// "STATIC [{:?}, {:?}]", +// unsafe { self.dir.0 }, +// unsafe { self.dir.1 } +// ))?; +// Ok(()) +// } +// } + +// #[derive(Debug)] +#[allow(dead_code)] +#[repr(u32)] +enum StaticIndexType { + ActiveFile, + PassiveFile, + Dir, + RuntimeDir, + RuntimeFile, +} + +// This function gets mutated by the virtualizer +#[no_mangle] +#[inline(never)] +pub fn passive_alloc(passive_idx: u32, offset: u32, len: u32) -> *mut u8 { + return (passive_idx + offset + len) as *mut u8; +} + +#[no_mangle] +pub static mut fs: Fs = Fs { + preopen_cnt: 0, // [byte 0] + preopens: 0 as *const usize, // [byte 4] + static_index_cnt: 0, // [byte 8] + static_index: 0 as *const StaticIndexEntry, // [byte 12] + host_passthrough: false, // [byte 16] +}; + +// local fs state +pub struct FsState { + initialized: bool, + descriptor_cnt: u32, + preopen_directories: Vec, + host_preopen_directories: BTreeMap, + descriptor_table: BTreeMap, + stream_cnt: u32, + stream_table: BTreeMap, +} + +static mut STATE: FsState = FsState { + initialized: false, + descriptor_cnt: 3, + preopen_directories: Vec::new(), + host_preopen_directories: BTreeMap::new(), + descriptor_table: BTreeMap::new(), + stream_cnt: 0, + stream_table: BTreeMap::new(), +}; + +enum Stream { + File(FileStream), + Dir(DirStream), +} + +impl From for Stream { + fn from(value: FileStream) -> Self { + Stream::File(value) + } +} + +impl From for Stream { + fn from(value: DirStream) -> Self { + Stream::Dir(value) + } +} + +struct FileStream { + // local file descriptor + fd: u32, + // current offset + offset: u64, +} + +struct DirStream { + fd: u32, + idx: usize, +} + +impl FileStream { + fn new(fd: u32) -> Self { + Self { fd, offset: 0 } + } + fn read(&mut self, len: u64) -> Result<(Vec, bool), StreamError> { + let Some(descriptor) = FsState::get_descriptor(self.fd) else { + return Err(StreamError {}); + }; + let (bytes, done) = descriptor + .get_bytes(self.offset, len) + .map_err(|_| StreamError {})?; + self.offset += bytes.len() as u64; + Ok((bytes, done)) + } +} + +impl DirStream { + fn new(fd: u32) -> Self { + Self { fd, idx: 0 } + } + fn next(&mut self) -> Result, ErrorCode> { + let Some(descriptor) = FsState::get_descriptor(self.fd) else { + return Err(ErrorCode::BadDescriptor); + }; + let child_list = descriptor.entry().child_list()?; + if self.idx < child_list.len() { + let child = &child_list[self.idx]; + self.idx += 1; + Ok(Some(DirectoryEntry { + inode: None, + type_: child.ty(), + name: child.name().into(), + })) + } else { + Ok(None) + } + } +} + +impl FsState { + fn initialize() { + if unsafe { STATE.initialized } { + return; + } + if unsafe { fs.host_passthrough } { + let host_preopen_directories = unsafe { &mut STATE.host_preopen_directories }; + for (fd, name) in preopens::get_directories() { + host_preopen_directories.insert(name, fd); + } + } + let preopens = Fs::preopens(); + for preopen in preopens { + let fd = FsState::create_descriptor(preopen, DescriptorFlags::READ); + unsafe { STATE.preopen_directories.push(fd) } + } + unsafe { STATE.initialized = true }; + } + fn get_host_preopen<'a>(path: &'a str) -> Option<(u32, &'a str)> { + let path = if path.starts_with("./") { + &path[2..] + } else { + path + }; + for (preopen_name, fd) in unsafe { &STATE.host_preopen_directories } { + let preopen_name = if preopen_name.starts_with("./") { + &preopen_name[2..] + } else if preopen_name.starts_with(".") { + &preopen_name[1..] + } else { + preopen_name + }; + if path.starts_with(preopen_name) { + // ambient relative + if preopen_name.len() == 0 { + if path.as_bytes()[0] != b'/' { + return Some((*fd, &path)); + } + } else { + // root '/' match + if preopen_name == "/" && path.as_bytes()[0] == b'/' { + return Some((*fd, &path[1..])); + } + // exact match + if preopen_name.len() == path.len() { + return Some((*fd, "")); + } + // normal [x]/ match + if path.as_bytes()[preopen_name.len()] == b'/' { + return Some((*fd, &path[preopen_name.len() + 1..])); + } + } + } + } + None + } + fn create_descriptor(entry: &StaticIndexEntry, _flags: DescriptorFlags) -> u32 { + let fd = unsafe { STATE.descriptor_cnt }; + unsafe { STATE.descriptor_cnt += 1 }; + let descriptor = Descriptor { + entry, + fd, + host_fd: None, + }; + assert!(unsafe { STATE.descriptor_table.insert(fd, descriptor) }.is_none()); + fd + } + fn get_descriptor<'a>(fd: u32) -> Option<&'a mut Descriptor> { + unsafe { STATE.descriptor_table.get_mut(&fd) } + } + fn get_preopen_directories() -> Vec<(u32, String)> { + FsState::initialize(); + unsafe { &STATE.preopen_directories } + .iter() + .map(|&fd| { + let descriptor = FsState::get_descriptor(fd).unwrap(); + let name = descriptor.entry().name(); + (fd, name.to_string()) + }) + .collect() + } + fn create_stream>(stream: S) -> Result { + let sid = unsafe { STATE.stream_cnt }; + unsafe { STATE.stream_cnt += 1 }; + unsafe { STATE.stream_table.insert(sid, stream.into()) }; + Ok(sid) + } + fn get_stream<'a>(sid: u32) -> Option<&'a mut Stream> { + unsafe { STATE.stream_table.get_mut(&sid) } + } + fn drop_stream(sid: u32) { + unsafe { STATE.stream_table.remove(&sid) }; + } +} + +impl Preopens for VirtAdapter { + fn get_directories() -> Vec<(u32, String)> { + FsState::get_preopen_directories() + } +} + +impl Filesystem for VirtAdapter { + fn read_via_stream(fd: u32, offset: u64) -> Result { + if offset != 0 { + return Err(ErrorCode::InvalidSeek); + } + FsState::create_stream(FileStream::new(fd)) + } + fn write_via_stream(_: u32, _: u64) -> Result { + Err(ErrorCode::Access) + } + fn append_via_stream(_fd: u32) -> Result { + Err(ErrorCode::Access) + } + fn advise(_: u32, _: u64, _: u64, _: Advice) -> Result<(), ErrorCode> { + todo!() + } + fn sync_data(_: u32) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn get_flags(_fd: u32) -> Result { + Ok(DescriptorFlags::READ) + } + fn get_type(fd: u32) -> Result { + let Some(descriptor) = FsState::get_descriptor(fd) else { + return Err(ErrorCode::BadDescriptor); + }; + Ok(descriptor.entry().ty()) + } + fn set_size(_: u32, _: u64) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn set_times(_: u32, _: NewTimestamp, _: NewTimestamp) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn read(fd: u32, len: u64, offset: u64) -> Result<(Vec, bool), ErrorCode> { + let sid = VirtAdapter::read_via_stream(fd, offset)?; + let stream = FsState::get_stream(sid).unwrap(); + let Stream::File(filestream) = stream else { + unreachable!() + }; + let result = filestream.read(len).map_err(|_| ErrorCode::Io)?; + FsState::drop_stream(sid); + Ok(result) + } + fn write(_: u32, _: Vec, _: u64) -> Result { + Err(ErrorCode::Access) + } + fn read_directory(fd: u32) -> Result { + let Some(descriptor) = FsState::get_descriptor(fd) else { + return Err(ErrorCode::BadDescriptor); + }; + if descriptor.entry().ty() != DescriptorType::Directory { + return Err(ErrorCode::NotDirectory); + } + FsState::create_stream(DirStream::new(fd)) + } + fn sync(_: u32) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn create_directory_at(_: u32, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn stat(fd: u32) -> Result { + let Some(descriptor) = FsState::get_descriptor(fd) else { + return Err(ErrorCode::BadDescriptor); + }; + Ok(DescriptorStat { + device: 0, + inode: 0, + type_: descriptor.entry().ty(), + link_count: 0, + size: descriptor.entry().size()?, + data_access_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + data_modification_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + status_change_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + }) + } + fn stat_at(fd: u32, _flags: PathFlags, path: String) -> Result { + let Some(descriptor) = FsState::get_descriptor(fd) else { + return Err(ErrorCode::BadDescriptor); + }; + let child = descriptor.entry().dir_lookup(&path)?; + Ok(DescriptorStat { + device: 0, + inode: 0, + type_: child.ty(), + link_count: 0, + size: child.size()?, + data_access_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + data_modification_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + status_change_timestamp: Datetime { + seconds: 0, + nanoseconds: 0, + }, + }) + } + fn set_times_at( + _: u32, + _: PathFlags, + _: String, + _: NewTimestamp, + _: NewTimestamp, + ) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn link_at(_: u32, _: PathFlags, _: String, _: u32, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn open_at( + fd: u32, + _path_flags: PathFlags, + path: String, + _open_flags: OpenFlags, + descriptor_flags: DescriptorFlags, + _modes: Modes, + ) -> Result { + let Some(descriptor) = FsState::get_descriptor(fd) else { + return Err(ErrorCode::BadDescriptor); + }; + let child = descriptor.entry().dir_lookup(&path)?; + Ok(FsState::create_descriptor(child, descriptor_flags)) + } + fn readlink_at(_: u32, _: String) -> Result { + todo!() + } + fn remove_directory_at(_: u32, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn rename_at(_: u32, _: String, _: u32, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn symlink_at(_: u32, _: String, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn access_at(_: u32, _: PathFlags, _: String, _: AccessType) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn unlink_file_at(_: u32, _: String) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn change_file_permissions_at( + _: u32, + _: PathFlags, + _: String, + _: Modes, + ) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn change_directory_permissions_at( + _: u32, + _: PathFlags, + _: String, + _: Modes, + ) -> Result<(), ErrorCode> { + Err(ErrorCode::Access) + } + fn lock_shared(_fd: u32) -> Result<(), ErrorCode> { + Ok(()) + } + fn lock_exclusive(_fd: u32) -> Result<(), ErrorCode> { + Ok(()) + } + fn try_lock_shared(_fd: u32) -> Result<(), ErrorCode> { + Ok(()) + } + fn try_lock_exclusive(_fd: u32) -> Result<(), ErrorCode> { + Ok(()) + } + fn unlock(_: u32) -> Result<(), ErrorCode> { + Ok(()) + } + fn drop_descriptor(fd: u32) { + if let Some(descriptor) = FsState::get_descriptor(fd) { + descriptor.drop(); + }; + } + fn read_directory_entry(sid: u32) -> Result, ErrorCode> { + let Some(stream) = FsState::get_stream(sid) else { + return Err(ErrorCode::BadDescriptor); + }; + match stream { + Stream::Dir(dirstream) => dirstream.next(), + _ => { + return Err(ErrorCode::BadDescriptor); + } + } + } + fn drop_directory_entry_stream(sid: u32) { + FsState::drop_stream(sid); + } +} + +impl Streams for VirtAdapter { + fn read(sid: u32, len: u64) -> Result<(Vec, bool), StreamError> { + VirtAdapter::blocking_read(sid, len) + } + fn blocking_read(sid: u32, len: u64) -> Result<(Vec, bool), StreamError> { + let Some(stream) = FsState::get_stream(sid) else { + return Err(StreamError {}); + }; + match stream { + Stream::File(filestream) => filestream.read(len), + _ => Err(StreamError {}), + } + } + fn skip(_: u32, _: u64) -> Result<(u64, bool), StreamError> { + todo!() + } + fn blocking_skip(_: u32, _: u64) -> Result<(u64, bool), StreamError> { + todo!() + } + fn subscribe_to_input_stream(_: u32) -> u32 { + todo!() + } + fn drop_input_stream(sid: u32) { + FsState::drop_stream(sid); + } + fn write(_: u32, _: Vec) -> Result { + Err(StreamError {}) + } + fn blocking_write(_: u32, _: Vec) -> Result { + Err(StreamError {}) + } + fn write_zeroes(_: u32, _: u64) -> Result { + Err(StreamError {}) + } + fn blocking_write_zeroes(_: u32, _: u64) -> Result { + Err(StreamError {}) + } + fn splice(_: u32, _: u32, _: u64) -> Result<(u64, bool), StreamError> { + todo!() + } + fn blocking_splice(_: u32, _: u32, _: u64) -> Result<(u64, bool), StreamError> { + todo!() + } + fn forward(_: u32, _: u32) -> Result { + todo!() + } + fn subscribe_to_output_stream(_: u32) -> u32 { + todo!() + } + fn drop_output_stream(_: u32) { + todo!() + } +} diff --git a/virtual-adapter/src/lib.rs b/virtual-adapter/src/lib.rs index 3630368..0db6e15 100644 --- a/virtual-adapter/src/lib.rs +++ b/virtual-adapter/src/lib.rs @@ -1,97 +1,13 @@ #![no_main] +mod env; +mod fs; + wit_bindgen::generate!({ path: "../wit", world: "virtual-adapter" }); -use crate::exports::wasi::cli_base::environment::Environment; - -struct VirtAdapter; +pub(crate) struct VirtAdapter; export_virtual_adapter!(VirtAdapter); - -#[repr(C)] -pub struct Env { - /// Whether to fallback to the host env - /// [byte 0] - host_fallback: bool, - /// Whether we are providing an allow list or a deny list - /// on the fallback lookups - /// [byte 1] - host_fallback_allow: bool, - /// How many host fields are defined in the data pointer - /// [byte 4] - host_field_cnt: u32, - /// Host many host fields are defined to be allow or deny - /// (these are concatenated at the end of the data with empty values) - /// [byte 8] - host_allow_or_deny_cnt: u32, - /// Byte data of u32 byte len followed by string bytes - /// up to the lengths previously provided. - /// [byte 12] - host_field_data: *const u8, -} - -#[no_mangle] -pub static mut env: Env = Env { - host_fallback: true, - host_fallback_allow: false, - host_field_cnt: 0, - host_allow_or_deny_cnt: 0, - host_field_data: 0 as *const u8, -}; - -fn read_data_str(offset: &mut isize) -> &'static str { - let data: *const u8 = unsafe { env.host_field_data.offset(*offset) }; - let byte_len = unsafe { (data as *const u32).read() } as usize; - *offset += 4; - let data: *const u8 = unsafe { env.host_field_data.offset(*offset) }; - let str_data = unsafe { std::slice::from_raw_parts(data, byte_len) }; - *offset += byte_len as isize; - let rem = *offset % 4; - if rem > 0 { - *offset += 4 - rem; - } - unsafe { core::str::from_utf8_unchecked(str_data) } -} - -impl Environment for VirtAdapter { - fn get_environment() -> Vec<(String, String)> { - let mut environment = Vec::new(); - let mut data_offset: isize = 0; - for _ in 0..unsafe { env.host_field_cnt } { - let env_key = read_data_str(&mut data_offset); - let env_val = read_data_str(&mut data_offset); - environment.push((env_key.to_string(), env_val.to_string())); - } - let override_len = environment.len(); - // fallback ASSUMES that all data is alphabetically ordered - if unsafe { env.host_fallback } { - let mut allow_or_deny = Vec::new(); - for _ in 0..unsafe { env.host_allow_or_deny_cnt } { - let allow_or_deny_key = read_data_str(&mut data_offset); - allow_or_deny.push(allow_or_deny_key); - } - - let is_allow_list = unsafe { env.host_fallback_allow }; - for (key, value) in wasi::cli_base::environment::get_environment() { - if environment[0..override_len] - .binary_search_by_key(&&key, |(s, _)| s) - .is_ok() - { - continue; - } - let in_list = allow_or_deny.binary_search(&key.as_ref()).is_ok(); - if is_allow_list && in_list || !is_allow_list && !in_list { - environment.push((key, value)); - } - } - } - environment - } - - fn get_arguments() -> Vec { - wasi::cli_base::environment::get_arguments() - } -} diff --git a/wit/virt.wit b/wit/virt.wit index df99b6c..75f7e82 100644 --- a/wit/virt.wit +++ b/wit/virt.wit @@ -1,11 +1,39 @@ package local:virt +// in future this should be defined as a union world of the various +// virtual subsystems, when union syntax lands world virtual-adapter { import wasi:cli-base/environment + import wasi:cli-base/preopens + import wasi:filesystem/filesystem + import wasi:io/streams import console: interface { log: func(msg: string) -> () } + export wasi:io/streams export wasi:cli-base/environment + export wasi:filesystem/filesystem + export wasi:cli-base/preopens +} + +world virtual-base { + import console: interface { + log: func(msg: string) -> () + } +} + +world virtual-env { + import wasi:cli-base/environment + export wasi:cli-base/environment +} + +world virtual-fs { + import wasi:cli-base/preopens + import wasi:filesystem/filesystem + import wasi:io/streams + export wasi:io/streams + export wasi:filesystem/filesystem + export wasi:cli-base/preopens } world virt-test { @@ -33,4 +61,5 @@ world virt-test { import wasi:cli-base/stderr export test-get-env: func() -> list> + export test-file-read: func(path: string) -> string }