From 40a6d0c3d460f20b97325d423fc0bc7600ef7e9e Mon Sep 17 00:00:00 2001 From: James Sturtevant Date: Fri, 21 Jun 2024 22:36:17 +0000 Subject: [PATCH] Add ability to generate new artifact type Signed-off-by: James Sturtevant --- Cargo.lock | 392 +++++++++++++++++++-- crates/containerd-shim-wasm/src/testing.rs | 2 +- crates/oci-tar-builder/Cargo.toml | 2 + crates/oci-tar-builder/README.md | 54 ++- crates/oci-tar-builder/src/bin.rs | 146 ++++++-- crates/oci-tar-builder/src/lib.rs | 84 ++++- crates/wasi-demo-app/build.rs | 1 + 7 files changed, 597 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b6571b14..4817147bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -208,9 +208,9 @@ dependencies = [ "bitflags 1.3.2", "bytes", "futures-util", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "itoa", "matchit", "memchr", @@ -234,8 +234,8 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "mime", "rustversion", "tower-layer", @@ -257,12 +257,24 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bincode" version = "1.3.3" @@ -520,8 +532,10 @@ checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", + "js-sys", "num-traits", "serde", + "wasm-bindgen", "windows-targets 0.52.5", ] @@ -1384,6 +1398,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", + "subtle", ] [[package]] @@ -1855,7 +1870,7 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", "indexmap 2.2.6", "slab", "tokio", @@ -1962,6 +1977,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" version = "0.5.9" @@ -1982,6 +2006,26 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-auth" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "643c9bbf6a4ea8a656d6b4cd53d34f79e3f841ad5203c1a55fb7d761923bc255" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "0.4.6" @@ -1989,7 +2033,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -2022,8 +2089,8 @@ dependencies = [ "futures-core", "futures-util", "h2", - "http", - "http-body", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -2035,6 +2102,25 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -2042,8 +2128,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "rustls", "tokio", "tokio-rustls", @@ -2055,7 +2141,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" dependencies = [ - "hyper", + "hyper 0.14.28", "pin-project-lite", "tokio", "tokio-io-timeout", @@ -2068,12 +2154,48 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", "native-tls", "tokio", "tokio-native-tls", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2 0.5.7", + "tokio", + "tower", + "tower-service", + "tracing", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -2242,6 +2364,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64 0.13.1", + "crypto-common", + "digest", + "hmac", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2739,6 +2876,31 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-distribution" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95a2c51531af0cb93761f66094044ca6ea879320bccd35ab747ff3fcab3f422" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http 1.1.0", + "http-auth", + "jwt", + "lazy_static", + "olpc-cjson", + "regex", + "reqwest 0.12.4", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tracing", + "unicase", +] + [[package]] name = "oci-spec" version = "0.6.5" @@ -2761,10 +2923,40 @@ dependencies = [ "indexmap 2.2.6", "log", "oci-spec", + "oci-wasm", "serde", "serde_json", "sha256", "tar", + "tokio", +] + +[[package]] +name = "oci-wasm" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a91502e5352f927156f2b6a28d2558cc59558b1f441b681df3f706ced6937e07" +dependencies = [ + "anyhow", + "chrono", + "oci-distribution", + "serde", + "serde_json", + "sha2", + "tokio", + "wit-component", + "wit-parser", +] + +[[package]] +name = "olpc-cjson" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d637c9c15b639ccff597da8f4fa968300651ad2f1e968aefc3b4927a6fb2027a" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", ] [[package]] @@ -2845,9 +3037,9 @@ checksum = "b0ba633e55c5ea6f431875ba55e71664f2fa5d3a90bd34ec9302eecc41c865dd" dependencies = [ "async-trait", "bytes", - "http", + "http 0.2.12", "opentelemetry", - "reqwest", + "reqwest 0.11.27", ] [[package]] @@ -2858,13 +3050,13 @@ checksum = "a94c69209c05319cdf7460c6d4c055ed102be242a0a6245835d7bc42c6ec7f54" dependencies = [ "async-trait", "futures-core", - "http", + "http 0.2.12", "opentelemetry", "opentelemetry-http", "opentelemetry-proto", "opentelemetry_sdk", "prost 0.12.6", - "reqwest", + "reqwest 0.11.27", "thiserror", "tokio", "tonic", @@ -3626,17 +3818,17 @@ version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-rustls", - "hyper-tls", + "hyper-tls 0.5.0", "ipnet", "js-sys", "log", @@ -3646,7 +3838,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rustls", - "rustls-pemfile", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", @@ -3661,7 +3853,48 @@ dependencies = [ "wasm-bindgen-futures", "web-sys", "webpki-roots", - "winreg", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-tls 0.6.0", + "hyper-util", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 2.1.2", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg 0.52.0", ] [[package]] @@ -3775,9 +4008,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4153,6 +4402,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spdx" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47317bbaf63785b53861e1ae2d11b80d6b624211d42cb20efcd210ee6f8a14bc" +dependencies = [ + "smallvec", +] + [[package]] name = "spin" version = "0.9.8" @@ -4186,6 +4444,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + [[package]] name = "syn" version = "1.0.109" @@ -4561,12 +4825,12 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64", + "base64 0.21.7", "bytes", "h2", - "http", - "http-body", - "hyper", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", "hyper-timeout", "percent-encoding", "pin-project", @@ -4937,7 +5201,7 @@ checksum = "306bb6ff6ed62c44f50a72c7c89f95439bb0f56cce7c3963cd3ae6d292a04af5" dependencies = [ "anyhow", "async-trait", - "base64", + "base64 0.21.7", "bincode", "bytes", "derivative", @@ -5208,6 +5472,35 @@ dependencies = [ "leb128", ] +[[package]] +name = "wasm-metadata" +version = "0.209.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d32029ce424f6d3c2b39b4419fb45a0e2d84fb0751e0c0a32b7ce8bd5d97f46" +dependencies = [ + "anyhow", + "indexmap 2.2.6", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder 0.209.1", + "wasmparser 0.209.1", +] + +[[package]] +name = "wasm-streams" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmedge-macro" version = "0.6.1" @@ -5251,7 +5544,7 @@ dependencies = [ "paste", "phf", "rand", - "reqwest", + "reqwest 0.11.27", "scoped-tls", "setjmp", "sha256", @@ -5434,7 +5727,7 @@ dependencies = [ "getrandom", "heapless", "hex", - "http", + "http 0.2.12", "lazy_static", "libc", "linked_hash_set", @@ -5443,7 +5736,7 @@ dependencies = [ "pin-project", "rand", "rayon", - "reqwest", + "reqwest 0.11.27", "semver", "serde", "serde_cbor", @@ -5616,7 +5909,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "916610f9ae9a6c22deb25bba2e6247ba9f00b093d30620875203b91328a1adfa" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "directories-next", "log", "postcard", @@ -5886,7 +6179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "973ca5a91b4fb3e4bb37cfebe03ef9364d0aff2765256abefdb7e79dc9188483" dependencies = [ "anyhow", - "base64", + "base64 0.21.7", "byteorder", "bytes", "flate2", @@ -6244,6 +6537,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if 1.0.0", + "windows-sys 0.48.0", +] + [[package]] name = "winx" version = "0.36.3" @@ -6254,6 +6557,25 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "wit-component" +version = "0.209.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bb5b039f9cb03425e1d5a6e54b441ca4ca1b1d4fa6a0924db67a55168f99" +dependencies = [ + "anyhow", + "bitflags 2.5.0", + "indexmap 2.2.6", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder 0.209.1", + "wasm-metadata", + "wasmparser 0.209.1", + "wit-parser", +] + [[package]] name = "wit-parser" version = "0.209.1" diff --git a/crates/containerd-shim-wasm/src/testing.rs b/crates/containerd-shim-wasm/src/testing.rs index 8a6760bc6..1e258d69d 100644 --- a/crates/containerd-shim-wasm/src/testing.rs +++ b/crates/containerd-shim-wasm/src/testing.rs @@ -314,7 +314,7 @@ pub mod oci_helpers { .unwrap(), ) .build()?; - builder.add_config(img, image_name.to_string()); + builder.add_config(img, image_name.to_string(), spec::MediaType::ImageConfig); let img_path = dir.join("img.tar"); let f = File::create(img_path.clone())?; builder.build(f)?; diff --git a/crates/oci-tar-builder/Cargo.toml b/crates/oci-tar-builder/Cargo.toml index 099517739..c01f0328c 100644 --- a/crates/oci-tar-builder/Cargo.toml +++ b/crates/oci-tar-builder/Cargo.toml @@ -18,6 +18,8 @@ serde = { workspace = true } serde_json = { workspace = true } clap = { version = "4.5.13", features = ["derive"] } indexmap = "2.2.6" +oci-wasm = "0.0.4" +tokio = { version = "1.38.0", features = [ "full" ] } [lib] path = "src/lib.rs" diff --git a/crates/oci-tar-builder/README.md b/crates/oci-tar-builder/README.md index 2437c2614..2d25c64c1 100644 --- a/crates/oci-tar-builder/README.md +++ b/crates/oci-tar-builder/README.md @@ -22,7 +22,7 @@ See [wasi-demo-app build script](../wasi-demo-app/build.rs) for an example. There is an experimental executable that uses the library and can package a wasm module as an OCI image with wasm layers. See the [OCI WASM in containerd](https://docs.google.com/document/d/11shgC3l6gplBjWF1VJCWvN_9do51otscAm0hBDGSSAc) for more information. -To generate the package and import to a registry using a tool such as [regctl](https://github.com/regclient/regclient/blob/main/docs/regctl.md#image-commands): +To generate the package and import to a registry using a tool such as [regctl](https://github.com/regclient/regclient/blob/main/docs/install.md). To [run a local image registry](https://www.docker.com/blog/how-to-use-your-own-registry-2/) use `docker run -d -p 5000:5000 --name registry registry:2.7` ``` cargo run --bin oci-tar-builder -- --name wasi-demo-oci --repo ghcr.io/containerd/runwasi --tag latest --module ./target/wasm32-wasi/debug/wasi-demo-app.wasm -o ./dist/img-oci.tar @@ -56,4 +56,54 @@ Layers: See the [OCI Image Spec](https://github.com/opencontainers/image-spec/blob/bc9c4bd/image-layout.md) for more information on the OCI tar format. -In order to be compatible with Docker, since Docker does not currently support the OCI format, this also includes a `manifest.json` file at the root of the tar that describes the image in a way that Docker can import it. \ No newline at end of file +In order to be compatible with Docker, since Docker does not currently support the OCI format, this also includes a `manifest.json` file at the root of the tar that describes the image in a way that Docker can import it. + +### Wasm Artifact usage + +The CNCF wg-wasm has published and [OCI artifact format](https://tag-runtime.cncf.io/wgs/wasm/deliverables/wasm-oci-artifact/) for packaging wasm modules and components. The artifact can be produced locally by running the `--as-artifact` flag: + +``` +cargo run --bin oci-tar-builder -- --name wasi-demo-oci --repo ghcr.io/containerd/runwasi --tag latest --as-artifact --module ./target/wasm32-wasi/debug/wasi-demo-app.wasm -o target/wasm32-wasi/debug/img-oci-artifact.tar +regctl image import localhost:5000/wasi-artifact:latest target/wasm32-wasi/debug/img-oci-artifact.tar +``` + +The manifest will follow the guidance: + +``` +regctl manifest get localhost:5000/wasi-artifact:latest + +Name: localhost:5000/wasi-artifact:latest +MediaType: application/vnd.oci.image.manifest.v1+json +Digest: sha256:7c31e635b3bef8b6c727a316e9a2dae777dbd184318d66a97da040fb11e37d70 +Annotations: + io.containerd.image.name: ghcr.io/containerd/runwasi/wasi-demo-oci:latest + org.opencontainers.image.ref.name: latest +Total Size: 2.006MB + +Config: + Digest: sha256:24f30be41b447bbaf3644dad1e1c23dd28b597f36a5f455399657d65945816ea + MediaType: application/vnd.wasm.config.v0+json + Size: 235B + +Layers: + + Digest: sha256:0db51ed1c94837f422b2259c473758f298eef69605eaae6195bc043e25971e94 + MediaType: application/wasm + Size: 2.006MB +``` + +As well as the `config.mediaType` will have the following format: + +``` + regctl blob get localhost:5000/wasi-artifact:latest sha256:24f30be41b447bbaf3644dad1e1c23dd28b597f36a5f455399657d65945816ea +{ + "created": "2024-06-25T15:58:49.377917735Z", + "author": null, + "architecture": "wasm", + "os": "wasip1", + "layerDigests": [ + "sha256:0db51ed1c94837f422b2259c473758f298eef69605eaae6195bc043e25971e94" + ], + "component": null +} +``` \ No newline at end of file diff --git a/crates/oci-tar-builder/src/bin.rs b/crates/oci-tar-builder/src/bin.rs index 95559d254..e46488d5a 100644 --- a/crates/oci-tar-builder/src/bin.rs +++ b/crates/oci-tar-builder/src/bin.rs @@ -5,11 +5,13 @@ use std::{env, fs}; use anyhow::Context; use clap::Parser; -use oci_spec::image::{self as spec, Arch}; -use oci_tar_builder::{Builder, WASM_LAYER_MEDIA_TYPE}; +use oci_spec::image::{self as spec, Arch, ImageConfiguration}; +use oci_tar_builder::Builder; +use oci_wasm::WasmConfig; use sha256::{digest, try_digest}; -pub fn main() { +#[tokio::main] +pub async fn main() { let args = Args::parse(); let out_dir; @@ -20,18 +22,88 @@ pub fn main() { out_dir = env::current_dir().unwrap(); } + if !args.module.is_empty() && args.components.is_some() { + println!("Mutually exclusive flags: module and components"); + return; + } + + if args.module.is_empty() && args.components.is_none() { + println!("Must supply module or components"); + return; + } + + if args.as_artifact { + generate_wasm_artifact(args, out_dir).await.unwrap(); + } else { + generate_wasi_image(args, out_dir).unwrap(); + } +} + +async fn generate_wasm_artifact(args: Args, out_dir: PathBuf) -> Result<(), anyhow::Error> { + println!("Generating wasm artifact"); + + let mut builder = Builder::::default(); + + let (conf, path) = match args.components { + Some(path) => { + let paths = fs::read_dir(&path)?; + if paths.count() != 1 { + println!( + "Currently only supports a single component file {:?}", + &path + ); + } + let (conf, _) = WasmConfig::from_component(&path, None).await?; + (conf, path) + } + None => { + let module_path = args.module.first().unwrap(); + let (conf, _) = WasmConfig::from_module(module_path, None).await?; + (conf, module_path.to_string()) + } + }; + + builder.add_config( + conf, + args.repo + "/" + &args.name + ":" + &args.tag, + spec::MediaType::Other(oci_wasm::WASM_MANIFEST_CONFIG_MEDIA_TYPE.to_string()), + ); + + let module_path = PathBuf::from(path); + builder.add_layer_with_media_type(&module_path, oci_wasm::WASM_LAYER_MEDIA_TYPE.to_string()); + + println!("Creating oci tar file {}", out_dir.clone().display()); + let f = File::create(out_dir.clone())?; + match builder.build(f) { + Ok(_) => println!("Successfully created oci tar file {}", out_dir.display()), + Err(e) => { + print!( + "Building oci tar file {} failed: {:?}", + out_dir.display(), + e + ); + fs::remove_file(out_dir).unwrap_or(print!("Failed to clean up oci tar file on error")); + } + } + + Ok(()) +} + +fn generate_wasi_image(args: Args, out_dir: PathBuf) -> Result<(), anyhow::Error> { + println!("Generating wasm oci image"); let entry_point = args.name.clone() + ".wasm"; - let mut builder = Builder::default(); + let mut builder = Builder::::default(); let mut layer_digests = Vec::new(); for module_path in args.module.iter() { let module_path = PathBuf::from(module_path); - builder.add_layer_with_media_type(&module_path, WASM_LAYER_MEDIA_TYPE.to_string()); - layer_digests.push( - try_digest(&module_path) - .context("failed to calculate digest for module") - .unwrap(), + builder.add_layer_with_media_type( + &module_path, + oci_tar_builder::WASM_LAYER_MEDIA_TYPE.to_string(), ); + + layer_digests + .push(try_digest(&module_path).context("failed to calculate digest for module")?); } for layer_config in args.layer.iter() { @@ -41,32 +113,33 @@ pub fn main() { let layer_type = layer_options.first().unwrap(); let layer_path = PathBuf::from(layer_options.last().unwrap()); builder.add_layer_with_media_type(&layer_path, layer_type.to_string()); - layer_digests.push( - try_digest(&layer_path) - .context("failed to calculate digest for module") - .unwrap(), - ); + layer_digests + .push(try_digest(&layer_path).context("failed to calculate digest for module")?); } if let Some(components_path) = args.components.as_deref() { - let paths = fs::read_dir(components_path).unwrap(); + let paths = fs::read_dir(components_path)?; for path in paths { - let path = path.unwrap().path(); - let ext = path.extension().unwrap().to_str().unwrap(); + let path = path?.path(); + let ext = path + .extension() + .unwrap_or(std::ffi::OsStr::new("")) + .to_str() + .unwrap_or(""); match ext { "wasm" => { - builder.add_layer_with_media_type(&path, WASM_LAYER_MEDIA_TYPE.to_string()); - layer_digests.push( - try_digest(&path) - .context("failed to calculate digest for module") - .unwrap(), + builder.add_layer_with_media_type( + &path, + oci_tar_builder::WASM_LAYER_MEDIA_TYPE.to_string(), ); + layer_digests + .push(try_digest(&path).context("failed to calculate digest for module")?); } _ => println!( "Skipping Unknown file type: {:?} with extension {:?}", path, - path.extension().unwrap() + path.extension().unwrap_or(std::ffi::OsStr::new("")) ), } } @@ -77,27 +150,25 @@ pub fn main() { let unique_id = digest(layer_digests.join("")); let mut labels: HashMap = HashMap::new(); labels.insert("containerd.runwasi.layers".to_string(), unique_id); + let config = spec::ConfigBuilder::default() .entrypoint(vec![entry_point]) .labels(labels) - .build() - .unwrap(); + .build()?; - let img = spec::ImageConfigurationBuilder::default() + let conf = spec::ImageConfigurationBuilder::default() .config(config) .os("wasip1") .architecture(Arch::Wasm) - .rootfs( - spec::RootFsBuilder::default() - .diff_ids(vec![]) - .build() - .unwrap(), - ) + .rootfs(spec::RootFsBuilder::default().diff_ids(vec![]).build()?) .build() - .context("failed to build image configuration") - .unwrap(); + .context("failed to build image configuration")?; - builder.add_config(img, args.repo + "/" + &args.name + ":" + &args.tag); + builder.add_config( + conf, + format!("{}/{}:{}", args.repo, args.name, args.tag), + spec::MediaType::ImageConfig, + ); println!("Creating oci tar file {}", out_dir.clone().display()); let f = File::create(out_dir.clone()).unwrap(); @@ -112,6 +183,8 @@ pub fn main() { fs::remove_file(out_dir).unwrap_or(print!("Failed to remove temporary file")); } } + + Ok(()) } #[derive(Parser, Debug)] @@ -137,4 +210,7 @@ struct Args { #[arg(short, long)] components: Option, + + #[arg(short, long)] + as_artifact: bool, } diff --git a/crates/oci-tar-builder/src/lib.rs b/crates/oci-tar-builder/src/lib.rs index 2a08f9e50..16d61065c 100644 --- a/crates/oci-tar-builder/src/lib.rs +++ b/crates/oci-tar-builder/src/lib.rs @@ -10,14 +10,76 @@ use oci_spec::image::{ DescriptorBuilder, ImageConfiguration, ImageIndexBuilder, ImageManifestBuilder, MediaType, PlatformBuilder, SCHEMA_VERSION, }; +use oci_wasm::{WasmConfig, WASM_ARCHITECTURE}; use serde::Serialize; use sha256::{digest, try_digest}; -#[derive(Debug, Default)] -pub struct Builder { - configs: Vec<(ImageConfiguration, String)>, +#[derive(Debug)] +pub struct Builder { + configs: Vec<(C, String, MediaType)>, layers: Vec<(PathBuf, String)>, } +pub trait OciConfig { + fn os(&self) -> String; + fn architecture(&self) -> String; + fn layers(&self) -> Vec; + fn to_string(&self) -> String; +} + +impl OciConfig for ImageConfiguration { + fn os(&self) -> String { + self.os().to_string() + } + + fn architecture(&self) -> String { + self.architecture().to_string() + } + + fn layers(&self) -> Vec { + self.rootfs().diff_ids().to_vec() + } + + fn to_string(&self) -> String { + self.to_string_pretty().unwrap() + } +} + +impl OciConfig for WasmConfig { + fn os(&self) -> String { + self.os.to_string() + } + + fn architecture(&self) -> String { + WASM_ARCHITECTURE.to_string() + } + + fn layers(&self) -> Vec { + self.layer_digests.clone() + } + + fn to_string(&self) -> String { + serde_json::to_string_pretty(self).unwrap() + } +} + +impl Default for Builder { + fn default() -> Self { + Self { + configs: Vec::new(), + layers: Vec::new(), + } + } +} + +impl Default for Builder { + fn default() -> Self { + Self { + configs: Vec::new(), + layers: Vec::new(), + } + } +} + #[derive(Serialize, Debug)] struct OciLayout { #[serde(rename = "imageLayoutVersion")] @@ -45,9 +107,9 @@ struct DockerManifest { pub const WASM_LAYER_MEDIA_TYPE: &str = "application/vnd.bytecodealliance.wasm.component.layer.v0+wasm"; -impl Builder { - pub fn add_config(&mut self, config: ImageConfiguration, name: String) -> &mut Self { - self.configs.push((config, name)); +impl Builder { + pub fn add_config(&mut self, config: C, name: String, media_type: MediaType) -> &mut Self { + self.configs.push((config, name, media_type)); self } @@ -108,7 +170,7 @@ impl Builder { } for config in self.configs.iter() { - let s = config.0.to_string().context("failed to serialize config")?; + let s = config.0.to_string(); let b = s.as_bytes(); let dgst = digest(b); let mut th = tar::Header::new_gnu(); @@ -122,7 +184,7 @@ impl Builder { mfst.config = p.to_string(); let desc = DescriptorBuilder::default() - .media_type(MediaType::ImageConfig) + .media_type(config.2.clone()) .size(b.len() as i64) .digest("sha256:".to_owned() + &dgst) .build() @@ -134,7 +196,7 @@ impl Builder { layers.push(v.clone()); } - for id in config.0.rootfs().diff_ids().iter() { + for id in config.0.layers().iter() { debug!("id: {}", id); if layer_digests.get(id).is_none() { warn!("rootfs diff with id {} not found in layers", id); @@ -176,8 +238,8 @@ impl Builder { tb.append(&th, b)?; let platform = PlatformBuilder::default() - .os(config.0.os().clone()) - .architecture(config.0.architecture().clone()) + .os(config.0.os().as_str()) + .architecture(config.0.architecture().as_str()) .build() .context("failed to build platform")?; diff --git a/crates/wasi-demo-app/build.rs b/crates/wasi-demo-app/build.rs index 70a336691..5b85b159a 100644 --- a/crates/wasi-demo-app/build.rs +++ b/crates/wasi-demo-app/build.rs @@ -59,6 +59,7 @@ fn main() { builder.add_config( img, "ghcr.io/containerd/runwasi/wasi-demo-app:latest".to_string(), + spec::MediaType::ImageConfig, ); let f = File::create(&p).unwrap();