feat(*)!: Uses wasm-pkg-client for loading dependencies

This is a fairly large PR because this dependency is used everywhere. cargo component now uses the new wasm-pkg-tools toolchain to load deps, which means that both OCI and Warg are supported. This tries to stay as close to the original code style as possible, but I did have to make large changes to how dependencies were being resolved to account for the new library. There are a couple major breaking changes to be aware of and one question to answer. This PR is already quite large, so I figured it would be better to probably merge this and then deal with follow ups to any of the outstanding questions below: - Bindings are now generated every time. Where the actual dependency is being loaded from is now abstracted away by the client, so we can't check if those files have changed on disk. I personally don't think this is the worst thing, but if people really want that functionality, we can add support for that into the package tools. - Offline deps currently does not work, but that is something that will be added into wasm-pkg-client instead of doing checks within cargo component itself. - Currently the registries specified within Cargo.toml are not used in favor of using the config file for wasm-pkg-tools. I am not sure if we wanted to use that to override the wasm-pkg-tools config or just drop it entirely. I would like to address this one before merge. - Until wasm-pkg-client has support for publishing, I have left support in to use warg directly. Once we have that added to wasm-pkg-tools, we can change the behavior here Signed-off-by: Taylor Thomas <[email protected]>
bytecodealliance · Jul 31, 2024 · f1895a5 · f1895a5
1 parent f16644d
commit f1895a5
Show file tree

Hide file tree

Showing 47 changed files with 2,601 additions and 1,944 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,96 +18,104 @@ keywords = ["webassembly", "wasm", "components", "component-model"]
 repository = "https://github.com/bytecodealliance/cargo-component"
 
 [dependencies]
-cargo-component-core = { workspace = true }
 anyhow = { workspace = true }
+bytes = { workspace = true }
+cargo_metadata = { workspace = true }
+cargo-component-core = { workspace = true }
+cargo-config2 = { workspace = true }
 clap = { workspace = true }
-toml_edit = { workspace = true }
-pretty_env_logger = { workspace = true }
-log = { workspace = true }
-tokio = { workspace = true }
-tokio-util = { workspace = true }
+futures = { workspace = true }
 heck = { workspace = true }
-semver = { workspace = true }
-serde = { workspace = true }
-serde_json = { workspace = true }
 indexmap = { workspace = true }
-url = { workspace = true }
-wit-bindgen-rust = { workspace = true }
-wit-bindgen-core = { workspace = true }
-wit-parser = { workspace = true }
-wit-component = { workspace = true }
-wasm-metadata = { workspace = true }
-wasmparser = { workspace = true }
-parse_arg = { workspace = true }
-cargo_metadata = { workspace = true }
-cargo-config2 = { workspace = true }
 libc = { workspace = true }
-warg-protocol = { workspace = true }
-warg-crypto = { workspace = true }
-warg-client = { workspace = true }
+log = { workspace = true }
 p256 = { workspace = true }
+parse_arg = { workspace = true }
+pretty_env_logger = { workspace = true }
 rand_core = { workspace = true }
 rpassword = { workspace = true }
-futures = { workspace = true }
-bytes = { workspace = true }
-which = { workspace = true }
+semver = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
 shell-escape = "0.1.5"
 tempfile = { workspace = true }
-wasi-preview1-component-adapter-provider = "23.0.1"
+tokio = { workspace = true }
+tokio-util = { workspace = true }
+toml_edit = { workspace = true }
+url = { workspace = true }
+warg-client = { workspace = true }
+warg-crypto = { workspace = true }
+warg-protocol = { workspace = true }
+wasi-preview1-component-adapter-provider = { workspace = true }
+wasm-metadata = { workspace = true }
+wasm-pkg-client = { workspace = true }
+wasmparser = { workspace = true }
+which = { workspace = true }
+wit-bindgen-core = { workspace = true }
+wit-bindgen-rust = { workspace = true }
+wit-component = { workspace = true }
+wit-parser = { workspace = true }
 
 [dev-dependencies]
 assert_cmd = { workspace = true }
 predicates = { workspace = true }
-wat = { workspace = true }
-warg-server = { workspace = true }
 tempfile = { workspace = true }
+warg-server = { workspace = true }
 wasmprinter = { workspace = true }
+wat = { workspace = true }
 
 [workspace]
 members = ["crates/core", "crates/wit"]
 
 [workspace.dependencies]
-cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
-warg-protocol = "0.7.0"
-warg-crypto = "0.7.0"
-warg-client = "0.7.0"
-warg-server = "0.7.0"
 anyhow = "1.0.82"
-clap = { version = "4.5.4", features = ["derive"] }
-toml_edit = { version = "0.22.9", features = ["serde"] }
-pretty_env_logger = "0.5.0"
-log = "0.4.21"
-tokio = { version = "1.37.0", default-features = false, features = ["macros", "rt-multi-thread"] }
-tokio-util = "0.7.10"
-heck = "0.5.0"
-semver = "1.0.22"
-serde = { version = "1.0.197", features = ["derive"] }
-serde_json = "1.0.115"
-indexmap = "2.2.6"
-url = { version = "2.5.0", features = ["serde"] }
-wit-parser = "0.208.1"
-wit-component = "0.208.1"
-wasm-metadata = "0.208.1"
-parse_arg = "0.1.4"
+assert_cmd = "2.0.14"
+bytes = "1.6.0"
 cargo_metadata = "0.18.1"
+cargo-component-core = { path = "crates/core", version = "0.15.0-dev" }
 cargo-config2 = "0.1.24"
+clap = { version = "4.5.4", features = ["derive", "env"] }
+dirs = "5"
+futures = "0.3.30"
+heck = "0.5.0"
+indexmap = "2.2.6"
 libc = "0.2.153"
+log = "0.4.21"
+oci-distribution = "0.11"
 owo-colors = "4.0.0"
-unicode-width = "0.1.11"
 p256 = "0.13.2"
+parse_arg = "0.1.4"
+predicates = "3.1.0"
+pretty_env_logger = "0.5.0"
 rand_core = "0.6.4"
 rpassword = "7.3.1"
-futures = "0.3.30"
-bytes = "1.6.0"
-which = "6.0.1"
-wit-bindgen-rust = "0.25.0"
-wit-bindgen-core = "0.25.0"
+semver = "1"
+serde = { version = "1.0.197", features = ["derive"] }
+serde_json = "1.0.115"
 tempfile = "3.10.1"
-assert_cmd = "2.0.14"
-predicates = "3.1.0"
+tokio = { version = "1.37.0", default-features = false, features = [
+    "macros",
+    "rt-multi-thread",
+] }
+tokio-util = "0.7.10"
+toml_edit = { version = "0.22.9", features = ["serde"] }
+unicode-width = "0.1.11"
+url = { version = "2.5.0", features = ["serde"] }
+warg-client = "0.7.0"
+warg-crypto = "0.7.0"
+warg-protocol = "0.7.0"
+warg-server = "0.7.0"
+wasi-preview1-component-adapter-provider = "23.0.1"
+wasm-metadata = "0.208.1"
+wasm-pkg-client = { git = "https://github.com/bytecodealliance/wasm-pkg-tools.git", branch = "main" }
 wasmparser = "0.208.1"
-wat = "1.208.1"
 wasmprinter = "0.208.1"
+wat = "1.208.1"
+which = "6.0.1"
+wit-bindgen-core = "0.25.0"
+wit-bindgen-rust = "0.25.0"
+wit-component = "0.208.1"
+wit-parser = "0.208.1"
 
 [profile.release]
 panic = "abort"

diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml
@@ -11,23 +11,26 @@ repository = { workspace = true }
 
 [dependencies]
 anyhow = { workspace = true }
+clap = { workspace = true }
+dirs = { workspace = true }
+futures = { workspace = true }
+indexmap = { workspace = true }
 libc = { workspace = true }
+log = { workspace = true }
 owo-colors = { workspace = true }
-unicode-width = { workspace = true }
-warg-crypto = { workspace = true }
-warg-protocol = { workspace = true }
-warg-client = { workspace = true }
-toml_edit = { workspace = true }
 semver = { workspace = true }
 serde = { workspace = true }
-indexmap = { workspace = true }
-futures = { workspace = true }
+tokio = { workspace = true }
+tokio-util = { workspace = true, features = ["io"] }
+toml_edit = { workspace = true }
+unicode-width = { workspace = true }
 url = { workspace = true }
+warg-client = { workspace = true }
+warg-crypto = { workspace = true }
+warg-protocol = { workspace = true }
+wasm-pkg-client = { workspace = true }
 wit-component = { workspace = true }
 wit-parser = { workspace = true }
-log = { workspace = true }
-tokio = { workspace = true }
-clap = { workspace = true }
 
 [target.'cfg(windows)'.dependencies.windows-sys]
 version = "0.52"

diff --git a/crates/core/src/command.rs b/crates/core/src/command.rs
@@ -1,8 +1,15 @@
 //! Module for common command implementation.
+use std::path::PathBuf;
 
-use crate::terminal::{Color, Terminal, Verbosity};
 use clap::{ArgAction, Args};
 
+use crate::terminal::{Color, Terminal, Verbosity};
+
+/// The environment variable name for setting a cache directory location
+pub const CACHE_DIR_ENV_VAR: &str = "CARGO_COMPONENT_CACHE_DIR";
+/// The environment variable name for setting a path to a config file
+pub const CONFIG_FILE_ENV_VAR: &str = "CARGO_COMPONENT_CONFIG_FILE";
+
 /// Common options for commands.
 #[derive(Args)]
 #[command(
@@ -24,6 +31,14 @@ pub struct CommonOptions {
     /// Coloring: auto, always, never
     #[clap(long = "color", value_name = "WHEN")]
     pub color: Option<Color>,
+
+    /// The path to the cache directory to store component dependencies.
+    #[clap(long = "cache-dir", env = CACHE_DIR_ENV_VAR)]
+    pub cache_dir: Option<PathBuf>,
+
+    /// The path to the pkg-tools config file
+    #[clap(long = "config", env = CONFIG_FILE_ENV_VAR)]
+    pub config: Option<PathBuf>,
 }
 
 impl CommonOptions {

diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs
@@ -2,22 +2,44 @@
 
 #![deny(missing_docs)]
 
+use std::path::PathBuf;
+use std::str::FromStr;
+
 use anyhow::Context;
 use semver::VersionReq;
-use std::str::FromStr;
-use warg_protocol::registry::PackageName;
+use wasm_pkg_client::PackageRef;
 
 pub mod command;
 pub mod lock;
 pub mod progress;
 pub mod registry;
 pub mod terminal;
 
+/// The root directory name used for default cargo component directories
+pub const CARGO_COMPONENT_DIR: &str = "cargo-component";
+/// The cache directory name used by default
+pub const CACHE_DIR: &str = "cache";
+
+/// Returns the path to the default cache directory, returning an error if a cache directory cannot be found.
+pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
+    dirs::cache_dir()
+        .map(|p| p.join(CARGO_COMPONENT_DIR).join(CACHE_DIR))
+        .ok_or_else(|| anyhow::anyhow!("failed to find cache directory"))
+}
+
+/// A helper that fetches the default directory if the given directory is `None`.
+pub fn cache_dir(dir: Option<PathBuf>) -> anyhow::Result<PathBuf> {
+    match dir {
+        Some(dir) => Ok(dir),
+        None => default_cache_dir(),
+    }
+}
+
 /// Represents a versioned component package name.
 #[derive(Clone)]
 pub struct VersionedPackageName {
     /// The package name.
-    pub name: PackageName,
+    pub name: PackageRef,
     /// The optional package version.
     pub version: Option<VersionReq>,
 }

diff --git a/crates/core/src/lock.rs b/crates/core/src/lock.rs
@@ -10,8 +10,7 @@ use std::{
     path::{Path, PathBuf},
 };
 use toml_edit::{DocumentMut, Item, Value};
-use warg_crypto::hash::AnyHash;
-use warg_protocol::registry::PackageName;
+use wasm_pkg_client::{ContentDigest, PackageRef};
 
 /// The file format version of the lock file.
 const LOCK_FILE_VERSION: i64 = 1;
@@ -21,7 +20,7 @@ const LOCK_FILE_VERSION: i64 = 1;
 #[serde(rename_all = "kebab-case")]
 pub struct LockedPackage {
     /// The name of the locked package.
-    pub name: PackageName,
+    pub name: PackageRef,
     /// The registry the package was resolved from.
     ///
     /// Defaults to the default registry if not specified.
@@ -37,9 +36,10 @@ pub struct LockedPackage {
 
 impl LockedPackage {
     /// Gets the key used in sorting and searching the package list.
-    pub fn key(&self) -> (&PackageName, &str) {
+    pub fn key(&self) -> (&str, &str, &str) {
         (
-            &self.name,
+            self.name.namespace().as_ref(),
+            self.name.name().as_ref(),
             self.registry.as_deref().unwrap_or(DEFAULT_REGISTRY_NAME),
         )
     }
@@ -53,7 +53,7 @@ pub struct LockedPackageVersion {
     /// The version the package is locked to.
     pub version: Version,
     /// The digest of the package contents.
-    pub digest: AnyHash,
+    pub digest: ContentDigest,
 }
 
 impl LockedPackageVersion {
@@ -81,13 +81,20 @@ impl<'a> LockFileResolver<'a> {
     pub fn resolve(
         &'a self,
         registry: &str,
-        name: &PackageName,
+        package_ref: &PackageRef,
         requirement: &VersionReq,
     ) -> Result<Option<&'a LockedPackageVersion>> {
         if let Some(pkg) = self
             .0
             .packages
-            .binary_search_by_key(&(name, registry), LockedPackage::key)
+            .binary_search_by_key(
+                &(
+                    package_ref.namespace().as_ref(),
+                    package_ref.name().as_ref(),
+                    registry,
+                ),
+                LockedPackage::key,
+            )
             .ok()
             .map(|i| &self.0.packages[i])
         {
@@ -96,12 +103,12 @@ impl<'a> LockFileResolver<'a> {
                 .binary_search_by_key(&requirement.to_string().as_str(), LockedPackageVersion::key)
             {
                 let locked = &pkg.versions[index];
-                log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
+                log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was resolved by the lock file to version {version}", version = locked.version);
                 return Ok(Some(locked));
             }
         }
 
-        log::info!("dependency package `{name}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
+        log::info!("dependency package `{package_ref}` from registry `{registry}` with requirement `{requirement}` was not in the lock file");
         Ok(None)
     }
 }