Merge #39

39: Cache installation tarballs r=Hoverbear a=Hoverbear

Adopts a strategy of caching installed packages, manifests, and keys to the decided OS cache directory and then using those if present by way of a `DownloadServerCache` which wraps a `DownloadServerClient` and provides a very similar albeit restricted API.

Alters `criticalup clean` to appropriately clean the cache directory.

Adds an `--offline` flag to `criticalup install`.

Co-authored-by: Ana Hobden <[email protected]>
Co-authored-by: Ana Hobden <[email protected]>

CHANGELOG.md

@@ -9,7 +9,12 @@ All notable changes to this project will be documented in this file.
 
 ## Added
 
-- Added `tracing` for structure and multi-level logging. `--verbose` and `-v` are now
+- An `--offline` flag has been added to `criticalup install`, when enabled only the download cache
+  will be used where possible, and the cache will not be populated on cache miss.
+- Caching of downloaded keys, manifests, and installation tarballs has been added. Newly downloaded
+  artifacts will also be stored in the OS-specific cache directory. The cache can be cleaned with
+  `criticalup clean` or any relevant OS behaviors.
+- `tracing` support was added for structured and multi-level logging. `--verbose` and `-v` are now
   generally accepted and enable debug logging. Passing the flag twice (eg. `-vv`) will enable
   trace logging as well. The `--log-level` argument can accept arbitrary
   [tracing directives](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html#directives)

crates/criticaltrust/src/manifests.rs

@@ -102,7 +102,7 @@ pub struct ReleaseArtifact {
     pub sha256: Vec<u8>,
 }
 
-#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Copy)]
 pub enum ReleaseArtifactFormat {
     #[serde(rename = "tar.zst")]
     TarZst,

crates/criticaltrust/src/signatures/keychain.rs

@@ -4,9 +4,11 @@
 use crate::keys::{KeyId, KeyRole, PublicKey};
 use crate::signatures::{PublicKeysRepository, SignedPayload};
 use crate::Error;
+use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 
 /// Collection of all trusted public keys.
+#[derive(Serialize, Deserialize)]
 pub struct Keychain {
     keys: HashMap<KeyId, PublicKey>,
 }

crates/criticalup-cli/src/commands/clean.rs

@@ -14,13 +14,22 @@ pub(crate) async fn run(ctx: &Context) -> Result<(), Error> {
     let installations_dir = &ctx.config.paths.installation_dir;
     let state = State::load(&ctx.config).await?;
 
+    delete_cache_directory(&ctx.config.paths.cache_dir).await?;
     delete_unused_installations(installations_dir, &state).await?;
     delete_untracked_installation_dirs(installations_dir, state).await?;
 
     Ok(())
 }
 
-/// Deletes installation from `State` wl; ith `InstallationId`s that have empty manifest section, and
+async fn delete_cache_directory(cache_dir: &Path) -> Result<(), Error> {
+    if cache_dir.exists() {
+        tracing::info!("Cleaning cache directory");
+        tokio::fs::remove_dir_all(&cache_dir).await?;
+    }
+    Ok(())
+}
+
+/// Deletes installation from `State` with `InstallationId`s that have empty manifest section, and
 /// deletes the installation directory from the disk if present.
 async fn delete_unused_installations(installations_dir: &Path, state: &State) -> Result<(), Error> {
     let unused_installations: Vec<InstallationId> = state

crates/criticalup-cli/src/commands/install.rs

@@ -5,9 +5,11 @@ use std::path::{Path, PathBuf};
 
 use criticaltrust::integrity::IntegrityVerifier;
 use criticaltrust::manifests::{Release, ReleaseArtifactFormat};
+use criticalup_core::download_server_cache::DownloadServerCache;
 use criticalup_core::download_server_client::DownloadServerClient;
 use criticalup_core::project_manifest::{ProjectManifest, ProjectManifestProduct};
 use criticalup_core::state::State;
+use tokio::fs::read;
 
 use crate::errors::Error;
 use crate::errors::Error::{IntegrityErrorsWhileInstallation, PackageDependenciesNotSupported};
@@ -18,12 +20,19 @@ pub const DEFAULT_RELEASE_ARTIFACT_FORMAT: ReleaseArtifactFormat = ReleaseArtifa
 pub(crate) async fn run(
     ctx: &Context,
     reinstall: bool,
+    offline: bool,
     project: Option<PathBuf>,
 ) -> Result<(), Error> {
     // TODO: If `std::io::stdout().is_terminal() == true``, provide a nice, fancy progress bar using indicatif.
     //       Retain existing behavior to support non-TTY usage.
 
     let state = State::load(&ctx.config).await?;
+    let maybe_client = if !offline {
+        Some(DownloadServerClient::new(&ctx.config, &state))
+    } else {
+        None
+    };
+    let cache = DownloadServerCache::new(&ctx.config.paths.cache_dir, &maybe_client).await?;
 
     // Get manifest location if arg `project` is None
     let manifest_path = ProjectManifest::discover_canonical_path(project.as_deref()).await?;
@@ -37,7 +46,7 @@ pub(crate) async fn run(
         let abs_installation_dir_path = installation_dir.join(product.installation_id());
 
         if !abs_installation_dir_path.exists() {
-            install_product_afresh(ctx, &state, &manifest_path, product).await?;
+            install_product_afresh(ctx, &state, &cache, &manifest_path, product).await?;
         } else {
             // Check if the state file has no mention of this installation.
             let does_this_installation_exist_in_state = state
@@ -46,7 +55,7 @@ pub(crate) async fn run(
             if !does_this_installation_exist_in_state || reinstall {
                 // If the installation directory exists, but the State has no installation of that
                 // InstallationId, then re-run the install command and go through installation.
-                install_product_afresh(ctx, &state, &manifest_path, product).await?;
+                install_product_afresh(ctx, &state, &cache, &manifest_path, product).await?;
             } else {
                 // If the installation directory exists AND there is an existing installation with
                 // that InstallationId, then merely update the installation in the State file to
@@ -67,27 +76,32 @@ pub(crate) async fn run(
     Ok(())
 }
 
+#[tracing::instrument(level = "debug", skip_all, fields(
+    manifest_path = %manifest_path.display(),
+    installation_id = %product.installation_id(),
+    release = %product.release(),
+    product = %product.name(),
+))]
 async fn install_product_afresh(
     ctx: &Context,
     state: &State,
+    cache: &DownloadServerCache<'_>,
     manifest_path: &Path,
     product: &ProjectManifestProduct,
 ) -> Result<(), Error> {
     let product_name = product.name();
     let release = product.release();
     let installation_dir = &ctx.config.paths.installation_dir;
     let abs_installation_dir_path = installation_dir.join(product.installation_id());
-    let client = DownloadServerClient::new(&ctx.config, state);
-    let keys = client.get_keys().await?;
+    let keys = cache.keys().await?;
 
-    // TODO: Add tracing to support log levels, structured logging.
     tracing::info!("Installing product '{product_name}' ({release})",);
 
     let mut integrity_verifier = IntegrityVerifier::new(&keys);
 
     // Get the release manifest for the product from the server and verify it.
-    let release_manifest_from_server = client
-        .get_product_release_manifest(product_name, product.release())
+    let release_manifest_from_server = cache
+        .product_release_manifest(product_name, product.release())
         .await?;
     let verified_release_manifest = release_manifest_from_server.signed.into_verified(&keys)?;
 
@@ -103,29 +117,19 @@ async fn install_product_afresh(
         .await?;
 
     for package in product.packages() {
-        tracing::info!("Downloading component '{package}' for '{product_name}' ({release})",);
-
-        let response_file = client
-            .download_package(
+        let package_path = cache
+            .package(
                 product_name,
                 release_name,
                 package,
                 DEFAULT_RELEASE_ARTIFACT_FORMAT,
             )
             .await?;
 
-        // Archive file path, path with the archive extension.
-        let package_name_with_extension =
-            format!("{}.{}", package, DEFAULT_RELEASE_ARTIFACT_FORMAT);
-        let abs_artifact_compressed_file_path: PathBuf =
-            abs_installation_dir_path.join(&package_name_with_extension);
-
-        // Save the downloaded package archive on disk.
-        tokio::fs::write(&abs_artifact_compressed_file_path, response_file.clone()).await?;
-
         tracing::info!("Installing component '{package}' for '{product_name}' ({release})",);
+        let package_data = read(package_path).await?;
 
-        let decoder = xz2::read::XzDecoder::new(response_file.as_slice());
+        let decoder = xz2::read::XzDecoder::new(package_data.as_slice());
         let mut archive = tar::Archive::new(decoder);
         archive.set_preserve_permissions(true);
         archive.set_preserve_mtime(true);
@@ -147,8 +151,6 @@ async fn install_product_afresh(
                 );
             }
         }
-
-        clean_archive_download(&abs_artifact_compressed_file_path).await?;
     }
 
     let verified_packages = integrity_verifier
@@ -173,11 +175,6 @@ fn check_for_package_dependencies(verified_release_manifest: &Release) -> Result
     Ok(())
 }
 
-async fn clean_archive_download(abs_artifact_compressed_file_path: &PathBuf) -> Result<(), Error> {
-    tokio::fs::remove_file(abs_artifact_compressed_file_path).await?;
-    Ok(())
-}
-
 #[test]
 fn dependencies_check() {
     use criticaltrust::manifests::ReleasePackage;

crates/criticalup-cli/src/lib.rs

@@ -55,9 +55,11 @@ async fn main_inner(whitelabel: WhitelabelConfig, args: &[OsString]) -> Result<(
             Some(AuthCommands::Remove) => commands::auth_remove::run(&ctx).await?,
             None => commands::auth::run(&ctx).await?,
         },
-        Commands::Install { project, reinstall } => {
-            commands::install::run(&ctx, reinstall, project).await?
-        }
+        Commands::Install {
+            project,
+            reinstall,
+            offline,
+        } => commands::install::run(&ctx, reinstall, offline, project).await?,
         Commands::Clean => commands::clean::run(&ctx).await?,
         Commands::Remove { project } => commands::remove::run(&ctx, project).await?,
         Commands::Run { command, project } => commands::run::run(&ctx, command, project).await?,
@@ -136,6 +138,9 @@ enum Commands {
         /// Reinstall products that may have already been installed
         #[arg(long)]
         reinstall: bool,
+        /// Don't download from the server, only use previously cached artifacts
+        #[arg(long)]
+        offline: bool,
     },
 
     /// Delete all unused and untracked installations

crates/criticalup-cli/tests/snapshots/cli__install__help_message.snap

@@ -16,6 +16,7 @@ Usage:
 Options:
       --project <PROJECT>           Path to the manifest `criticalup.toml`
       --reinstall                   Reinstall products that may have already been installed
+      --offline                     Don't download from the server, only use previously cached artifacts
   -v, --verbose...                  Enable debug logs, -vv for trace
       --log-level [<LOG_LEVEL>...]  Tracing directives
   -h, --help                        Print help

crates/criticalup-core/Cargo.toml

@@ -21,6 +21,7 @@ dirs = { version = "5.0.1", default-features = false }
 tokio.workspace = true
 reqwest-middleware.workspace = true
 reqwest-retry.workspace = true
+tracing.workspace = true
 
 [dev-dependencies]
 mock-download-server = { path = "../mock-download-server" }

crates/criticalup-core/src/config/mod.rs

@@ -21,20 +21,24 @@ pub struct Config {
 impl Config {
     /// Detect and load the criticalup configuration from the execution environment.
     pub fn detect(whitelabel: WhitelabelConfig) -> Result<Self, Error> {
-        Self::detect_inner(whitelabel, None)
+        Self::detect_inner(whitelabel, None, None)
     }
 
     fn detect_inner(
         whitelabel: WhitelabelConfig,
         root: Option<std::path::PathBuf>,
+        cache_dir: Option<std::path::PathBuf>,
     ) -> Result<Self, Error> {
-        let paths = Paths::detect(&whitelabel, root)?;
+        let paths = Paths::detect(&whitelabel, root, cache_dir)?;
         Ok(Self { whitelabel, paths })
     }
 
     #[cfg(test)]
-    pub(crate) fn test(root: std::path::PathBuf) -> Result<Self, Error> {
-        Self::detect_inner(WhitelabelConfig::test(), Some(root))
+    pub(crate) fn test(
+        root: std::path::PathBuf,
+        cache_dir: std::path::PathBuf,
+    ) -> Result<Self, Error> {
+        Self::detect_inner(WhitelabelConfig::test(), Some(root), Some(cache_dir))
     }
 }
 

crates/criticalup-core/src/config/paths.rs

@@ -14,6 +14,7 @@ pub struct Paths {
 
     pub proxies_dir: PathBuf,
     pub installation_dir: PathBuf,
+    pub cache_dir: PathBuf,
 
     #[cfg(test)]
     pub(crate) root: PathBuf,
@@ -23,6 +24,7 @@ impl Paths {
     pub(super) fn detect(
         whitelabel: &WhitelabelConfig,
         root: Option<std::path::PathBuf>,
+        cache_dir: Option<std::path::PathBuf>,
     ) -> Result<Paths, Error> {
         let root = if let Some(root) = root {
             if root != Path::new("") {
@@ -34,10 +36,18 @@ impl Paths {
             find_root(whitelabel).ok_or(Error::CouldNotDetectRootDirectory)?
         };
 
+        let cache_dir = match cache_dir {
+            Some(cache_dir) => cache_dir,
+            None => {
+                find_cache_dir(whitelabel).ok_or_else(|| Error::CouldNotDetectCacheDirectory)?
+            }
+        };
+
         Ok(Paths {
             state_file: root.join("state.json"),
             proxies_dir: root.join("bin"),
             installation_dir: root.join(DEFAULT_INSTALLATION_DIR_NAME),
+            cache_dir,
             #[cfg(test)]
             root,
         })
@@ -56,6 +66,18 @@ fn platform_specific_root(whitelabel: &WhitelabelConfig) -> Option<PathBuf> {
     dirs::data_dir().map(|v| v.join(whitelabel.name))
 }
 
+fn find_cache_dir(whitelabel: &WhitelabelConfig) -> Option<PathBuf> {
+    match env::var_os("CRITICALUP_CACHE_DIR") {
+        Some(val) if val.is_empty() => platform_specific_cache_dir(whitelabel),
+        Some(val) => Some(PathBuf::from(val)),
+        None => platform_specific_cache_dir(whitelabel),
+    }
+}
+
+fn platform_specific_cache_dir(whitelabel: &WhitelabelConfig) -> Option<PathBuf> {
+    dirs::cache_dir().map(|v| v.join(whitelabel.name))
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -72,9 +94,15 @@ mod tests {
                 state_file: "/opt/criticalup/state.json".into(),
                 proxies_dir: "/opt/criticalup/bin".into(),
                 installation_dir: "/opt/criticalup/toolchains".into(),
+                cache_dir: "/cache/criticalup".into(),
                 root: "/opt/criticalup".into()
             },
-            Paths::detect(&WhitelabelConfig::test(), Some("/opt/criticalup".into()),).unwrap()
+            Paths::detect(
+                &WhitelabelConfig::test(),
+                Some("/opt/criticalup".into()),
+                Some("/cache/criticalup".into())
+            )
+            .unwrap()
         );
     }
 
@@ -159,7 +187,7 @@ mod tests {
     ) {
         assert_eq!(
             expected.as_ref(),
-            Paths::detect(whitelabel, root).unwrap().root
+            Paths::detect(whitelabel, root, None).unwrap().root
         );
     }
 
@@ -168,7 +196,7 @@ mod tests {
         whitelabel: &WhitelabelConfig,
         root: Option<PathBuf>,
     ) {
-        match Paths::detect(whitelabel, root) {
+        match Paths::detect(whitelabel, root, None) {
             Ok(paths) => assert_ne!(expected.as_ref(), paths.root),
             Err(err) => assert!(matches!(err, Error::CouldNotDetectRootDirectory)),
         }