From c277a5580add55b51fb8f6e64fff05b34308ab97 Mon Sep 17 00:00:00 2001 From: Zac Mrowicki Date: Thu, 12 Aug 2021 21:08:26 +0000 Subject: [PATCH] tuftool: Add `clone` subcommand This adds a `clone` subcommand to `tuftool`, allowing a user to download a fully functioning TUF repository. A user has the option to download a full repository, a subset of the targets, or just metadata. --- tuftool/src/clone.rs | 144 +++++++++++++++++++++++++ tuftool/src/error.rs | 6 ++ tuftool/src/main.rs | 6 +- tuftool/tests/clone_command.rs | 190 +++++++++++++++++++++++++++++++++ 4 files changed, 345 insertions(+), 1 deletion(-) create mode 100644 tuftool/src/clone.rs create mode 100644 tuftool/tests/clone_command.rs diff --git a/tuftool/src/clone.rs b/tuftool/src/clone.rs new file mode 100644 index 000000000..d5ae48672 --- /dev/null +++ b/tuftool/src/clone.rs @@ -0,0 +1,144 @@ +// Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT OR Apache-2.0 + +use crate::common::UNUSED_URL; +use crate::download_root::download_root; +use crate::error::{self, Result}; +use snafu::ResultExt; +use std::fs::File; +use std::num::NonZeroU64; +use std::path::{Path, PathBuf}; +use structopt::StructOpt; +use tough::{ExpirationEnforcement, RepositoryLoader}; +use url::Url; + +#[derive(Debug, StructOpt)] +pub(crate) struct CloneArgs { + /// Path to root.json file for the repository + #[structopt(short = "r", long = "root")] + root: Option, + + /// Remote root.json version number + #[structopt(short = "v", long = "root-version", default_value = "1")] + root_version: NonZeroU64, + + /// TUF repository metadata base URL + #[structopt(short = "m", long = "metadata-url")] + metadata_base_url: Url, + + /// TUF repository targets base URL + #[structopt(short = "t", long = "targets-url", required_unless = "metadata-only")] + targets_base_url: Option, + + /// Allow downloading the root.json file (unsafe) + #[structopt(long)] + allow_root_download: bool, + + /// Allow repo download for expired metadata + #[structopt(long)] + allow_expired_repo: bool, + + /// Download only these targets, if specified + #[structopt(short = "n", long = "target-names", conflicts_with = "metadata-only")] + target_names: Vec, + + /// Output directory of targets + #[structopt(long, required_unless = "metadata-only")] + targets_dir: Option, + + /// Output directory of metadata + #[structopt(long)] + metadata_dir: PathBuf, + + /// Only download the repository metadata, not the targets + #[structopt(long)] + metadata_only: bool, +} + +fn expired_repo_warning>(path: P) { + #[rustfmt::skip] + eprintln!("\ +================================================================= +Downloading repo to {} +WARNING: `--allow-expired-repo` was passed; this is unsafe and will not establish trust, use only for testing! +=================================================================", + path.as_ref().display()); +} + +impl CloneArgs { + pub(crate) fn run(&self) -> Result<()> { + // Use local root.json or download from repository + let root_path = if let Some(path) = &self.root { + PathBuf::from(path) + } else if self.allow_root_download { + let outdir = std::env::current_dir().context(error::CurrentDir)?; + download_root(&self.metadata_base_url, self.root_version, outdir)? + } else { + eprintln!("No root.json available"); + std::process::exit(1); + }; + + // Structopt won't allow `targets_base_url` to be None when it is required. We require the + // user to supply `targets_base_url` in the case they actually plan to download targets. + // When downloading metadata, we don't ever need to access the targets URL, so we use a + // fake URL to satisfy the library. + let targets_base_url = self + .targets_base_url + .as_ref() + .unwrap_or(&Url::parse(UNUSED_URL).context(error::UrlParse { + url: UNUSED_URL.to_owned(), + })?) + .clone(); + + // Load repository + let expiration_enforcement = if self.allow_expired_repo { + expired_repo_warning(&self.metadata_dir); + ExpirationEnforcement::Unsafe + } else { + ExpirationEnforcement::Safe + }; + let repository = RepositoryLoader::new( + File::open(&root_path).context(error::OpenRoot { path: &root_path })?, + self.metadata_base_url.clone(), + targets_base_url, + ) + .expiration_enforcement(expiration_enforcement) + .load() + .context(error::RepoLoad)?; + + // Clone the repository, downloading none, all, or a subset of targets + let clone_result = if self.metadata_only { + println!("Cloning repository metadata to {:?}", self.metadata_dir); + repository.cache_metadata(&self.metadata_dir, true) + } else { + // Similar to `targets_base_url, structopt's guard rails won't let us have a + // `targets_dir` that is None when the argument is required. We only require the user + // to supply a targets directory if they actually plan on downloading targets. + let targets_dir = self.targets_dir.as_ref().expect( + "Developer error: `targets_dir` is required unless downloading metadata only", + ); + + if self.target_names.is_empty() { + println!( + "Cloning repository:\n\tmetadata location: {:?}\n\ttargets location: {:?}", + self.metadata_dir, targets_dir + ); + // The None turbofish is required to satisfy the compiler, sorry. + repository.cache(&self.metadata_dir, &targets_dir, None::<&[&str]>, true) + } else { + println!( + "Cloning repository with a subset of targets:\n\tmetadata location:{:?}\n\ttargets location: {:?}", + self.metadata_dir, targets_dir + ); + repository.cache( + &self.metadata_dir, + &targets_dir, + Some(self.target_names.as_slice()), + true, + ) + } + }; + + clone_result.context(error::CloneRepository) + } +} diff --git a/tuftool/src/error.rs b/tuftool/src/error.rs index 57f8f5bfb..fab14c34f 100644 --- a/tuftool/src/error.rs +++ b/tuftool/src/error.rs @@ -22,6 +22,12 @@ fn get_status_code(source: &reqwest::Error) -> String { #[derive(Debug, Snafu)] #[snafu(visibility = "pub(crate)")] pub(crate) enum Error { + #[snafu(display("Failed to clone repository: {}", source))] + CloneRepository { + source: tough::error::Error, + backtrace: Backtrace, + }, + #[snafu(display("Failed to run {}: {}", command_str, source))] CommandExec { command_str: String, diff --git a/tuftool/src/main.rs b/tuftool/src/main.rs index d54a76311..f432bab76 100644 --- a/tuftool/src/main.rs +++ b/tuftool/src/main.rs @@ -13,6 +13,7 @@ mod add_key_role; mod add_role; +mod clone; mod common; mod create; mod create_role; @@ -77,7 +78,7 @@ impl Program { enum Command { /// Create a TUF repository Create(create::CreateArgs), - /// Download a TUF repository's resources + /// Download a TUF repository's targets Download(download::DownloadArgs), /// Update a TUF repository's metadata and optionally add targets Update(Box), @@ -85,6 +86,8 @@ enum Command { Root(root::Command), /// Delegation Commands Delegation(Delegation), + /// Clone a fully functional TUF repository + Clone(clone::CloneArgs), } impl Command { @@ -95,6 +98,7 @@ impl Command { Command::Download(args) => args.run(), Command::Update(args) => args.run(), Command::Delegation(cmd) => cmd.run(), + Command::Clone(cmd) => cmd.run(), } } } diff --git a/tuftool/tests/clone_command.rs b/tuftool/tests/clone_command.rs new file mode 100644 index 000000000..5c5e838d5 --- /dev/null +++ b/tuftool/tests/clone_command.rs @@ -0,0 +1,190 @@ +mod test_utils; + +use assert_cmd::Command; +use std::fs::read_to_string; +use std::path::PathBuf; +use tempfile::TempDir; +use test_utils::{dir_url, test_data}; +use url::Url; + +struct RepoPaths { + root_path: PathBuf, + metadata_base_url: Url, + targets_base_url: Url, + metadata_outdir: TempDir, + targets_outdir: TempDir, +} + +impl RepoPaths { + fn new() -> Self { + let base = test_data().join("tuf-reference-impl"); + RepoPaths { + root_path: base.join("metadata").join("1.root.json"), + metadata_base_url: dir_url(base.join("metadata")), + targets_base_url: dir_url(base.join("targets")), + metadata_outdir: TempDir::new().unwrap(), + targets_outdir: TempDir::new().unwrap(), + } + } +} + +enum FileType { + Metadata, + Target, +} + +/// Asserts that a target file is identical to the TUF reference example +fn assert_target_match(indir: &TempDir, filename: &str) { + assert_reference_file_match(indir, filename, FileType::Target) +} + +/// Asserts that a metadata file is identical to the TUF reference example +fn assert_metadata_match(indir: &TempDir, filename: &str) { + assert_reference_file_match(indir, filename, FileType::Metadata) +} + +/// Asserts that the named file in `indir` exactly matches the file in `tuf-reference-impl/` +fn assert_reference_file_match(indir: &TempDir, filename: &str, filetype: FileType) { + let got = read_to_string(indir.path().join(filename)).unwrap(); + + let ref_dir = match filetype { + FileType::Metadata => "metadata", + FileType::Target => "targets", + }; + let reference = read_to_string( + test_utils::test_data() + .join("tuf-reference-impl") + .join(ref_dir) + .join(filename), + ) + .unwrap(); + + assert_eq!(got, reference, "{} contents do not match.", filename); +} + +/// Asserts that all metadata files that should exist do and are identical to the reference example +fn assert_all_metadata(metadata_dir: &TempDir) { + for f in &[ + "snapshot.json", + "targets.json", + "timestamp.json", + "1.root.json", + "role1.json", + "role2.json", + ] { + assert_metadata_match(&metadata_dir, f) + } +} + +/// Given a `Command`, attach all the base args necessary for the `clone` subcommand +fn clone_base_command<'a>(cmd: &'a mut Command, repo_paths: &RepoPaths) -> &'a mut Command { + cmd.args(&[ + "clone", + "--root", + repo_paths.root_path.to_str().unwrap(), + "--metadata-url", + repo_paths.metadata_base_url.as_str(), + "--metadata-dir", + repo_paths.metadata_outdir.path().to_str().unwrap(), + ]) +} + +#[test] +// Ensure that we successfully clone all metadata +fn clone_metadata() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&["--metadata-only"]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir) +} + +#[test] +// Ensure that the `--target-names` argument collides with the `--metadata-only` argument +fn clone_metadata_target_names_failure() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&["--metadata-only", "--target-names", "foo"]) + .assert() + .failure(); +} + +#[test] +// Ensure that, even when provided with arguments for targets, the `--metadata-only` argument +// _only_ downloads metadata files +fn clone_metadata_target_args() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--metadata-only", + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + ]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir); + assert!(repo_paths + .targets_outdir + .path() + .read_dir() + .unwrap() + .next() + .is_none()) +} + +#[test] +// Ensure we can clone a subset of targets +fn clone_subset_targets() { + let target_name = "file1.txt"; + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + "--target-names", + target_name, + ]) + .assert() + .success(); + + assert_all_metadata(&repo_paths.metadata_outdir); + assert_target_match(&repo_paths.targets_outdir, target_name); + + assert_eq!( + repo_paths.targets_outdir.path().read_dir().unwrap().count(), + 1 + ); +} + +#[test] +// Ensure we can clone an entire repo +fn clone_full_repo() { + let repo_paths = RepoPaths::new(); + let mut cmd = Command::cargo_bin("tuftool").unwrap(); + clone_base_command(&mut cmd, &repo_paths) + .args(&[ + "--targets-url", + repo_paths.targets_base_url.as_str(), + "--targets-dir", + repo_paths.targets_outdir.path().to_str().unwrap(), + ]) + .assert() + .success(); + + assert_all_metadata(&&repo_paths.metadata_outdir); + + for f in &["file1.txt", "file2.txt", "file3.txt"] { + assert_target_match(&repo_paths.targets_outdir, f) + } +}