diff --git a/.github/scripts/download_kernel_images.sh b/.github/scripts/download_kernel_images.sh index 5d6351d6a..91e8c576f 100755 --- a/.github/scripts/download_kernel_images.sh +++ b/.github/scripts/download_kernel_images.sh @@ -29,5 +29,30 @@ for VERSION in "${VERSIONS[@]}"; do )" done -printf '%s\n' "${FILES[@]}" \ -| xargs -t curl -sfSL --create-dirs --output-dir "$OUTPUT_DIR" --parallel --remote-name-all +# TODO(https://github.com/curl/curl/issues/15729): restore --parallel here if and when it properly +# supports ETags. +# TODO(https://github.com/curl/curl/issues/15730): restore --create-dirs when it works in the +# presence of `--etag-save`.` +# +# Note: `--etag-{compare,save}` are not idempotent until curl 8.9.0 which included +# https://github.com/curl/curl/commit/85efbb92b8e6679705e122cee45ce76c56414a3e. At the time of +# writing our CI uses Ubuntu 22.04 which has curl 7.81.0 and the latest available is Ubuntu 24.04 +# which has curl 8.5.0. Since neither has a new enough curl, we don't bother to update, but we +# should do so when Ubuntu 24.10 or later is available. +mkdir -p "$OUTPUT_DIR" +KEEP=() +for FILE in "${FILES[@]}"; do + name=$(basename "$FILE") + etag_name="$name.etag" + KEEP+=("$name" "$etag_name") + + etag="$OUTPUT_DIR/$etag_name" + curl -sfSL --output-dir "$OUTPUT_DIR" --remote-name-all --etag-compare "$etag" --etag-save "$etag" "$FILE" +done + +# Remove any files that were previously downloaded that are no longer needed. +FIND_ARGS=() +for FILE in "${KEEP[@]}"; do + FIND_ARGS+=("!" "-name" "$FILE") +done +find "$OUTPUT_DIR" -type f "${FIND_ARGS[@]}" -exec rm {} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4d4d6a24..3f8639005 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,9 @@ jobs: - name: Install prerequisites if: runner.os == 'macOS' - # The xargs shipped on macOS always exits 0 with -P0, so we need GNU findutils. + # The curl shipped on macOS doesn't contain + # https://github.com/curl/curl/commit/85efbb92b8e6679705e122cee45ce76c56414a3e which is + # needed for proper handling of `--etag-{compare,save}`. # # The tar shipped on macOS doesn't support --wildcards, so we need GNU tar. # @@ -237,9 +239,10 @@ jobs: # https://github.com/actions/setup-python/issues/577 find /usr/local/bin -type l -exec sh -c 'readlink -f "$1" \ | grep -q ^/Library/Frameworks/Python.framework/Versions/' _ {} \; -exec rm -v {} \; - brew install --formula dpkg gnu-tar llvm lynx pkg-config qemu - echo $(brew --prefix)/opt/gnu-tar/libexec/gnubin >> $GITHUB_PATH - echo $(brew --prefix)/opt/llvm/bin >> $GITHUB_PATH + brew install --formula curl dpkg gnu-tar llvm lynx pkg-config qemu + echo $(brew --prefix curl)/bin >> $GITHUB_PATH + echo $(brew --prefix gnu-tar)/libexec/gnubin >> $GITHUB_PATH + echo $(brew --prefix llvm)/bin >> $GITHUB_PATH - uses: dtolnay/rust-toolchain@nightly with: @@ -278,6 +281,12 @@ jobs: # Do this on all system (not just macOS) to avoid relying on rustc-provided libLLVM.so. run: cargo install --force bpf-linker --git https://github.com/aya-rs/bpf-linker.git --no-default-features + - name: Cache test cache + uses: actions/cache@v4 + with: + path: test/.tmp + key: ${{ runner.arch }}-${{ runner.os }}-test-cache + - name: Download debian kernels if: runner.arch == 'ARM64' # TODO: enable tests on kernels before 6.0. @@ -302,7 +311,7 @@ jobs: run: | set -euxo pipefail find test/.tmp -name 'vmlinuz-*' -print0 | xargs -t -0 \ - cargo xtask integration-test vm --github-api-token ${{ secrets.GITHUB_TOKEN }} + cargo xtask integration-test vm --cache-dir test/.tmp --github-api-token ${{ secrets.GITHUB_TOKEN }} # Provides a single status check for the entire build workflow. # This is used for merge automation, like Mergify, since GH actions diff --git a/test/README.md b/test/README.md index fa3eebcad..57da51ed1 100644 --- a/test/README.md +++ b/test/README.md @@ -27,7 +27,7 @@ cargo xtask integration-test local ### Virtualized ```bash -cargo xtask integration-test vm +cargo xtask integration-test vm --cache-dir ... ``` ### Writing an integration test diff --git a/xtask/src/run.rs b/xtask/src/run.rs index ddd003c68..a6f919f5b 100644 --- a/xtask/src/run.rs +++ b/xtask/src/run.rs @@ -3,7 +3,7 @@ use std::{ fmt::Write as _, fs::{copy, create_dir_all, OpenOptions}, io::{BufRead as _, BufReader, Write as _}, - path::{Path, PathBuf}, + path::PathBuf, process::{Child, ChildStdin, Command, Output, Stdio}, sync::{Arc, Mutex}, thread, @@ -25,6 +25,10 @@ enum Environment { }, /// Runs the integration tests in a VM. VM { + /// The cache directory in which to store intermediate artifacts. + #[clap(long)] + cache_dir: PathBuf, + /// The Github API token to use if network requests to Github are made. /// /// This may be required if Github rate limits are exceeded. @@ -175,6 +179,7 @@ pub fn run(opts: Options) -> Result<()> { } } Environment::VM { + cache_dir, github_api_token, kernel_image, } => { @@ -195,13 +200,13 @@ pub fn run(opts: Options) -> Result<()> { // We consume the output of QEMU, looking for the output of our init program. This is // the only way to distinguish success from failure. We batch up the errors across all // VM images and report to the user. The end. - let cache_dir = Path::new("test/.tmp"); - create_dir_all(cache_dir).context("failed to create cache dir")?; + create_dir_all(&cache_dir).context("failed to create cache dir")?; let gen_init_cpio = cache_dir.join("gen_init_cpio"); if !gen_init_cpio .try_exists() .context("failed to check existence of gen_init_cpio")? { + // TODO(https://github.com/oxidecomputer/third-party-api-clients/issues/96): Use ETag-based caching. let client = octorust::Client::new( String::from("aya-xtask-integration-test-run"), github_api_token.map(octorust::auth::Credentials::Token),