From 4c5470226a51fe4c18785fb4a6b5dc44ebb6cb02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lauren=C8=9Biu=20Nicola?= Date: Sun, 7 Apr 2024 11:41:39 +0300 Subject: [PATCH] Use josh for subtree syncs --- Cargo.lock | 49 +++++ crates/rust-analyzer/tests/slow-tests/tidy.rs | 1 + rust-version | 1 + xtask/Cargo.toml | 1 + xtask/src/flags.rs | 17 +- xtask/src/main.rs | 3 +- xtask/src/release.rs | 168 ++++++++++++++++-- 7 files changed, 219 insertions(+), 21 deletions(-) create mode 100644 rust-version diff --git a/Cargo.lock b/Cargo.lock index c7cf4479b330..9451ef2f9156 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -324,6 +324,27 @@ dependencies = [ "syn", ] +[[package]] +name = "directories" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dissimilar" version = "1.0.7" @@ -896,6 +917,16 @@ dependencies = [ "libc", ] +[[package]] +name = "libredox" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +dependencies = [ + "bitflags 2.4.2", + "libc", +] + [[package]] name = "limit" version = "0.0.0" @@ -1184,6 +1215,12 @@ version = "11.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "parking_lot" version = "0.12.1" @@ -1564,6 +1601,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_users" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" +dependencies = [ + "getrandom", + "libredox", + "thiserror", +] + [[package]] name = "rowan" version = "0.15.15" @@ -2442,6 +2490,7 @@ name = "xtask" version = "0.1.0" dependencies = [ "anyhow", + "directories", "flate2", "itertools", "proc-macro2", diff --git a/crates/rust-analyzer/tests/slow-tests/tidy.rs b/crates/rust-analyzer/tests/slow-tests/tidy.rs index 34439391333d..4a7415b016da 100644 --- a/crates/rust-analyzer/tests/slow-tests/tidy.rs +++ b/crates/rust-analyzer/tests/slow-tests/tidy.rs @@ -144,6 +144,7 @@ MIT OR Apache-2.0 MIT OR Apache-2.0 OR Zlib MIT OR Zlib OR Apache-2.0 MIT/Apache-2.0 +MPL-2.0 Unlicense OR MIT Unlicense/MIT Zlib OR Apache-2.0 OR MIT diff --git a/rust-version b/rust-version new file mode 100644 index 000000000000..e2a22b2395a8 --- /dev/null +++ b/rust-version @@ -0,0 +1 @@ +688c30dc9f8434d63bddb65bd6a4d2258d19717c diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index a83d32e4141d..192de8694721 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -8,6 +8,7 @@ rust-version.workspace = true [dependencies] anyhow.workspace = true +directories = "5.0" flate2 = "1.0.24" write-json = "0.1.2" xshell.workspace = true diff --git a/xtask/src/flags.rs b/xtask/src/flags.rs index 906654592089..1b665a1e4a81 100644 --- a/xtask/src/flags.rs +++ b/xtask/src/flags.rs @@ -32,9 +32,12 @@ xflags::xflags! { cmd release { optional --dry-run } - cmd promote { - optional --dry-run + + cmd pull { + optional --commit refspec: String } + cmd push {} + cmd dist { /// Use mimalloc allocator for server optional --mimalloc @@ -77,7 +80,8 @@ pub enum XtaskCmd { Install(Install), FuzzTests(FuzzTests), Release(Release), - Promote(Promote), + Pull(Pull), + Push(Push), Dist(Dist), PublishReleaseNotes(PublishReleaseNotes), Metrics(Metrics), @@ -104,10 +108,13 @@ pub struct Release { } #[derive(Debug)] -pub struct Promote { - pub dry_run: bool, +pub struct Pull { + pub commit: Option, } +#[derive(Debug)] +pub struct Push; + #[derive(Debug)] pub struct Dist { pub mimalloc: bool, diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9418675a348c..b5fcf7da7c4e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -34,7 +34,8 @@ fn main() -> anyhow::Result<()> { flags::XtaskCmd::Install(cmd) => cmd.run(sh), flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh), flags::XtaskCmd::Release(cmd) => cmd.run(sh), - flags::XtaskCmd::Promote(cmd) => cmd.run(sh), + flags::XtaskCmd::Pull(cmd) => cmd.run(sh), + flags::XtaskCmd::Push(cmd) => cmd.run(sh), flags::XtaskCmd::Dist(cmd) => cmd.run(sh), flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh), flags::XtaskCmd::Metrics(cmd) => cmd.run(sh), diff --git a/xtask/src/release.rs b/xtask/src/release.rs index 1a5e6dfb4ccf..8fc196ce4dbd 100644 --- a/xtask/src/release.rs +++ b/xtask/src/release.rs @@ -1,5 +1,12 @@ mod changelog; +use std::process::{Command, Stdio}; +use std::time::Duration; +use std::{env, thread}; + +use anyhow::{bail, Context as _}; +use directories::ProjectDirs; +use stdx::JodChild; use xshell::{cmd, Shell}; use crate::{codegen, date_iso, flags, is_release_tag, project_root}; @@ -71,26 +78,157 @@ impl flags::Release { } } -impl flags::Promote { +// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs +impl flags::Pull { pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { - let _dir = sh.push_dir("../rust-rust-analyzer"); - cmd!(sh, "git switch master").run()?; - cmd!(sh, "git fetch upstream").run()?; - cmd!(sh, "git reset --hard upstream/master").run()?; + sh.change_dir(project_root()); + let commit = self.commit.map(Result::Ok).unwrap_or_else(|| { + let rust_repo_head = + cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?; + rust_repo_head + .split_whitespace() + .next() + .map(|front| front.trim().to_owned()) + .ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote.")) + })?; + // Make sure the repo is clean. + if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() { + bail!("working directory must be clean before running `cargo xtask pull`"); + } + // Make sure josh is running. + let josh = start_josh()?; - let date = date_iso(sh)?; - let branch = format!("rust-analyzer-{date}"); - cmd!(sh, "git switch -c {branch}").run()?; - cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?; + // Update rust-version file. As a separate commit, since making it part of + // the merge has confused the heck out of josh in the past. + // We pass `--no-verify` to avoid running any git hooks that might exist, + // in case they dirty the repository. + sh.write_file("rust-version", format!("{commit}\n"))?; + const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rustc"; + cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}") + .run() + .context("FAILED to commit rust-version file, something went wrong")?; - if !self.dry_run { - cmd!(sh, "git push -u origin {branch}").run()?; - cmd!( - sh, - "xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost" - ) + // Fetch given rustc commit. + cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git") + .run() + .map_err(|e| { + // Try to un-do the previous `git commit`, to leave the repo in the state we found it it. + cmd!(sh, "git reset --hard HEAD^") + .run() + .expect("FAILED to clean up again after failed `git fetch`, sorry for that"); + e + }) + .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?; + + // Merge the fetched commit. + const MERGE_COMMIT_MESSAGE: &str = "Merge from downstream"; + cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}") + .run() + .context("FAILED to merge new commits, something went wrong")?; + + drop(josh); + Ok(()) + } +} + +impl flags::Push { + pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> { + let branch = "sync-from-ra"; + let Ok(github_user) = env::var("GITHUB_USER") else { + bail!("please set `GITHUB_USER` to the GitHub username of your `rust-lang/rust` clone"); + }; + + sh.change_dir(project_root()); + let base = sh.read_file("rust-version")?.trim().to_owned(); + // Make sure the repo is clean. + if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() { + bail!("working directory must be clean before running `cargo xtask push`"); + } + // Make sure josh is running. + let josh = start_josh()?; + + // Find a repo we can do our preparation in. + if let Ok(rustc_git) = env::var("RUSTC_GIT") { + // If rustc_git is `Some`, we'll use an existing fork for the branch updates. + sh.change_dir(rustc_git); + } else { + bail!("please set `RUSTC_GIT` to a `rust-lang/rust` clone"); + }; + // Prepare the branch. Pushing works much better if we use as base exactly + // the commit that we pulled from last time, so we use the `rust-version` + // file to find out which commit that would be. + println!("Preparing {github_user}/rust (base: {base})..."); + if cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}") + .ignore_stderr() + .read() + .is_ok() + { + bail!( + "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again." + ); + } + cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?; + cmd!(sh, "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch}") + .ignore_stdout() + .ignore_stderr() // silence the "create GitHub PR" message .run()?; + println!(); + + // Do the actual push. + sh.change_dir(project_root()); + println!("Pushing rust-analyzer changes..."); + cmd!( + sh, + "git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}" + ) + .run()?; + println!(); + + // Do a round-trip check to make sure the push worked as expected. + cmd!( + sh, + "git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}" + ) + .ignore_stderr() + .read()?; + let head = cmd!(sh, "git rev-parse HEAD").read()?; + let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?; + if head != fetch_head { + bail!("Josh created a non-roundtrip push! Do NOT merge this into rustc!"); } + println!("Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:"); + println!( + " https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost" + ); + + drop(josh); Ok(()) } } + +/// Used for rustc syncs. +const JOSH_FILTER: &str = + ":rev(f5a9250147f6569d8d89334dc9cca79c0322729f:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer"; +const JOSH_PORT: &str = "42042"; + +fn start_josh() -> anyhow::Result { + // Determine cache directory. + let local_dir = { + let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap(); + user_dirs.cache_dir().to_owned() + }; + + // Start josh, silencing its output. + let mut cmd = Command::new("josh-proxy"); + cmd.arg("--local").arg(local_dir); + cmd.arg("--remote").arg("https://github.com"); + cmd.arg("--port").arg(JOSH_PORT); + cmd.arg("--no-background"); + cmd.stdout(Stdio::null()); + cmd.stderr(Stdio::null()); + let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?; + // Give it some time so hopefully the port is open. (100ms was not enough.) + thread::sleep(Duration::from_millis(200)); + + Ok(JodChild(josh)) +}