Auto merge of #17025 - lnicola:josh, r=lnicola

internal: Use josh for subtree syncs

Cargo.lock

@@ -323,6 +323,27 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "directories"
+version = "5.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a49173b84e034382284f27f1af4dcbbd231ffa358c0fe316541a7337f376a35"
+dependencies = [
+ "dirs-sys",
+]
+
+[[package]]
+name = "dirs-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c"
+dependencies = [
+ "libc",
+ "option-ext",
+ "redox_users",
+ "windows-sys 0.48.0",
+]
+
 [[package]]
 name = "dissimilar"
 version = "1.0.7"
@@ -898,6 +919,16 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "libredox"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
+dependencies = [
+ "bitflags 2.4.2",
+ "libc",
+]
+
 [[package]]
 name = "limit"
 version = "0.0.0"
@@ -1186,6 +1217,12 @@ version = "11.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575"
 
+[[package]]
+name = "option-ext"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
+
 [[package]]
 name = "parking_lot"
 version = "0.12.1"
@@ -1566,6 +1603,17 @@ dependencies = [
  "bitflags 1.3.2",
 ]
 
+[[package]]
+name = "redox_users"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891"
+dependencies = [
+ "getrandom",
+ "libredox",
+ "thiserror",
+]
+
 [[package]]
 name = "rowan"
 version = "0.15.15"
@@ -2499,6 +2547,7 @@ name = "xtask"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "directories",
  "flate2",
  "itertools",
  "proc-macro2",

crates/rust-analyzer/tests/slow-tests/tidy.rs

@@ -144,6 +144,7 @@ MIT OR Apache-2.0
 MIT OR Apache-2.0 OR Zlib
 MIT OR Zlib OR Apache-2.0
 MIT/Apache-2.0
+MPL-2.0
 Unlicense OR MIT
 Unlicense/MIT
 Zlib OR Apache-2.0 OR MIT

docs/dev/README.md

@@ -229,12 +229,22 @@ Release steps:
      * publishes the VS Code extension to the marketplace
    * call the GitHub API for PR details
    * create a new changelog in `rust-analyzer.github.io`
-3. While the release is in progress, fill in the changelog
+3. While the release is in progress, fill in the changelog.
-4. Commit & push the changelog
+4. Commit & push the changelog.
 5. Run `cargo xtask publish-release-notes <CHANGELOG>` -- this will convert the changelog entry in AsciiDoc to Markdown and update the body of GitHub Releases entry.
-6. Tweet
+6. Tweet.
-7. Inside `rust-analyzer`, run `cargo xtask promote` -- this will create a PR to rust-lang/rust updating rust-analyzer's subtree.
+7. Make a new branch and run `cargo xtask rustc-pull`, open a PR, and merge it.
-   Self-approve the PR.
+   This will pull any changes from `rust-lang/rust` into `rust-analyzer`.
+8. Switch to `master`, pull, then run `cargo xtask rustc-push --rust-path ../rust-rust-analyzer --rust-fork matklad/rust`.
+   Replace `matklad/rust` with your own fork of `rust-lang/rust`.
+   You can use the token to authenticate when you get prompted for a password, since `josh` will push over HTTPS, not SSH.
+   This will push the `rust-analyzer` changes to your fork.
+   You can then open a PR against `rust-lang/rust`.
+
+Note: besides the `rust-rust-analyzer` clone, the Josh cache (stored under `~/.cache/rust-analyzer-josh`) will contain a bare clone of `rust-lang/rust`.
+This currently takes about 3.5 GB.
+
+This [HackMD](https://hackmd.io/7pOuxnkdQDaL1Y1FQr65xg) has details about how `josh` syncs work.
 
 If the GitHub Actions release fails because of a transient problem like a timeout, you can re-run the job from the Actions console.
 If it fails because of something that needs to be fixed, remove the release tag (if needed), fix the problem, then start over.

rust-version

@@ -0,0 +1 @@
+688c30dc9f8434d63bddb65bd6a4d2258d19717c

xtask/Cargo.toml

@@ -8,6 +8,7 @@ rust-version.workspace = true
 
 [dependencies]
 anyhow.workspace = true
+directories = "5.0"
 flate2 = "1.0.24"
 write-json = "0.1.2"
 xshell.workspace = true

xtask/src/flags.rs

@@ -14,16 +14,16 @@ xflags::xflags! {
         cmd install {
             /// Install only VS Code plugin.
             optional --client
-            /// One of 'code', 'code-exploration', 'code-insiders', 'codium', or 'code-oss'.
+            /// One of `code`, `code-exploration`, `code-insiders`, `codium`, or `code-oss`.
             optional --code-bin name: String
 
             /// Install only the language server.
             optional --server
-            /// Use mimalloc allocator for server
+            /// Use mimalloc allocator for server.
             optional --mimalloc
-            /// Use jemalloc allocator for server
+            /// Use jemalloc allocator for server.
             optional --jemalloc
-            /// build in release with debug info set to 2
+            /// build in release with debug info set to 2.
             optional --dev-rel
         }
 
@@ -32,9 +32,21 @@ xflags::xflags! {
         cmd release {
             optional --dry-run
         }
-        cmd promote {
+
-            optional --dry-run
+        cmd rustc-pull {
+            /// rustc commit to pull.
+            optional --commit refspec: String
         }
+
+        cmd rustc-push {
+            /// rust local path, e.g. `../rust-rust-analyzer`.
+            required --rust-path rust_path: String
+            /// rust fork name, e.g.  `matklad/rust`.
+            required --rust-fork rust_fork: String
+            /// branch name.
+            optional --branch branch: String
+        }
+
         cmd dist {
             /// Use mimalloc allocator for server
             optional --mimalloc
@@ -77,7 +89,8 @@ pub enum XtaskCmd {
     Install(Install),
     FuzzTests(FuzzTests),
     Release(Release),
-    Promote(Promote),
+    RustcPull(RustcPull),
+    RustcPush(RustcPush),
     Dist(Dist),
     PublishReleaseNotes(PublishReleaseNotes),
     Metrics(Metrics),
@@ -104,8 +117,15 @@ pub struct Release {
 }
 
 #[derive(Debug)]
-pub struct Promote {
+pub struct RustcPull {
-    pub dry_run: bool,
+    pub commit: Option<String>,
+}
+
+#[derive(Debug)]
+pub struct RustcPush {
+    pub rust_path: String,
+    pub rust_fork: String,
+    pub branch: Option<String>,
 }
 
 #[derive(Debug)]

xtask/src/main.rs

@@ -34,7 +34,8 @@ fn main() -> anyhow::Result<()> {
         flags::XtaskCmd::Install(cmd) => cmd.run(sh),
         flags::XtaskCmd::FuzzTests(_) => run_fuzzer(sh),
         flags::XtaskCmd::Release(cmd) => cmd.run(sh),
-        flags::XtaskCmd::Promote(cmd) => cmd.run(sh),
+        flags::XtaskCmd::RustcPull(cmd) => cmd.run(sh),
+        flags::XtaskCmd::RustcPush(cmd) => cmd.run(sh),
         flags::XtaskCmd::Dist(cmd) => cmd.run(sh),
         flags::XtaskCmd::PublishReleaseNotes(cmd) => cmd.run(sh),
         flags::XtaskCmd::Metrics(cmd) => cmd.run(sh),

xtask/src/release.rs

@@ -1,5 +1,12 @@
 mod changelog;
 
+use std::process::{Command, Stdio};
+use std::thread;
+use std::time::Duration;
+
+use anyhow::{bail, Context as _};
+use directories::ProjectDirs;
+use stdx::JodChild;
 use xshell::{cmd, Shell};
 
 use crate::{codegen, date_iso, flags, is_release_tag, project_root};
@@ -71,26 +78,154 @@ impl flags::Release {
     }
 }
 
-impl flags::Promote {
+// git sync implementation adapted from https://github.com/rust-lang/miri/blob/62039ac/miri-script/src/commands.rs
+impl flags::RustcPull {
     pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
-        let _dir = sh.push_dir("../rust-rust-analyzer");
+        sh.change_dir(project_root());
-        cmd!(sh, "git switch master").run()?;
+        let commit = self.commit.map(Result::Ok).unwrap_or_else(|| {
-        cmd!(sh, "git fetch upstream").run()?;
+            let rust_repo_head =
-        cmd!(sh, "git reset --hard upstream/master").run()?;
+                cmd!(sh, "git ls-remote https://github.com/rust-lang/rust/ HEAD").read()?;
-
+            rust_repo_head
-        let date = date_iso(sh)?;
+                .split_whitespace()
-        let branch = format!("rust-analyzer-{date}");
+                .next()
-        cmd!(sh, "git switch -c {branch}").run()?;
+                .map(|front| front.trim().to_owned())
-        cmd!(sh, "git subtree pull -m ':arrow_up: rust-analyzer' -P src/tools/rust-analyzer rust-analyzer release").run()?;
+                .ok_or_else(|| anyhow::format_err!("Could not obtain Rust repo HEAD from remote."))
-
+        })?;
-        if !self.dry_run {
+        // Make sure the repo is clean.
-            cmd!(sh, "git push -u origin {branch}").run()?;
+        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
-            cmd!(
+            bail!("working directory must be clean before running `cargo xtask pull`");
-                sh,
-                "xdg-open https://github.com/matklad/rust/pull/new/{branch}?body=r%3F%20%40ghost"
-            )
-            .run()?;
         }
+        // Make sure josh is running.
+        let josh = start_josh()?;
+
+        // Update rust-version file. As a separate commit, since making it part of
+        // the merge has confused the heck out of josh in the past.
+        // We pass `--no-verify` to avoid running any git hooks that might exist,
+        // in case they dirty the repository.
+        sh.write_file("rust-version", format!("{commit}\n"))?;
+        const PREPARING_COMMIT_MESSAGE: &str = "Preparing for merge from rust-lang/rust";
+        cmd!(sh, "git commit rust-version --no-verify -m {PREPARING_COMMIT_MESSAGE}")
+            .run()
+            .context("FAILED to commit rust-version file, something went wrong")?;
+
+        // Fetch given rustc commit.
+        cmd!(sh, "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git")
+            .run()
+            .map_err(|e| {
+                // Try to un-do the previous `git commit`, to leave the repo in the state we found it it.
+                cmd!(sh, "git reset --hard HEAD^")
+                    .run()
+                    .expect("FAILED to clean up again after failed `git fetch`, sorry for that");
+                e
+            })
+            .context("FAILED to fetch new commits, something went wrong (committing the rust-version file has been undone)")?;
+
+        // Merge the fetched commit.
+        const MERGE_COMMIT_MESSAGE: &str = "Merge from rust-lang/rust";
+        cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
+            .run()
+            .context("FAILED to merge new commits, something went wrong")?;
+
+        drop(josh);
         Ok(())
     }
 }
+
+impl flags::RustcPush {
+    pub(crate) fn run(self, sh: &Shell) -> anyhow::Result<()> {
+        let branch = self.branch.as_deref().unwrap_or("sync-from-ra");
+        let rust_path = self.rust_path;
+        let rust_fork = self.rust_fork;
+
+        sh.change_dir(project_root());
+        let base = sh.read_file("rust-version")?.trim().to_owned();
+        // Make sure the repo is clean.
+        if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
+            bail!("working directory must be clean before running `cargo xtask push`");
+        }
+        // Make sure josh is running.
+        let josh = start_josh()?;
+
+        // Find a repo we can do our preparation in.
+        sh.change_dir(rust_path);
+
+        // Prepare the branch. Pushing works much better if we use as base exactly
+        // the commit that we pulled from last time, so we use the `rust-version`
+        // file to find out which commit that would be.
+        println!("Preparing {rust_fork} (base: {base})...");
+        if cmd!(sh, "git fetch https://github.com/{rust_fork} {branch}")
+            .ignore_stderr()
+            .read()
+            .is_ok()
+        {
+            bail!(
+                "The branch `{branch}` seems to already exist in `https://github.com/{rust_fork}`. Please delete it and try again."
+            );
+        }
+        cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}").run()?;
+        cmd!(sh, "git push https://github.com/{rust_fork} {base}:refs/heads/{branch}")
+            .ignore_stdout()
+            .ignore_stderr() // silence the "create GitHub PR" message
+            .run()?;
+        println!();
+
+        // Do the actual push.
+        sh.change_dir(project_root());
+        println!("Pushing rust-analyzer changes...");
+        cmd!(
+            sh,
+            "git push http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git HEAD:{branch}"
+        )
+        .run()?;
+        println!();
+
+        // Do a round-trip check to make sure the push worked as expected.
+        cmd!(
+            sh,
+            "git fetch http://localhost:{JOSH_PORT}/{rust_fork}.git{JOSH_FILTER}.git {branch}"
+        )
+        .ignore_stderr()
+        .read()?;
+        let head = cmd!(sh, "git rev-parse HEAD").read()?;
+        let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD").read()?;
+        if head != fetch_head {
+            bail!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
+        }
+        println!("Confirmed that the push round-trips back to rust-analyzer properly. Please create a rustc PR:");
+        // https://github.com/github-linguist/linguist/compare/master...octocat:linguist:master
+        let fork_path = rust_fork.replace('/', ":");
+        println!(
+            "    https://github.com/rust-lang/rust/compare/{fork_path}:{branch}?quick_pull=1&title=Subtree+update+of+rust-analyzer&body=r?+@ghost"
+        );
+
+        drop(josh);
+        Ok(())
+    }
+}
+
+/// Used for rustc syncs.
+const JOSH_FILTER: &str =
+    ":rev(55d9a533b309119c8acd13061581b43ae8840823:prefix=src/tools/rust-analyzer):/src/tools/rust-analyzer";
+const JOSH_PORT: &str = "42042";
+
+fn start_josh() -> anyhow::Result<impl Drop> {
+    // Determine cache directory.
+    let local_dir = {
+        let user_dirs = ProjectDirs::from("org", "rust-lang", "rust-analyzer-josh").unwrap();
+        user_dirs.cache_dir().to_owned()
+    };
+
+    // Start josh, silencing its output.
+    let mut cmd = Command::new("josh-proxy");
+    cmd.arg("--local").arg(local_dir);
+    cmd.arg("--remote").arg("https://github.com");
+    cmd.arg("--port").arg(JOSH_PORT);
+    cmd.arg("--no-background");
+    cmd.stdout(Stdio::null());
+    cmd.stderr(Stdio::null());
+    let josh = cmd.spawn().context("failed to start josh-proxy, make sure it is installed")?;
+    // Give it some time so hopefully the port is open. (100ms was not enough.)
+    thread::sleep(Duration::from_millis(200));
+
+    Ok(JodChild(josh))
+}