diff --git a/rust-version b/rust-version
index e2a22b2395..e69de29bb2 100644
--- a/rust-version
+++ b/rust-version
@@ -1 +0,0 @@
-688c30dc9f8434d63bddb65bd6a4d2258d19717c
diff --git a/xtask/src/release.rs b/xtask/src/release.rs
index 9dcf7af00b..5699053a23 100644
--- a/xtask/src/release.rs
+++ b/xtask/src/release.rs
@@ -95,6 +95,14 @@ impl flags::RustcPull {
         if !cmd!(sh, "git status --untracked-files=no --porcelain").read()?.is_empty() {
             bail!("working directory must be clean before running `cargo xtask pull`");
         }
+        // This should not add any new root commits. So count those before and after merging.
+        let num_roots = || -> anyhow::Result<u32> {
+            Ok(cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
+                .read()
+                .context("failed to determine the number of root commits")?
+                .parse::<u32>()?)
+        };
+        let num_roots_before = num_roots()?;
         // Make sure josh is running.
         let josh = start_josh()?;
 
@@ -126,6 +134,11 @@ impl flags::RustcPull {
             .run()
             .context("FAILED to merge new commits, something went wrong")?;
 
+        // Check that the number of roots did not increase.
+        if num_roots()? != num_roots_before {
+            bail!("Josh created a new root commit. This is probably not the history you want.");
+        }
+
         drop(josh);
         Ok(())
     }