diff --git a/crates/proc-macro-srv/src/dylib.rs b/crates/proc-macro-srv/src/dylib.rs
index 78ae4574c4..49a249f2cb 100644
--- a/crates/proc-macro-srv/src/dylib.rs
+++ b/crates/proc-macro-srv/src/dylib.rs
@@ -3,7 +3,12 @@
 mod version;
 
 use proc_macro::bridge;
-use std::{fmt, fs::File, io};
+use std::{
+    fmt,
+    fs::{self, File},
+    io,
+    time::SystemTime,
+};
 
 use libloading::Library;
 use memmap2::Mmap;
@@ -136,6 +141,7 @@ impl ProcMacroLibraryLibloading {
 pub(crate) struct Expander {
     inner: ProcMacroLibraryLibloading,
     path: Utf8PathBuf,
+    modified_time: SystemTime,
 }
 
 impl Drop for Expander {
@@ -151,12 +157,13 @@ impl Expander {
         // Some libraries for dynamic loading require canonicalized path even when it is
         // already absolute
         let lib = lib.canonicalize_utf8()?;
+        let modified_time = fs::metadata(&lib).and_then(|it| it.modified())?;
 
         let path = ensure_file_with_lock_free_access(&lib)?;
 
         let library = ProcMacroLibraryLibloading::open(path.as_ref())?;
 
-        Ok(Expander { inner: library, path })
+        Ok(Expander { inner: library, path, modified_time })
     }
 
     pub(crate) fn expand<S: ProcMacroSrvSpan>(
@@ -181,6 +188,10 @@ impl Expander {
     pub(crate) fn list_macros(&self) -> Vec<(String, ProcMacroKind)> {
         self.inner.proc_macros.list_macros()
     }
+
+    pub(crate) fn modified_time(&self) -> SystemTime {
+        self.modified_time
+    }
 }
 
 /// Copy the dylib to temp directory to prevent locking in Windows
diff --git a/crates/proc-macro-srv/src/lib.rs b/crates/proc-macro-srv/src/lib.rs
index f0aa6b3f93..8e78e6f2e0 100644
--- a/crates/proc-macro-srv/src/lib.rs
+++ b/crates/proc-macro-srv/src/lib.rs
@@ -35,7 +35,6 @@ use std::{
     fs,
     path::{Path, PathBuf},
     thread,
-    time::SystemTime,
 };
 
 use paths::{Utf8Path, Utf8PathBuf};
@@ -53,7 +52,7 @@ use crate::server_impl::TokenStream;
 pub const RUSTC_VERSION_STRING: &str = env!("RUSTC_VERSION");
 
 pub struct ProcMacroSrv<'env> {
-    expanders: HashMap<(Utf8PathBuf, SystemTime), dylib::Expander>,
+    expanders: HashMap<Utf8PathBuf, dylib::Expander>,
     span_mode: SpanMode,
     env: &'env EnvSnapshot,
 }
@@ -81,10 +80,9 @@ impl<'env> ProcMacroSrv<'env> {
     ) -> Result<(msg::FlatTree, Vec<u32>), msg::PanicMessage> {
         let span_mode = self.span_mode;
         let snapped_env = self.env;
-        let expander = self.expander(lib.as_ref()).map_err(|err| {
-            debug_assert!(false, "should list macros before asking to expand");
-            msg::PanicMessage(format!("failed to load macro: {err}"))
-        })?;
+        let expander = self
+            .expander(lib.as_ref())
+            .map_err(|err| msg::PanicMessage(format!("failed to load macro: {err}")))?;
 
         let prev_env = EnvChange::apply(snapped_env, env, current_dir.as_ref().map(<_>::as_ref));
 
@@ -107,16 +105,20 @@ impl<'env> ProcMacroSrv<'env> {
     }
 
     fn expander(&mut self, path: &Utf8Path) -> Result<&dylib::Expander, String> {
-        let time = fs::metadata(path)
-            .and_then(|it| it.modified())
-            .map_err(|err| format!("Failed to get file metadata for {path}: {err}",))?;
+        let expander = || {
+            dylib::Expander::new(path)
+                .map_err(|err| format!("Cannot create expander for {path}: {err}",))
+        };
 
-        Ok(match self.expanders.entry((path.to_path_buf(), time)) {
-            Entry::Vacant(v) => v.insert(
-                dylib::Expander::new(path)
-                    .map_err(|err| format!("Cannot create expander for {path}: {err}",))?,
-            ),
-            Entry::Occupied(e) => e.into_mut(),
+        Ok(match self.expanders.entry(path.to_path_buf()) {
+            Entry::Vacant(v) => v.insert(expander()?),
+            Entry::Occupied(mut e) => {
+                let time = fs::metadata(path).and_then(|it| it.modified()).ok();
+                if Some(e.get().modified_time()) != time {
+                    e.insert(expander()?);
+                }
+                e.into_mut()
+            }
         })
     }
 }