diff --git a/crates/bevy_ecs/src/archetype.rs b/crates/bevy_ecs/src/archetype.rs
index 15e962291c..5b56e1cff4 100644
--- a/crates/bevy_ecs/src/archetype.rs
+++ b/crates/bevy_ecs/src/archetype.rs
@@ -766,7 +766,7 @@ pub struct Archetypes {
     /// find the archetype id by the archetype's components
     by_components: HashMap<ArchetypeComponents, ArchetypeId>,
     /// find all the archetypes that contain a component
-    by_component: ComponentIndex,
+    pub(crate) by_component: ComponentIndex,
 }
 
 /// Metadata about how a component is stored in an [`Archetype`].
diff --git a/crates/bevy_ecs/src/observer/mod.rs b/crates/bevy_ecs/src/observer/mod.rs
index 2571157302..1c1507fdce 100644
--- a/crates/bevy_ecs/src/observer/mod.rs
+++ b/crates/bevy_ecs/src/observer/mod.rs
@@ -456,14 +456,17 @@ impl World {
                     if observers.map.is_empty() && observers.entity_map.is_empty() {
                         cache.component_observers.remove(component);
                         if let Some(flag) = Observers::is_archetype_cached(event_type) {
-                            for archetype in &mut archetypes.archetypes {
-                                if archetype.contains(*component) {
-                                    let no_longer_observed = archetype
-                                        .components()
-                                        .all(|id| !cache.component_observers.contains_key(&id));
+                            if let Some(by_component) = archetypes.by_component.get(component) {
+                                for archetype in by_component.keys() {
+                                    let archetype = &mut archetypes.archetypes[archetype.index()];
+                                    if archetype.contains(*component) {
+                                        let no_longer_observed = archetype
+                                            .components()
+                                            .all(|id| !cache.component_observers.contains_key(&id));
 
-                                    if no_longer_observed {
-                                        archetype.flags.set(flag, false);
+                                        if no_longer_observed {
+                                            archetype.flags.set(flag, false);
+                                        }
                                     }
                                 }
                             }