diff --git a/crates/hir-ty/src/inhabitedness.rs b/crates/hir-ty/src/inhabitedness.rs
index 505f80f324..39324d2730 100644
--- a/crates/hir-ty/src/inhabitedness.rs
+++ b/crates/hir-ty/src/inhabitedness.rs
@@ -1,8 +1,5 @@
 //! Type inhabitedness logic.
-use std::{
-    collections::HashMap,
-    ops::ControlFlow::{self, Break, Continue},
-};
+use std::ops::ControlFlow::{self, Break, Continue};
 
 use chalk_ir::{
     visit::{TypeSuperVisitable, TypeVisitable, TypeVisitor},
@@ -12,7 +9,7 @@ use hir_def::{
     adt::VariantData, attr::Attrs, visibility::Visibility, AdtId, EnumVariantId, HasModule, Lookup,
     ModuleId, VariantId,
 };
-use rustc_hash::FxHashMap;
+use rustc_hash::FxHashSet;
 
 use crate::{
     consteval::try_const_usize, db::HirDatabase, Binders, Interner, Substitution, Ty, TyKind,
@@ -21,7 +18,7 @@ use crate::{
 /// Checks whether a type is visibly uninhabited from a particular module.
 pub(crate) fn is_ty_uninhabited_from(ty: &Ty, target_mod: ModuleId, db: &dyn HirDatabase) -> bool {
     let mut uninhabited_from =
-        UninhabitedFrom { target_mod, db, max_depth: 500, recursive_ty: HashMap::default() };
+        UninhabitedFrom { target_mod, db, max_depth: 500, recursive_ty: FxHashSet::default() };
     let inhabitedness = ty.visit_with(&mut uninhabited_from, DebruijnIndex::INNERMOST);
     inhabitedness == BREAK_VISIBLY_UNINHABITED
 }
@@ -38,7 +35,7 @@ pub(crate) fn is_enum_variant_uninhabited_from(
     let is_local = variant.parent.lookup(db.upcast()).container.krate() == target_mod.krate();
 
     let mut uninhabited_from =
-        UninhabitedFrom { target_mod, db, max_depth: 500, recursive_ty: HashMap::default() };
+        UninhabitedFrom { target_mod, db, max_depth: 500, recursive_ty: FxHashSet::default() };
     let inhabitedness = uninhabited_from.visit_variant(
         variant.into(),
         &enum_data.variants[variant.local_id].variant_data,
@@ -51,7 +48,7 @@ pub(crate) fn is_enum_variant_uninhabited_from(
 
 struct UninhabitedFrom<'a> {
     target_mod: ModuleId,
-    recursive_ty: FxHashMap<Ty, ()>,
+    recursive_ty: FxHashSet<Ty>,
     // guard for preventing stack overflow in non trivial non terminating types
     max_depth: usize,
     db: &'a dyn HirDatabase,
@@ -74,12 +71,12 @@ impl TypeVisitor<Interner> for UninhabitedFrom<'_> {
         ty: &Ty,
         outer_binder: DebruijnIndex,
     ) -> ControlFlow<VisiblyUninhabited> {
-        if self.recursive_ty.contains_key(ty) || self.max_depth == 0 {
+        if self.recursive_ty.contains(ty) || self.max_depth == 0 {
             // rustc considers recursive types always inhabited. I think it is valid to consider
             // recursive types as always uninhabited, but we should do what rustc is doing.
             return CONTINUE_OPAQUELY_INHABITED;
         }
-        self.recursive_ty.insert(ty.clone(), ());
+        self.recursive_ty.insert(ty.clone());
         self.max_depth -= 1;
         let r = match ty.kind(Interner) {
             TyKind::Adt(adt, subst) => self.visit_adt(adt.0, subst),