diff --git a/clippy_lints/src/types.rs b/clippy_lints/src/types.rs
index 36a4b699f..b5023e268 100644
--- a/clippy_lints/src/types.rs
+++ b/clippy_lints/src/types.rs
@@ -1210,17 +1210,25 @@ impl<'a, 'tcx> LateLintPass<'a, 'tcx> for Casts {
             if_chain! {
                 if let ty::RawPtr(from_ptr_ty) = &cast_from.sty;
                 if let ty::RawPtr(to_ptr_ty) = &cast_to.sty;
-                if let Some(from_align) = cx.layout_of(from_ptr_ty.ty).ok().map(|a| a.align.abi);
-                if let Some(to_align) = cx.layout_of(to_ptr_ty.ty).ok().map(|a| a.align.abi);
-                if from_align < to_align;
+                if let Ok(from_layout) = cx.layout_of(from_ptr_ty.ty);
+                if let Ok(to_layout) = cx.layout_of(to_ptr_ty.ty);
+                if from_layout.align.abi < to_layout.align.abi;
                 // with c_void, we inherently need to trust the user
                 if !is_c_void(cx, from_ptr_ty.ty);
+                // when casting from a ZST, we don't know enough to properly lint
+                if !from_layout.is_zst();
                 then {
                     span_lint(
                         cx,
                         CAST_PTR_ALIGNMENT,
                         expr.span,
-                        &format!("casting from `{}` to a more-strictly-aligned pointer (`{}`)", cast_from, cast_to)
+                        &format!(
+                            "casting from `{}` to a more-strictly-aligned pointer (`{}`) ({} < {} bytes)",
+                            cast_from,
+                            cast_to,
+                            from_layout.align.abi.bytes(),
+                            to_layout.align.abi.bytes(),
+                        ),
                     );
                 }
             }
diff --git a/tests/ui/cast_alignment.rs b/tests/ui/cast_alignment.rs
index 08450ba11..4c0893563 100644
--- a/tests/ui/cast_alignment.rs
+++ b/tests/ui/cast_alignment.rs
@@ -22,4 +22,6 @@ fn main() {
     // For c_void, we should trust the user. See #2677
     (&1u32 as *const u32 as *const std::os::raw::c_void) as *const u32;
     (&1u32 as *const u32 as *const libc::c_void) as *const u32;
+    // For ZST, we should trust the user. See #4256
+    (&1u32 as *const u32 as *const ()) as *const u32;
 }
diff --git a/tests/ui/cast_alignment.stderr b/tests/ui/cast_alignment.stderr
index 0077be1b5..79219f861 100644
--- a/tests/ui/cast_alignment.stderr
+++ b/tests/ui/cast_alignment.stderr
@@ -1,4 +1,4 @@
-error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`)
+error: casting from `*const u8` to a more-strictly-aligned pointer (`*const u16`) (1 < 2 bytes)
   --> $DIR/cast_alignment.rs:12:5
    |
 LL |     (&1u8 as *const u8) as *const u16;
@@ -6,7 +6,7 @@ LL |     (&1u8 as *const u8) as *const u16;
    |
    = note: `-D clippy::cast-ptr-alignment` implied by `-D warnings`
 
-error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`)
+error: casting from `*mut u8` to a more-strictly-aligned pointer (`*mut u16`) (1 < 2 bytes)
   --> $DIR/cast_alignment.rs:13:5
    |
 LL |     (&mut 1u8 as *mut u8) as *mut u16;