Add optional transparency passthrough for sprite backend with bevy_picking (#16388)

# Objective

- Allow bevy_sprite_picking backend to pass through transparent sections
of the sprite.
- Fixes #14929

## Solution

- After sprite picking detects the cursor is within a sprites rect,
check the pixel at that location on the texture and check that it meets
an optional transparency cutoff. Change originally created for
mod_picking on bevy 0.14
(https://github.com/aevyrie/bevy_mod_picking/pull/373)

## Testing

- Ran Sprite Picking example to check it was working both with
transparency enabled and disabled
- ModPicking version is currently in use in my own isometric game where
this has been an extremely noticeable issue

## Showcase

![Sprite Picking
Text](https://github.com/user-attachments/assets/76568c0d-c359-422b-942d-17c84d3d3009)

## Migration Guide

Sprite picking now ignores transparent regions (with an alpha value less
than or equal to 0.1). To configure this, modify the
`SpriteBackendSettings` resource.

---------

Co-authored-by: andriyDev <andriydzikh@gmail.com>

crates/bevy_sprite/src/lib.rs

@@ -41,6 +41,8 @@ use bevy_reflect::{std_traits::ReflectDefault, Reflect};
 pub use bundle::*;
 pub use dynamic_texture_atlas_builder::*;
 pub use mesh2d::*;
+#[cfg(feature = "bevy_sprite_picking_backend")]
+pub use picking_backend::*;
 pub use render::*;
 pub use sprite::*;
 pub use texture_atlas::*;
@@ -148,7 +150,7 @@ impl Plugin for SpritePlugin {
 
         #[cfg(feature = "bevy_sprite_picking_backend")]
         if self.add_picking {
-            app.add_plugins(picking_backend::SpritePickingPlugin);
+            app.add_plugins(SpritePickingPlugin);
         }
 
         if let Some(render_app) = app.get_sub_app_mut(RenderApp) {

crates/bevy_sprite/src/picking_backend.rs

@@ -7,29 +7,63 @@ use core::cmp::Reverse;
 use crate::{Sprite, TextureAtlasLayout};
 use bevy_app::prelude::*;
 use bevy_asset::prelude::*;
+use bevy_color::Alpha;
 use bevy_ecs::prelude::*;
 use bevy_image::Image;
 use bevy_math::{prelude::*, FloatExt, FloatOrd};
 use bevy_picking::backend::prelude::*;
+use bevy_reflect::prelude::*;
 use bevy_render::prelude::*;
 use bevy_transform::prelude::*;
 use bevy_window::PrimaryWindow;
 
+/// How should the [`SpritePickingPlugin`] handle picking and how should it handle transparent pixels
+#[derive(Debug, Clone, Copy, Reflect)]
+pub enum SpritePickingMode {
+    /// Even if a sprite is picked on a transparent pixel, it should still count within the backend.
+    /// Only consider the rect of a given sprite.
+    BoundingBox,
+    /// Ignore any part of a sprite which has a lower alpha value than the threshold (inclusive)
+    /// Threshold is given as an f32 representing the alpha value in a Bevy Color Value
+    AlphaThreshold(f32),
+}
+
+/// Runtime settings for the [`SpritePickingPlugin`].
+#[derive(Resource, Reflect)]
+#[reflect(Resource, Default)]
+pub struct SpritePickingSettings {
+    /// Should the backend count transparent pixels as part of the sprite for picking purposes or should it use the bounding box of the sprite alone.
+    ///
+    /// Defaults to an incusive alpha threshold of 0.1
+    pub picking_mode: SpritePickingMode,
+}
+
+impl Default for SpritePickingSettings {
+    fn default() -> Self {
+        Self {
+            picking_mode: SpritePickingMode::AlphaThreshold(0.1),
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct SpritePickingPlugin;
 
 impl Plugin for SpritePickingPlugin {
     fn build(&self, app: &mut App) {
-        app.add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend));
+        app.init_resource::<SpritePickingSettings>()
+            .add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend));
     }
 }
 
-pub fn sprite_picking(
+#[allow(clippy::too_many_arguments)]
+fn sprite_picking(
     pointers: Query<(&PointerId, &PointerLocation)>,
     cameras: Query<(Entity, &Camera, &GlobalTransform, &OrthographicProjection)>,
     primary_window: Query<Entity, With<PrimaryWindow>>,
     images: Res<Assets<Image>>,
     texture_atlas_layout: Res<Assets<TextureAtlasLayout>>,
+    settings: Res<SpritePickingSettings>,
     sprite_query: Query<(
         Entity,
         &Sprite,
@@ -91,22 +125,6 @@ pub fn sprite_picking(
                     return None;
                 }
 
-                // Hit box in sprite coordinate system
-                let extents = match (sprite.custom_size, &sprite.texture_atlas) {
-                    (Some(custom_size), _) => custom_size,
-                    (None, None) => images.get(&sprite.image)?.size().as_vec2(),
-                    (None, Some(atlas)) => texture_atlas_layout
-                        .get(&atlas.layout)
-                        .and_then(|layout| layout.textures.get(atlas.index))
-                        // Dropped atlas layouts and indexes out of bounds are rendered as a sprite
-                        .map_or(images.get(&sprite.image)?.size().as_vec2(), |rect| {
-                            rect.size().as_vec2()
-                        }),
-                };
-                let anchor = sprite.anchor.as_vec();
-                let center = -anchor * extents;
-                let rect = Rect::from_center_half_size(center, extents / 2.0);
-
                 // Transform cursor line segment to sprite coordinate system
                 let world_to_sprite = sprite_transform.affine().inverse();
                 let cursor_start_sprite = world_to_sprite.transform_point3(cursor_ray_world.origin);
@@ -133,14 +151,46 @@ pub fn sprite_picking(
                     .lerp(cursor_end_sprite, lerp_factor)
                     .xy();
 
-                let is_cursor_in_sprite = rect.contains(cursor_pos_sprite);
+                let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point(
+                    cursor_pos_sprite,
+                    &images,
+                    &texture_atlas_layout,
+                ) else {
+                    return None;
+                };
 
-                blocked = is_cursor_in_sprite
+                // Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of
+                // the sprite.
+
+                let cursor_in_valid_pixels_of_sprite = 'valid_pixel: {
+                    match settings.picking_mode {
+                        SpritePickingMode::AlphaThreshold(cutoff) => {
+                            let Some(image) = images.get(&sprite.image) else {
+                                // [`Sprite::from_color`] returns a defaulted handle.
+                                // This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible
+                                break 'valid_pixel true;
+                            };
+                            // grab pixel and check alpha
+                            let Ok(color) = image.get_color_at(
+                                cursor_pixel_space.x as u32,
+                                cursor_pixel_space.y as u32,
+                            ) else {
+                                // We don't know how to interpret the pixel.
+                                break 'valid_pixel false;
+                            };
+                            // Check the alpha is above the cutoff.
+                            color.alpha() > cutoff
+                        }
+                        SpritePickingMode::BoundingBox => true,
+                    }
+                };
+
+                blocked = cursor_in_valid_pixels_of_sprite
                     && picking_behavior
                         .map(|p| p.should_block_lower)
                         .unwrap_or(true);
 
-                is_cursor_in_sprite.then(|| {
+                cursor_in_valid_pixels_of_sprite.then(|| {
                     let hit_pos_world =
                         sprite_transform.transform_point(cursor_pos_sprite.extend(0.0));
                     // Transform point from world to camera space to get the Z distance

crates/bevy_sprite/src/sprite.rs

@@ -1,13 +1,13 @@
-use bevy_asset::Handle;
+use bevy_asset::{Assets, Handle};
 use bevy_color::Color;
 use bevy_ecs::{component::Component, reflect::ReflectComponent};
 use bevy_image::Image;
-use bevy_math::{Rect, Vec2};
+use bevy_math::{Rect, UVec2, Vec2};
 use bevy_reflect::{std_traits::ReflectDefault, Reflect};
 use bevy_render::{sync_world::SyncToRenderWorld, view::Visibility};
 use bevy_transform::components::Transform;
 
-use crate::{TextureAtlas, TextureSlicer};
+use crate::{TextureAtlas, TextureAtlasLayout, TextureSlicer};
 
 /// Describes a sprite to be rendered to a 2D camera
 #[derive(Component, Debug, Default, Clone, Reflect)]
@@ -73,6 +73,73 @@ impl Sprite {
             ..Default::default()
         }
     }
+
+    /// Computes the pixel point where `point_relative_to_sprite` is sampled
+    /// from in this sprite. `point_relative_to_sprite` must be in the sprite's
+    /// local frame. Returns an Ok if the point is inside the bounds of the
+    /// sprite (not just the image), and returns an Err otherwise.
+    pub fn compute_pixel_space_point(
+        &self,
+        point_relative_to_sprite: Vec2,
+        images: &Assets<Image>,
+        texture_atlases: &Assets<TextureAtlasLayout>,
+    ) -> Result<Vec2, Vec2> {
+        let image_size = images
+            .get(&self.image)
+            .map(Image::size)
+            .unwrap_or(UVec2::ONE);
+
+        let atlas_rect = self
+            .texture_atlas
+            .as_ref()
+            .and_then(|s| s.texture_rect(texture_atlases))
+            .map(|r| r.as_rect());
+        let texture_rect = match (atlas_rect, self.rect) {
+            (None, None) => Rect::new(0.0, 0.0, image_size.x as f32, image_size.y as f32),
+            (None, Some(sprite_rect)) => sprite_rect,
+            (Some(atlas_rect), None) => atlas_rect,
+            (Some(atlas_rect), Some(mut sprite_rect)) => {
+                // Make the sprite rect relative to the atlas rect.
+                sprite_rect.min += atlas_rect.min;
+                sprite_rect.max += atlas_rect.min;
+                sprite_rect
+            }
+        };
+
+        let sprite_size = self.custom_size.unwrap_or_else(|| texture_rect.size());
+        let sprite_center = -self.anchor.as_vec() * sprite_size;
+
+        let mut point_relative_to_sprite_center = point_relative_to_sprite - sprite_center;
+
+        if self.flip_x {
+            point_relative_to_sprite_center.x *= -1.0;
+        }
+        // Texture coordinates start at the top left, whereas world coordinates start at the bottom
+        // left. So flip by default, and then don't flip if `flip_y` is set.
+        if !self.flip_y {
+            point_relative_to_sprite_center.y *= -1.0;
+        }
+
+        let sprite_to_texture_ratio = {
+            let texture_size = texture_rect.size();
+            let div_or_zero = |a, b| if b == 0.0 { 0.0 } else { a / b };
+            Vec2::new(
+                div_or_zero(texture_size.x, sprite_size.x),
+                div_or_zero(texture_size.y, sprite_size.y),
+            )
+        };
+
+        let point_relative_to_texture =
+            point_relative_to_sprite_center * sprite_to_texture_ratio + texture_rect.center();
+
+        // TODO: Support `SpriteImageMode`.
+
+        if texture_rect.contains(point_relative_to_texture) {
+            Ok(point_relative_to_texture)
+        } else {
+            Err(point_relative_to_texture)
+        }
+    }
 }
 
 impl From<Handle<Image>> for Sprite {
@@ -150,3 +217,265 @@ impl Anchor {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use bevy_asset::{Assets, RenderAssetUsages};
+    use bevy_color::Color;
+    use bevy_image::Image;
+    use bevy_math::{Rect, URect, UVec2, Vec2};
+    use bevy_render::render_resource::{Extent3d, TextureDimension, TextureFormat};
+
+    use crate::{Anchor, TextureAtlas, TextureAtlasLayout};
+
+    use super::Sprite;
+
+    /// Makes a new image of the specified size.
+    fn make_image(size: UVec2) -> Image {
+        Image::new_fill(
+            Extent3d {
+                width: size.x,
+                height: size.y,
+                depth_or_array_layers: 1,
+            },
+            TextureDimension::D2,
+            &[0, 0, 0, 255],
+            TextureFormat::Rgba8Unorm,
+            RenderAssetUsages::all(),
+        )
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_regular_sprite() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(-2.0, -4.5)), Ok(Vec2::new(0.5, 9.5)));
+        assert_eq!(compute(Vec2::new(0.0, 0.0)), Ok(Vec2::new(2.5, 5.0)));
+        assert_eq!(compute(Vec2::new(0.0, 4.5)), Ok(Vec2::new(2.5, 0.5)));
+        assert_eq!(compute(Vec2::new(3.0, 0.0)), Err(Vec2::new(5.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-3.0, 0.0)), Err(Vec2::new(-0.5, 5.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_color_sprite() {
+        let image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        // This also tests the `custom_size` field.
+        let sprite = Sprite::from_color(Color::BLACK, Vec2::new(50.0, 100.0));
+
+        let compute = |point| {
+            sprite
+                .compute_pixel_space_point(point, &image_assets, &texture_atlas_assets)
+                // Round to remove floating point errors.
+                .map(|x| (x * 1e5).round() / 1e5)
+                .map_err(|x| (x * 1e5).round() / 1e5)
+        };
+        assert_eq!(compute(Vec2::new(-20.0, -40.0)), Ok(Vec2::new(0.1, 0.9)));
+        assert_eq!(compute(Vec2::new(0.0, 10.0)), Ok(Vec2::new(0.5, 0.4)));
+        assert_eq!(compute(Vec2::new(75.0, 100.0)), Err(Vec2::new(2.0, -0.5)));
+        assert_eq!(compute(Vec2::new(-75.0, -100.0)), Err(Vec2::new(-1.0, 1.5)));
+        assert_eq!(compute(Vec2::new(-30.0, -40.0)), Err(Vec2::new(-0.1, 0.9)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_anchor_bottom_left() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::BottomLeft,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(0.5, 9.5)), Ok(Vec2::new(0.5, 0.5)));
+        assert_eq!(compute(Vec2::new(2.5, 5.0)), Ok(Vec2::new(2.5, 5.0)));
+        assert_eq!(compute(Vec2::new(2.5, 9.5)), Ok(Vec2::new(2.5, 0.5)));
+        assert_eq!(compute(Vec2::new(5.5, 5.0)), Err(Vec2::new(5.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-0.5, 5.0)), Err(Vec2::new(-0.5, 5.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_anchor_top_right() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::TopRight,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(-4.5, -0.5)), Ok(Vec2::new(0.5, 0.5)));
+        assert_eq!(compute(Vec2::new(-2.5, -5.0)), Ok(Vec2::new(2.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-2.5, -0.5)), Ok(Vec2::new(2.5, 0.5)));
+        assert_eq!(compute(Vec2::new(0.5, -5.0)), Err(Vec2::new(5.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-5.5, -5.0)), Err(Vec2::new(-0.5, 5.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_anchor_flip_x() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::BottomLeft,
+            flip_x: true,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(0.5, 9.5)), Ok(Vec2::new(4.5, 0.5)));
+        assert_eq!(compute(Vec2::new(2.5, 5.0)), Ok(Vec2::new(2.5, 5.0)));
+        assert_eq!(compute(Vec2::new(2.5, 9.5)), Ok(Vec2::new(2.5, 0.5)));
+        assert_eq!(compute(Vec2::new(5.5, 5.0)), Err(Vec2::new(-0.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-0.5, 5.0)), Err(Vec2::new(5.5, 5.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_anchor_flip_y() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::TopRight,
+            flip_y: true,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(-4.5, -0.5)), Ok(Vec2::new(0.5, 9.5)));
+        assert_eq!(compute(Vec2::new(-2.5, -5.0)), Ok(Vec2::new(2.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-2.5, -0.5)), Ok(Vec2::new(2.5, 9.5)));
+        assert_eq!(compute(Vec2::new(0.5, -5.0)), Err(Vec2::new(5.5, 5.0)));
+        assert_eq!(compute(Vec2::new(-5.5, -5.0)), Err(Vec2::new(-0.5, 5.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_rect() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            rect: Some(Rect::new(1.5, 3.0, 3.0, 9.5)),
+            anchor: Anchor::BottomLeft,
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(2.0, 9.0)));
+        // The pixel is outside the rect, but is still a valid pixel in the image.
+        assert_eq!(compute(Vec2::new(2.0, 2.5)), Err(Vec2::new(3.5, 7.0)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_texture_atlas_sprite() {
+        let mut image_assets = Assets::<Image>::default();
+        let mut texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+        let texture_atlas = texture_atlas_assets.add(TextureAtlasLayout {
+            size: UVec2::new(5, 10),
+            textures: vec![URect::new(1, 1, 4, 4)],
+        });
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::BottomLeft,
+            texture_atlas: Some(TextureAtlas {
+                layout: texture_atlas,
+                index: 0,
+            }),
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(1.5, 3.5)));
+        // The pixel is outside the texture atlas, but is still a valid pixel in the image.
+        assert_eq!(compute(Vec2::new(4.0, 2.5)), Err(Vec2::new(5.0, 1.5)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_texture_atlas_sprite_with_rect() {
+        let mut image_assets = Assets::<Image>::default();
+        let mut texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+        let texture_atlas = texture_atlas_assets.add(TextureAtlasLayout {
+            size: UVec2::new(5, 10),
+            textures: vec![URect::new(1, 1, 4, 4)],
+        });
+
+        let sprite = Sprite {
+            image,
+            anchor: Anchor::BottomLeft,
+            texture_atlas: Some(TextureAtlas {
+                layout: texture_atlas,
+                index: 0,
+            }),
+            // The rect is relative to the texture atlas sprite.
+            rect: Some(Rect::new(1.5, 1.5, 3.0, 3.0)),
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(3.0, 3.5)));
+        // The pixel is outside the texture atlas, but is still a valid pixel in the image.
+        assert_eq!(compute(Vec2::new(4.0, 2.5)), Err(Vec2::new(6.5, 1.5)));
+    }
+
+    #[test]
+    fn compute_pixel_space_point_for_sprite_with_custom_size_and_rect() {
+        let mut image_assets = Assets::<Image>::default();
+        let texture_atlas_assets = Assets::<TextureAtlasLayout>::default();
+
+        let image = image_assets.add(make_image(UVec2::new(5, 10)));
+
+        let sprite = Sprite {
+            image,
+            custom_size: Some(Vec2::new(100.0, 50.0)),
+            rect: Some(Rect::new(0.0, 0.0, 5.0, 5.0)),
+            ..Default::default()
+        };
+
+        let compute =
+            |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets);
+        assert_eq!(compute(Vec2::new(30.0, 15.0)), Ok(Vec2::new(4.0, 1.0)));
+        assert_eq!(compute(Vec2::new(-10.0, -15.0)), Ok(Vec2::new(2.0, 4.0)));
+        // The pixel is outside the texture atlas, but is still a valid pixel in the image.
+        assert_eq!(compute(Vec2::new(0.0, 35.0)), Err(Vec2::new(2.5, -1.0)));
+    }
+}