From 93dc596d2eb1ca8ad254fe1b5150d96b926df4d8 Mon Sep 17 00:00:00 2001 From: Michael Walter Van Der Velden Date: Tue, 3 Dec 2024 19:32:52 +0000 Subject: [PATCH] Add optional transparency passthrough for sprite backend with bevy_picking (#16388) # Objective - Allow bevy_sprite_picking backend to pass through transparent sections of the sprite. - Fixes #14929 ## Solution - After sprite picking detects the cursor is within a sprites rect, check the pixel at that location on the texture and check that it meets an optional transparency cutoff. Change originally created for mod_picking on bevy 0.14 (https://github.com/aevyrie/bevy_mod_picking/pull/373) ## Testing - Ran Sprite Picking example to check it was working both with transparency enabled and disabled - ModPicking version is currently in use in my own isometric game where this has been an extremely noticeable issue ## Showcase ![Sprite Picking Text](https://github.com/user-attachments/assets/76568c0d-c359-422b-942d-17c84d3d3009) ## Migration Guide Sprite picking now ignores transparent regions (with an alpha value less than or equal to 0.1). To configure this, modify the `SpriteBackendSettings` resource. --------- Co-authored-by: andriyDev --- crates/bevy_sprite/src/lib.rs | 4 +- crates/bevy_sprite/src/picking_backend.rs | 92 ++++-- crates/bevy_sprite/src/sprite.rs | 335 +++++++++++++++++++++- 3 files changed, 406 insertions(+), 25 deletions(-) diff --git a/crates/bevy_sprite/src/lib.rs b/crates/bevy_sprite/src/lib.rs index c6023b2c30..2d531e7859 100644 --- a/crates/bevy_sprite/src/lib.rs +++ b/crates/bevy_sprite/src/lib.rs @@ -41,6 +41,8 @@ use bevy_reflect::{std_traits::ReflectDefault, Reflect}; pub use bundle::*; pub use dynamic_texture_atlas_builder::*; pub use mesh2d::*; +#[cfg(feature = "bevy_sprite_picking_backend")] +pub use picking_backend::*; pub use render::*; pub use sprite::*; pub use texture_atlas::*; @@ -148,7 +150,7 @@ impl Plugin for SpritePlugin { #[cfg(feature = "bevy_sprite_picking_backend")] if self.add_picking { - app.add_plugins(picking_backend::SpritePickingPlugin); + app.add_plugins(SpritePickingPlugin); } if let Some(render_app) = app.get_sub_app_mut(RenderApp) { diff --git a/crates/bevy_sprite/src/picking_backend.rs b/crates/bevy_sprite/src/picking_backend.rs index b367efcc00..bd57aaf202 100644 --- a/crates/bevy_sprite/src/picking_backend.rs +++ b/crates/bevy_sprite/src/picking_backend.rs @@ -7,29 +7,63 @@ use core::cmp::Reverse; use crate::{Sprite, TextureAtlasLayout}; use bevy_app::prelude::*; use bevy_asset::prelude::*; +use bevy_color::Alpha; use bevy_ecs::prelude::*; use bevy_image::Image; use bevy_math::{prelude::*, FloatExt, FloatOrd}; use bevy_picking::backend::prelude::*; +use bevy_reflect::prelude::*; use bevy_render::prelude::*; use bevy_transform::prelude::*; use bevy_window::PrimaryWindow; +/// How should the [`SpritePickingPlugin`] handle picking and how should it handle transparent pixels +#[derive(Debug, Clone, Copy, Reflect)] +pub enum SpritePickingMode { + /// Even if a sprite is picked on a transparent pixel, it should still count within the backend. + /// Only consider the rect of a given sprite. + BoundingBox, + /// Ignore any part of a sprite which has a lower alpha value than the threshold (inclusive) + /// Threshold is given as an f32 representing the alpha value in a Bevy Color Value + AlphaThreshold(f32), +} + +/// Runtime settings for the [`SpritePickingPlugin`]. +#[derive(Resource, Reflect)] +#[reflect(Resource, Default)] +pub struct SpritePickingSettings { + /// Should the backend count transparent pixels as part of the sprite for picking purposes or should it use the bounding box of the sprite alone. + /// + /// Defaults to an incusive alpha threshold of 0.1 + pub picking_mode: SpritePickingMode, +} + +impl Default for SpritePickingSettings { + fn default() -> Self { + Self { + picking_mode: SpritePickingMode::AlphaThreshold(0.1), + } + } +} + #[derive(Clone)] pub struct SpritePickingPlugin; impl Plugin for SpritePickingPlugin { fn build(&self, app: &mut App) { - app.add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend)); + app.init_resource::() + .add_systems(PreUpdate, sprite_picking.in_set(PickSet::Backend)); } } -pub fn sprite_picking( +#[allow(clippy::too_many_arguments)] +fn sprite_picking( pointers: Query<(&PointerId, &PointerLocation)>, cameras: Query<(Entity, &Camera, &GlobalTransform, &OrthographicProjection)>, primary_window: Query>, images: Res>, texture_atlas_layout: Res>, + settings: Res, sprite_query: Query<( Entity, &Sprite, @@ -91,22 +125,6 @@ pub fn sprite_picking( return None; } - // Hit box in sprite coordinate system - let extents = match (sprite.custom_size, &sprite.texture_atlas) { - (Some(custom_size), _) => custom_size, - (None, None) => images.get(&sprite.image)?.size().as_vec2(), - (None, Some(atlas)) => texture_atlas_layout - .get(&atlas.layout) - .and_then(|layout| layout.textures.get(atlas.index)) - // Dropped atlas layouts and indexes out of bounds are rendered as a sprite - .map_or(images.get(&sprite.image)?.size().as_vec2(), |rect| { - rect.size().as_vec2() - }), - }; - let anchor = sprite.anchor.as_vec(); - let center = -anchor * extents; - let rect = Rect::from_center_half_size(center, extents / 2.0); - // Transform cursor line segment to sprite coordinate system let world_to_sprite = sprite_transform.affine().inverse(); let cursor_start_sprite = world_to_sprite.transform_point3(cursor_ray_world.origin); @@ -133,14 +151,46 @@ pub fn sprite_picking( .lerp(cursor_end_sprite, lerp_factor) .xy(); - let is_cursor_in_sprite = rect.contains(cursor_pos_sprite); + let Ok(cursor_pixel_space) = sprite.compute_pixel_space_point( + cursor_pos_sprite, + &images, + &texture_atlas_layout, + ) else { + return None; + }; - blocked = is_cursor_in_sprite + // Since the pixel space coordinate is `Ok`, we know the cursor is in the bounds of + // the sprite. + + let cursor_in_valid_pixels_of_sprite = 'valid_pixel: { + match settings.picking_mode { + SpritePickingMode::AlphaThreshold(cutoff) => { + let Some(image) = images.get(&sprite.image) else { + // [`Sprite::from_color`] returns a defaulted handle. + // This handle doesn't return a valid image, so returning false here would make picking "color sprites" impossible + break 'valid_pixel true; + }; + // grab pixel and check alpha + let Ok(color) = image.get_color_at( + cursor_pixel_space.x as u32, + cursor_pixel_space.y as u32, + ) else { + // We don't know how to interpret the pixel. + break 'valid_pixel false; + }; + // Check the alpha is above the cutoff. + color.alpha() > cutoff + } + SpritePickingMode::BoundingBox => true, + } + }; + + blocked = cursor_in_valid_pixels_of_sprite && picking_behavior .map(|p| p.should_block_lower) .unwrap_or(true); - is_cursor_in_sprite.then(|| { + cursor_in_valid_pixels_of_sprite.then(|| { let hit_pos_world = sprite_transform.transform_point(cursor_pos_sprite.extend(0.0)); // Transform point from world to camera space to get the Z distance diff --git a/crates/bevy_sprite/src/sprite.rs b/crates/bevy_sprite/src/sprite.rs index f6b8b266d5..5305d023b4 100644 --- a/crates/bevy_sprite/src/sprite.rs +++ b/crates/bevy_sprite/src/sprite.rs @@ -1,13 +1,13 @@ -use bevy_asset::Handle; +use bevy_asset::{Assets, Handle}; use bevy_color::Color; use bevy_ecs::{component::Component, reflect::ReflectComponent}; use bevy_image::Image; -use bevy_math::{Rect, Vec2}; +use bevy_math::{Rect, UVec2, Vec2}; use bevy_reflect::{std_traits::ReflectDefault, Reflect}; use bevy_render::{sync_world::SyncToRenderWorld, view::Visibility}; use bevy_transform::components::Transform; -use crate::{TextureAtlas, TextureSlicer}; +use crate::{TextureAtlas, TextureAtlasLayout, TextureSlicer}; /// Describes a sprite to be rendered to a 2D camera #[derive(Component, Debug, Default, Clone, Reflect)] @@ -73,6 +73,73 @@ impl Sprite { ..Default::default() } } + + /// Computes the pixel point where `point_relative_to_sprite` is sampled + /// from in this sprite. `point_relative_to_sprite` must be in the sprite's + /// local frame. Returns an Ok if the point is inside the bounds of the + /// sprite (not just the image), and returns an Err otherwise. + pub fn compute_pixel_space_point( + &self, + point_relative_to_sprite: Vec2, + images: &Assets, + texture_atlases: &Assets, + ) -> Result { + let image_size = images + .get(&self.image) + .map(Image::size) + .unwrap_or(UVec2::ONE); + + let atlas_rect = self + .texture_atlas + .as_ref() + .and_then(|s| s.texture_rect(texture_atlases)) + .map(|r| r.as_rect()); + let texture_rect = match (atlas_rect, self.rect) { + (None, None) => Rect::new(0.0, 0.0, image_size.x as f32, image_size.y as f32), + (None, Some(sprite_rect)) => sprite_rect, + (Some(atlas_rect), None) => atlas_rect, + (Some(atlas_rect), Some(mut sprite_rect)) => { + // Make the sprite rect relative to the atlas rect. + sprite_rect.min += atlas_rect.min; + sprite_rect.max += atlas_rect.min; + sprite_rect + } + }; + + let sprite_size = self.custom_size.unwrap_or_else(|| texture_rect.size()); + let sprite_center = -self.anchor.as_vec() * sprite_size; + + let mut point_relative_to_sprite_center = point_relative_to_sprite - sprite_center; + + if self.flip_x { + point_relative_to_sprite_center.x *= -1.0; + } + // Texture coordinates start at the top left, whereas world coordinates start at the bottom + // left. So flip by default, and then don't flip if `flip_y` is set. + if !self.flip_y { + point_relative_to_sprite_center.y *= -1.0; + } + + let sprite_to_texture_ratio = { + let texture_size = texture_rect.size(); + let div_or_zero = |a, b| if b == 0.0 { 0.0 } else { a / b }; + Vec2::new( + div_or_zero(texture_size.x, sprite_size.x), + div_or_zero(texture_size.y, sprite_size.y), + ) + }; + + let point_relative_to_texture = + point_relative_to_sprite_center * sprite_to_texture_ratio + texture_rect.center(); + + // TODO: Support `SpriteImageMode`. + + if texture_rect.contains(point_relative_to_texture) { + Ok(point_relative_to_texture) + } else { + Err(point_relative_to_texture) + } + } } impl From> for Sprite { @@ -150,3 +217,265 @@ impl Anchor { } } } + +#[cfg(test)] +mod tests { + use bevy_asset::{Assets, RenderAssetUsages}; + use bevy_color::Color; + use bevy_image::Image; + use bevy_math::{Rect, URect, UVec2, Vec2}; + use bevy_render::render_resource::{Extent3d, TextureDimension, TextureFormat}; + + use crate::{Anchor, TextureAtlas, TextureAtlasLayout}; + + use super::Sprite; + + /// Makes a new image of the specified size. + fn make_image(size: UVec2) -> Image { + Image::new_fill( + Extent3d { + width: size.x, + height: size.y, + depth_or_array_layers: 1, + }, + TextureDimension::D2, + &[0, 0, 0, 255], + TextureFormat::Rgba8Unorm, + RenderAssetUsages::all(), + ) + } + + #[test] + fn compute_pixel_space_point_for_regular_sprite() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(-2.0, -4.5)), Ok(Vec2::new(0.5, 9.5))); + assert_eq!(compute(Vec2::new(0.0, 0.0)), Ok(Vec2::new(2.5, 5.0))); + assert_eq!(compute(Vec2::new(0.0, 4.5)), Ok(Vec2::new(2.5, 0.5))); + assert_eq!(compute(Vec2::new(3.0, 0.0)), Err(Vec2::new(5.5, 5.0))); + assert_eq!(compute(Vec2::new(-3.0, 0.0)), Err(Vec2::new(-0.5, 5.0))); + } + + #[test] + fn compute_pixel_space_point_for_color_sprite() { + let image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + // This also tests the `custom_size` field. + let sprite = Sprite::from_color(Color::BLACK, Vec2::new(50.0, 100.0)); + + let compute = |point| { + sprite + .compute_pixel_space_point(point, &image_assets, &texture_atlas_assets) + // Round to remove floating point errors. + .map(|x| (x * 1e5).round() / 1e5) + .map_err(|x| (x * 1e5).round() / 1e5) + }; + assert_eq!(compute(Vec2::new(-20.0, -40.0)), Ok(Vec2::new(0.1, 0.9))); + assert_eq!(compute(Vec2::new(0.0, 10.0)), Ok(Vec2::new(0.5, 0.4))); + assert_eq!(compute(Vec2::new(75.0, 100.0)), Err(Vec2::new(2.0, -0.5))); + assert_eq!(compute(Vec2::new(-75.0, -100.0)), Err(Vec2::new(-1.0, 1.5))); + assert_eq!(compute(Vec2::new(-30.0, -40.0)), Err(Vec2::new(-0.1, 0.9))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_anchor_bottom_left() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + anchor: Anchor::BottomLeft, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(0.5, 9.5)), Ok(Vec2::new(0.5, 0.5))); + assert_eq!(compute(Vec2::new(2.5, 5.0)), Ok(Vec2::new(2.5, 5.0))); + assert_eq!(compute(Vec2::new(2.5, 9.5)), Ok(Vec2::new(2.5, 0.5))); + assert_eq!(compute(Vec2::new(5.5, 5.0)), Err(Vec2::new(5.5, 5.0))); + assert_eq!(compute(Vec2::new(-0.5, 5.0)), Err(Vec2::new(-0.5, 5.0))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_anchor_top_right() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + anchor: Anchor::TopRight, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(-4.5, -0.5)), Ok(Vec2::new(0.5, 0.5))); + assert_eq!(compute(Vec2::new(-2.5, -5.0)), Ok(Vec2::new(2.5, 5.0))); + assert_eq!(compute(Vec2::new(-2.5, -0.5)), Ok(Vec2::new(2.5, 0.5))); + assert_eq!(compute(Vec2::new(0.5, -5.0)), Err(Vec2::new(5.5, 5.0))); + assert_eq!(compute(Vec2::new(-5.5, -5.0)), Err(Vec2::new(-0.5, 5.0))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_anchor_flip_x() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + anchor: Anchor::BottomLeft, + flip_x: true, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(0.5, 9.5)), Ok(Vec2::new(4.5, 0.5))); + assert_eq!(compute(Vec2::new(2.5, 5.0)), Ok(Vec2::new(2.5, 5.0))); + assert_eq!(compute(Vec2::new(2.5, 9.5)), Ok(Vec2::new(2.5, 0.5))); + assert_eq!(compute(Vec2::new(5.5, 5.0)), Err(Vec2::new(-0.5, 5.0))); + assert_eq!(compute(Vec2::new(-0.5, 5.0)), Err(Vec2::new(5.5, 5.0))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_anchor_flip_y() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + anchor: Anchor::TopRight, + flip_y: true, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(-4.5, -0.5)), Ok(Vec2::new(0.5, 9.5))); + assert_eq!(compute(Vec2::new(-2.5, -5.0)), Ok(Vec2::new(2.5, 5.0))); + assert_eq!(compute(Vec2::new(-2.5, -0.5)), Ok(Vec2::new(2.5, 9.5))); + assert_eq!(compute(Vec2::new(0.5, -5.0)), Err(Vec2::new(5.5, 5.0))); + assert_eq!(compute(Vec2::new(-5.5, -5.0)), Err(Vec2::new(-0.5, 5.0))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_rect() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + rect: Some(Rect::new(1.5, 3.0, 3.0, 9.5)), + anchor: Anchor::BottomLeft, + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(2.0, 9.0))); + // The pixel is outside the rect, but is still a valid pixel in the image. + assert_eq!(compute(Vec2::new(2.0, 2.5)), Err(Vec2::new(3.5, 7.0))); + } + + #[test] + fn compute_pixel_space_point_for_texture_atlas_sprite() { + let mut image_assets = Assets::::default(); + let mut texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + let texture_atlas = texture_atlas_assets.add(TextureAtlasLayout { + size: UVec2::new(5, 10), + textures: vec![URect::new(1, 1, 4, 4)], + }); + + let sprite = Sprite { + image, + anchor: Anchor::BottomLeft, + texture_atlas: Some(TextureAtlas { + layout: texture_atlas, + index: 0, + }), + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(1.5, 3.5))); + // The pixel is outside the texture atlas, but is still a valid pixel in the image. + assert_eq!(compute(Vec2::new(4.0, 2.5)), Err(Vec2::new(5.0, 1.5))); + } + + #[test] + fn compute_pixel_space_point_for_texture_atlas_sprite_with_rect() { + let mut image_assets = Assets::::default(); + let mut texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + let texture_atlas = texture_atlas_assets.add(TextureAtlasLayout { + size: UVec2::new(5, 10), + textures: vec![URect::new(1, 1, 4, 4)], + }); + + let sprite = Sprite { + image, + anchor: Anchor::BottomLeft, + texture_atlas: Some(TextureAtlas { + layout: texture_atlas, + index: 0, + }), + // The rect is relative to the texture atlas sprite. + rect: Some(Rect::new(1.5, 1.5, 3.0, 3.0)), + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(0.5, 0.5)), Ok(Vec2::new(3.0, 3.5))); + // The pixel is outside the texture atlas, but is still a valid pixel in the image. + assert_eq!(compute(Vec2::new(4.0, 2.5)), Err(Vec2::new(6.5, 1.5))); + } + + #[test] + fn compute_pixel_space_point_for_sprite_with_custom_size_and_rect() { + let mut image_assets = Assets::::default(); + let texture_atlas_assets = Assets::::default(); + + let image = image_assets.add(make_image(UVec2::new(5, 10))); + + let sprite = Sprite { + image, + custom_size: Some(Vec2::new(100.0, 50.0)), + rect: Some(Rect::new(0.0, 0.0, 5.0, 5.0)), + ..Default::default() + }; + + let compute = + |point| sprite.compute_pixel_space_point(point, &image_assets, &texture_atlas_assets); + assert_eq!(compute(Vec2::new(30.0, 15.0)), Ok(Vec2::new(4.0, 1.0))); + assert_eq!(compute(Vec2::new(-10.0, -15.0)), Ok(Vec2::new(2.0, 4.0))); + // The pixel is outside the texture atlas, but is still a valid pixel in the image. + assert_eq!(compute(Vec2::new(0.0, 35.0)), Err(Vec2::new(2.5, -1.0))); + } +}