From 98938a8555eb414ee5e9b023c45fe3b5f202edc8 Mon Sep 17 00:00:00 2001 From: Carter Anderson Date: Fri, 18 Feb 2022 22:56:57 +0000 Subject: [PATCH] Internal Asset Hot Reloading (#3966) Adds "hot reloading" of internal assets, which is normally not possible because they are loaded using `include_str` / direct Asset collection access. This is accomplished via the following: * Add a new `debug_asset_server` feature flag * When that feature flag is enabled, create a second App with a second AssetServer that points to a configured location (by default the `crates` folder). Plugins that want to add hot reloading support for their assets can call the new `app.add_debug_asset::()` and `app.init_debug_asset_loader::()` functions. * Load "internal" assets using the new `load_internal_asset` macro. By default this is identical to the current "include_str + register in asset collection" approach. But if the `debug_asset_server` feature flag is enabled, it will also load the asset dynamically in the debug asset server using the file path. It will then set up a correlation between the "debug asset" and the "actual asset" by listening for asset change events. This is an alternative to #3673. The goal was to keep the boilerplate and features flags to a minimum for bevy plugin authors, and allow them to home their shaders near relevant code. This is a draft because I haven't done _any_ quality control on this yet. I'll probably rename things and remove a bunch of unwraps. I just got it working and wanted to use it to start a conversation. Fixes #3660 --- Cargo.toml | 3 + crates/bevy_asset/Cargo.toml | 1 + crates/bevy_asset/src/asset_server.rs | 4 + crates/bevy_asset/src/assets.rs | 75 ++++++++++ crates/bevy_asset/src/debug_asset_server.rs | 138 ++++++++++++++++++ crates/bevy_asset/src/io/file_asset_io.rs | 4 + crates/bevy_asset/src/lib.rs | 2 + crates/bevy_internal/Cargo.toml | 1 + crates/bevy_internal/src/default_plugins.rs | 2 + crates/bevy_pbr/src/lib.rs | 14 +- crates/bevy_pbr/src/render/mesh.rs | 20 +-- crates/bevy_pbr/src/wireframe.rs | 9 +- crates/bevy_render/src/lib.rs | 2 + .../bevy_sprite/src/mesh2d/color_material.rs | 9 +- crates/bevy_sprite/src/mesh2d/mesh.rs | 20 +-- crates/bevy_ui/src/render/mod.rs | 6 +- docs/cargo_features.md | 1 + 17 files changed, 271 insertions(+), 40 deletions(-) create mode 100644 crates/bevy_asset/src/debug_asset_server.rs diff --git a/Cargo.toml b/Cargo.toml index ef6cb40a7d..eb7cab0be8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,6 +90,9 @@ subpixel_glyph_atlas = ["bevy_internal/subpixel_glyph_atlas"] # Enable systems that allow for automated testing on CI bevy_ci_testing = ["bevy_internal/bevy_ci_testing"] +# Enable the "debug asset server" for hot reloading internal assets +debug_asset_server = ["bevy_internal/debug_asset_server"] + [dependencies] bevy_dylib = { path = "crates/bevy_dylib", version = "0.6.0", default-features = false, optional = true } bevy_internal = { path = "crates/bevy_internal", version = "0.6.0", default-features = false } diff --git a/crates/bevy_asset/Cargo.toml b/crates/bevy_asset/Cargo.toml index 7ba0cece7b..e5f864e0ee 100644 --- a/crates/bevy_asset/Cargo.toml +++ b/crates/bevy_asset/Cargo.toml @@ -11,6 +11,7 @@ keywords = ["bevy"] [features] default = [] filesystem_watcher = ["notify"] +debug_asset_server = ["filesystem_watcher"] [dependencies] # bevy diff --git a/crates/bevy_asset/src/asset_server.rs b/crates/bevy_asset/src/asset_server.rs index 97296abc62..c1656368b5 100644 --- a/crates/bevy_asset/src/asset_server.rs +++ b/crates/bevy_asset/src/asset_server.rs @@ -92,6 +92,10 @@ impl AssetServer { } } + pub fn asset_io(&self) -> &dyn AssetIo { + &*self.server.asset_io + } + pub(crate) fn register_asset_type(&self) -> Assets { if self .server diff --git a/crates/bevy_asset/src/assets.rs b/crates/bevy_asset/src/assets.rs index a74bb19155..a848132363 100644 --- a/crates/bevy_asset/src/assets.rs +++ b/crates/bevy_asset/src/assets.rs @@ -259,9 +259,15 @@ impl Assets { /// [App] extension methods for adding new asset types pub trait AddAsset { fn add_asset(&mut self) -> &mut Self + where + T: Asset; + fn add_debug_asset(&mut self) -> &mut Self where T: Asset; fn init_asset_loader(&mut self) -> &mut Self + where + T: AssetLoader + FromWorld; + fn init_debug_asset_loader(&mut self) -> &mut Self where T: AssetLoader + FromWorld; fn add_asset_loader(&mut self, loader: T) -> &mut Self @@ -292,6 +298,23 @@ impl AddAsset for App { .add_event::>() } + fn add_debug_asset(&mut self) -> &mut Self + where + T: Asset, + { + #[cfg(feature = "debug_asset_server")] + { + self.add_system(crate::debug_asset_server::sync_debug_assets::); + let mut app = self + .world + .get_non_send_resource_mut::() + .unwrap(); + app.add_asset::() + .init_resource::>(); + } + self + } + fn init_asset_loader(&mut self) -> &mut Self where T: AssetLoader + FromWorld, @@ -300,6 +323,21 @@ impl AddAsset for App { self.add_asset_loader(result) } + fn init_debug_asset_loader(&mut self) -> &mut Self + where + T: AssetLoader + FromWorld, + { + #[cfg(feature = "debug_asset_server")] + { + let mut app = self + .world + .get_non_send_resource_mut::() + .unwrap(); + app.init_asset_loader::(); + } + self + } + fn add_asset_loader(&mut self, loader: T) -> &mut Self where T: AssetLoader, @@ -312,6 +350,43 @@ impl AddAsset for App { } } +#[cfg(feature = "debug_asset_server")] +#[macro_export] +macro_rules! load_internal_asset { + ($app: ident, $handle: ident, $path_str: expr, $loader: expr) => {{ + { + let mut debug_app = $app + .world + .get_non_send_resource_mut::() + .unwrap(); + bevy_asset::debug_asset_server::register_handle_with_loader( + $loader, + &mut debug_app, + $handle, + file!(), + $path_str, + ); + } + let mut assets = $app + .world + .get_resource_mut::>() + .unwrap(); + assets.set_untracked($handle, ($loader)(include_str!($path_str))); + }}; +} + +#[cfg(not(feature = "debug_asset_server"))] +#[macro_export] +macro_rules! load_internal_asset { + ($app: ident, $handle: ident, $path_str: expr, $loader: expr) => {{ + let mut assets = $app + .world + .get_resource_mut::>() + .unwrap(); + assets.set_untracked($handle, ($loader)(include_str!($path_str))); + }}; +} + #[cfg(test)] mod tests { use bevy_app::App; diff --git a/crates/bevy_asset/src/debug_asset_server.rs b/crates/bevy_asset/src/debug_asset_server.rs new file mode 100644 index 0000000000..c7907644da --- /dev/null +++ b/crates/bevy_asset/src/debug_asset_server.rs @@ -0,0 +1,138 @@ +use bevy_app::{App, Events, Plugin}; +use bevy_ecs::{ + schedule::SystemLabel, + system::{NonSendMut, Res, ResMut, SystemState}, +}; +use bevy_tasks::{IoTaskPool, TaskPoolBuilder}; +use bevy_utils::HashMap; +use std::{ + ops::{Deref, DerefMut}, + path::Path, +}; + +use crate::{ + Asset, AssetEvent, AssetPlugin, AssetServer, AssetServerSettings, Assets, FileAssetIo, Handle, + HandleUntyped, +}; + +/// A "debug asset app", whose sole responsibility is hot reloading assets that are +/// "internal" / compiled-in to Bevy Plugins. +pub struct DebugAssetApp(App); + +impl Deref for DebugAssetApp { + type Target = App; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for DebugAssetApp { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[derive(SystemLabel, Debug, Clone, PartialEq, Eq, Hash)] +pub struct DebugAssetAppRun; + +/// Facilitates the creation of a "debug asset app", whose sole responsibility is hot reloading +/// assets that are "internal" / compiled-in to Bevy Plugins. +/// Pair with [`load_internal_asset`](crate::load_internal_asset) to load "hot reloadable" assets +/// The `debug_asset_server` feature flag must also be enabled for hot reloading to work. +/// Currently only hot reloads assets stored in the `crates` folder. +#[derive(Default)] +pub struct DebugAssetServerPlugin; +pub struct HandleMap { + pub handles: HashMap, Handle>, +} + +impl Default for HandleMap { + fn default() -> Self { + Self { + handles: Default::default(), + } + } +} + +impl Plugin for DebugAssetServerPlugin { + fn build(&self, app: &mut bevy_app::App) { + let mut debug_asset_app = App::new(); + debug_asset_app + .insert_resource(IoTaskPool( + TaskPoolBuilder::default() + .num_threads(2) + .thread_name("Debug Asset Server IO Task Pool".to_string()) + .build(), + )) + .insert_resource(AssetServerSettings { + asset_folder: "crates".to_string(), + watch_for_changes: true, + }) + .add_plugin(AssetPlugin); + app.insert_non_send_resource(DebugAssetApp(debug_asset_app)); + app.add_system(run_debug_asset_app); + } +} + +fn run_debug_asset_app(mut debug_asset_app: NonSendMut) { + debug_asset_app.0.update(); +} + +pub(crate) fn sync_debug_assets( + mut debug_asset_app: NonSendMut, + mut assets: ResMut>, +) { + let world = &mut debug_asset_app.0.world; + let mut state = SystemState::<( + Res>>, + Res>, + Res>, + )>::new(world); + let (changed_shaders, handle_map, debug_assets) = state.get_mut(world); + for changed in changed_shaders.iter_current_update_events() { + let debug_handle = match changed { + AssetEvent::Created { handle } => handle, + AssetEvent::Modified { handle } => handle, + AssetEvent::Removed { .. } => continue, + }; + if let Some(handle) = handle_map.handles.get(debug_handle) { + if let Some(debug_asset) = debug_assets.get(debug_handle) { + assets.set_untracked(handle, debug_asset.clone()); + } + } + } +} + +/// Uses the return type of the given loader to register the given handle with the appropriate type +/// and load the asset with the given `path` and parent `file_path`. +/// If this feels a bit odd ... thats because it is. This was built to improve the UX of the +/// `load_internal_asset` macro. +pub fn register_handle_with_loader( + _loader: fn(&'static str) -> A, + app: &mut DebugAssetApp, + handle: HandleUntyped, + file_path: &str, + path: &'static str, +) { + let mut state = SystemState::<(ResMut>, Res)>::new(&mut app.world); + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").unwrap(); + let manifest_dir_path = Path::new(&manifest_dir); + let (mut handle_map, asset_server) = state.get_mut(&mut app.world); + let asset_io = asset_server + .asset_io() + .downcast_ref::() + .expect("The debug AssetServer only works with FileAssetIo-backed AssetServers"); + let absolute_file_path = manifest_dir_path.join( + Path::new(file_path) + .parent() + .expect("file path must have a parent"), + ); + let asset_folder_relative_path = absolute_file_path + .strip_prefix(asset_io.root_path()) + .expect("The AssetIo root path should be a prefix of the absolute file path"); + handle_map.handles.insert( + asset_server.load(asset_folder_relative_path.join(path)), + handle.clone_weak().typed::(), + ); +} diff --git a/crates/bevy_asset/src/io/file_asset_io.rs b/crates/bevy_asset/src/io/file_asset_io.rs index 3888380805..1b5b1630b2 100644 --- a/crates/bevy_asset/src/io/file_asset_io.rs +++ b/crates/bevy_asset/src/io/file_asset_io.rs @@ -62,6 +62,10 @@ impl FileAssetIo { .unwrap() } } + + pub fn root_path(&self) -> &PathBuf { + &self.root_path + } } impl AssetIo for FileAssetIo { diff --git a/crates/bevy_asset/src/lib.rs b/crates/bevy_asset/src/lib.rs index b62d7537f2..6e31a62ea6 100644 --- a/crates/bevy_asset/src/lib.rs +++ b/crates/bevy_asset/src/lib.rs @@ -1,5 +1,7 @@ mod asset_server; mod assets; +#[cfg(feature = "debug_asset_server")] +pub mod debug_asset_server; pub mod diagnostic; #[cfg(all( feature = "filesystem_watcher", diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index 6763b3117c..6153c0c612 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -14,6 +14,7 @@ trace = [ "bevy_app/trace", "bevy_ecs/trace", "bevy_render/trace" ] trace_chrome = [ "bevy_log/tracing-chrome" ] trace_tracy = [ "bevy_log/tracing-tracy" ] wgpu_trace = ["bevy_render/wgpu_trace"] +debug_asset_server = ["bevy_asset/debug_asset_server"] # Image format support for texture loading (PNG and HDR are enabled by default) hdr = ["bevy_render/hdr"] diff --git a/crates/bevy_internal/src/default_plugins.rs b/crates/bevy_internal/src/default_plugins.rs index c4a102588d..76ccb2954c 100644 --- a/crates/bevy_internal/src/default_plugins.rs +++ b/crates/bevy_internal/src/default_plugins.rs @@ -31,6 +31,8 @@ impl PluginGroup for DefaultPlugins { group.add(bevy_input::InputPlugin::default()); group.add(bevy_window::WindowPlugin::default()); group.add(bevy_asset::AssetPlugin::default()); + #[cfg(feature = "debug_asset_server")] + group.add(bevy_asset::debug_asset_server::DebugAssetServerPlugin::default()); group.add(bevy_scene::ScenePlugin::default()); #[cfg(feature = "bevy_winit")] diff --git a/crates/bevy_pbr/src/lib.rs b/crates/bevy_pbr/src/lib.rs index 8325030519..723c1e6818 100644 --- a/crates/bevy_pbr/src/lib.rs +++ b/crates/bevy_pbr/src/lib.rs @@ -33,7 +33,7 @@ pub mod draw_3d_graph { } use bevy_app::prelude::*; -use bevy_asset::{Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, Assets, Handle, HandleUntyped}; use bevy_ecs::prelude::*; use bevy_reflect::TypeUuid; use bevy_render::{ @@ -57,14 +57,12 @@ pub struct PbrPlugin; impl Plugin for PbrPlugin { fn build(&self, app: &mut App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - shaders.set_untracked( - PBR_SHADER_HANDLE, - Shader::from_wgsl(include_str!("render/pbr.wgsl")), - ); - shaders.set_untracked( + load_internal_asset!(app, PBR_SHADER_HANDLE, "render/pbr.wgsl", Shader::from_wgsl); + load_internal_asset!( + app, SHADOW_SHADER_HANDLE, - Shader::from_wgsl(include_str!("render/depth.wgsl")), + "render/depth.wgsl", + Shader::from_wgsl ); app.register_type::() diff --git a/crates/bevy_pbr/src/render/mesh.rs b/crates/bevy_pbr/src/render/mesh.rs index 546b09454f..942945cf83 100644 --- a/crates/bevy_pbr/src/render/mesh.rs +++ b/crates/bevy_pbr/src/render/mesh.rs @@ -3,7 +3,7 @@ use crate::{ ViewClusterBindings, ViewLightsUniformOffset, ViewShadowBindings, }; use bevy_app::Plugin; -use bevy_asset::{Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, Handle, HandleUntyped}; use bevy_ecs::{ prelude::*, system::{lifetimeless::*, SystemParamItem}, @@ -35,18 +35,18 @@ pub const MESH_SHADER_HANDLE: HandleUntyped = impl Plugin for MeshRenderPlugin { fn build(&self, app: &mut bevy_app::App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - shaders.set_untracked( - MESH_SHADER_HANDLE, - Shader::from_wgsl(include_str!("mesh.wgsl")), - ); - shaders.set_untracked( + load_internal_asset!(app, MESH_SHADER_HANDLE, "mesh.wgsl", Shader::from_wgsl); + load_internal_asset!( + app, MESH_STRUCT_HANDLE, - Shader::from_wgsl(include_str!("mesh_struct.wgsl")), + "mesh_struct.wgsl", + Shader::from_wgsl ); - shaders.set_untracked( + load_internal_asset!( + app, MESH_VIEW_BIND_GROUP_HANDLE, - Shader::from_wgsl(include_str!("mesh_view_bind_group.wgsl")), + "mesh_view_bind_group.wgsl", + Shader::from_wgsl ); app.add_plugin(UniformComponentPlugin::::default()); diff --git a/crates/bevy_pbr/src/wireframe.rs b/crates/bevy_pbr/src/wireframe.rs index 9f1308179e..93a74d79f2 100644 --- a/crates/bevy_pbr/src/wireframe.rs +++ b/crates/bevy_pbr/src/wireframe.rs @@ -1,7 +1,7 @@ use crate::MeshPipeline; use crate::{DrawMesh, MeshPipelineKey, MeshUniform, SetMeshBindGroup, SetMeshViewBindGroup}; use bevy_app::Plugin; -use bevy_asset::{Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, Handle, HandleUntyped}; use bevy_core_pipeline::Opaque3d; use bevy_ecs::{prelude::*, reflect::ReflectComponent}; use bevy_reflect::{Reflect, TypeUuid}; @@ -23,10 +23,11 @@ pub struct WireframePlugin; impl Plugin for WireframePlugin { fn build(&self, app: &mut bevy_app::App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - shaders.set_untracked( + load_internal_asset!( + app, WIREFRAME_SHADER_HANDLE, - Shader::from_wgsl(include_str!("render/wireframe.wgsl")), + "render/wireframe.wgsl", + Shader::from_wgsl ); app.init_resource::(); diff --git a/crates/bevy_render/src/lib.rs b/crates/bevy_render/src/lib.rs index d44cb38ee8..4222a29cf8 100644 --- a/crates/bevy_render/src/lib.rs +++ b/crates/bevy_render/src/lib.rs @@ -115,7 +115,9 @@ impl Plugin for RenderPlugin { .unwrap_or_default(); app.add_asset::() + .add_debug_asset::() .init_asset_loader::() + .init_debug_asset_loader::() .register_type::(); if let Some(backends) = options.backends { diff --git a/crates/bevy_sprite/src/mesh2d/color_material.rs b/crates/bevy_sprite/src/mesh2d/color_material.rs index eed38c3f14..cfe1f22c47 100644 --- a/crates/bevy_sprite/src/mesh2d/color_material.rs +++ b/crates/bevy_sprite/src/mesh2d/color_material.rs @@ -1,5 +1,5 @@ use bevy_app::{App, Plugin}; -use bevy_asset::{AssetServer, Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, AssetServer, Assets, Handle, HandleUntyped}; use bevy_ecs::system::{lifetimeless::SRes, SystemParamItem}; use bevy_math::Vec4; use bevy_reflect::TypeUuid; @@ -25,10 +25,11 @@ pub struct ColorMaterialPlugin; impl Plugin for ColorMaterialPlugin { fn build(&self, app: &mut App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - shaders.set_untracked( + load_internal_asset!( + app, COLOR_MATERIAL_SHADER_HANDLE, - Shader::from_wgsl(include_str!("color_material.wgsl")), + "color_material.wgsl", + Shader::from_wgsl ); app.add_plugin(Material2dPlugin::::default()); diff --git a/crates/bevy_sprite/src/mesh2d/mesh.rs b/crates/bevy_sprite/src/mesh2d/mesh.rs index a375561ed9..aaf4666bdf 100644 --- a/crates/bevy_sprite/src/mesh2d/mesh.rs +++ b/crates/bevy_sprite/src/mesh2d/mesh.rs @@ -1,5 +1,5 @@ use bevy_app::Plugin; -use bevy_asset::{Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, Handle, HandleUntyped}; use bevy_ecs::{ prelude::*, system::{lifetimeless::*, SystemParamItem}, @@ -43,18 +43,18 @@ pub const MESH2D_SHADER_HANDLE: HandleUntyped = impl Plugin for Mesh2dRenderPlugin { fn build(&self, app: &mut bevy_app::App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - shaders.set_untracked( - MESH2D_SHADER_HANDLE, - Shader::from_wgsl(include_str!("mesh2d.wgsl")), - ); - shaders.set_untracked( + load_internal_asset!(app, MESH2D_SHADER_HANDLE, "mesh2d.wgsl", Shader::from_wgsl); + load_internal_asset!( + app, MESH2D_STRUCT_HANDLE, - Shader::from_wgsl(include_str!("mesh2d_struct.wgsl")), + "mesh2d_struct.wgsl", + Shader::from_wgsl ); - shaders.set_untracked( + load_internal_asset!( + app, MESH2D_VIEW_BIND_GROUP_HANDLE, - Shader::from_wgsl(include_str!("mesh2d_view_bind_group.wgsl")), + "mesh2d_view_bind_group.wgsl", + Shader::from_wgsl ); app.add_plugin(UniformComponentPlugin::::default()); diff --git a/crates/bevy_ui/src/render/mod.rs b/crates/bevy_ui/src/render/mod.rs index 947eda82c7..3644d1037f 100644 --- a/crates/bevy_ui/src/render/mod.rs +++ b/crates/bevy_ui/src/render/mod.rs @@ -9,7 +9,7 @@ pub use render_pass::*; use std::ops::Range; use bevy_app::prelude::*; -use bevy_asset::{AssetEvent, Assets, Handle, HandleUntyped}; +use bevy_asset::{load_internal_asset, AssetEvent, Assets, Handle, HandleUntyped}; use bevy_core::FloatOrd; use bevy_ecs::prelude::*; use bevy_math::{const_vec3, Mat4, Vec2, Vec3, Vec4Swizzles}; @@ -59,9 +59,7 @@ pub enum RenderUiSystem { } pub fn build_ui_render(app: &mut App) { - let mut shaders = app.world.get_resource_mut::>().unwrap(); - let ui_shader = Shader::from_wgsl(include_str!("ui.wgsl")); - shaders.set_untracked(UI_SHADER_HANDLE, ui_shader); + load_internal_asset!(app, UI_SHADER_HANDLE, "ui.wgsl", Shader::from_wgsl); let mut active_cameras = app.world.get_resource_mut::().unwrap(); active_cameras.add(CAMERA_UI); diff --git a/docs/cargo_features.md b/docs/cargo_features.md index a820f183a9..409832cde2 100644 --- a/docs/cargo_features.md +++ b/docs/cargo_features.md @@ -36,3 +36,4 @@ |wayland|Enable this to use Wayland display server protocol other than X11.| |subpixel_glyph_atlas|Enable this to cache glyphs using subpixel accuracy. This increases texture memory usage as each position requires a separate sprite in the glyph atlas, but provide more accurate character spacing.| |bevy_ci_testing|Used for running examples in CI.| +|debug_asset_server|Enabling this turns on "hot reloading" of built in assets, such as shaders.|