add Input Method Editor support (#7325)

# Objective

- Fix #7315
- Add IME support

## Solution

- Add two new fields to `Window`, to control if IME is enabled and the candidate box position

This allows the use of dead keys which are needed in French, or the full IME experience to type using Pinyin

I also added a basic general text input example that can handle IME input.

https://user-images.githubusercontent.com/8672791/213941353-5ed73a73-5dd1-4e66-a7d6-a69b49694c52.mp4

Cargo.toml

@@ -1147,6 +1147,16 @@ description = "Prints out all touch inputs"
 category = "Input"
 wasm = false
 
+[[example]]
+name = "text_input"
+path = "examples/input/text_input.rs"
+
+[package.metadata.example.text_input]
+name = "Text Input"
+description = "Simple text input with IME support"
+category = "Input"
+wasm = false
+
 # Reflection
 [[example]]
 name = "reflection"

crates/bevy_window/src/event.rs

@@ -153,6 +153,52 @@ pub struct ReceivedCharacter {
     pub char: char,
 }
 
+/// A Input Method Editor event.
+///
+/// This event is the translated version of the `WindowEvent::Ime` from the `winit` crate.
+///
+/// It is only sent if IME was enabled on the window with [`Window::ime_enabled`](crate::window::Window::ime_enabled).
+#[derive(Debug, Clone, PartialEq, Eq, Reflect, FromReflect)]
+#[reflect(Debug, PartialEq)]
+#[cfg_attr(
+    feature = "serialize",
+    derive(serde::Serialize, serde::Deserialize),
+    reflect(Serialize, Deserialize)
+)]
+pub enum Ime {
+    /// Notifies when a new composing text should be set at the cursor position.
+    Preedit {
+        /// Window that received the event.
+        window: Entity,
+        /// Current value.
+        value: String,
+        /// Cursor begin and end position.
+        ///
+        /// `None` indicated the cursor should be hidden
+        cursor: Option<(usize, usize)>,
+    },
+    /// Notifies when text should be inserted into the editor widget.
+    Commit {
+        /// Window that received the event.
+        window: Entity,
+        /// Input string
+        value: String,
+    },
+    /// Notifies when the IME was enabled.
+    ///
+    /// After this event, you will receive events `Ime::Preedit` and `Ime::Commit`,
+    /// and stop receiving events [`ReceivedCharacter`].
+    Enabled {
+        /// Window that received the event.
+        window: Entity,
+    },
+    /// Notifies when the IME was disabled.
+    Disabled {
+        /// Window that received the event.
+        window: Entity,
+    },
+}
+
 /// An event that indicates a window has received or lost focus.
 #[derive(Debug, Clone, PartialEq, Eq, Reflect, FromReflect)]
 #[reflect(Debug, PartialEq)]

crates/bevy_window/src/lib.rs

@@ -15,7 +15,7 @@ pub use window::*;
 pub mod prelude {
     #[doc(hidden)]
     pub use crate::{
-        CursorEntered, CursorIcon, CursorLeft, CursorMoved, FileDragAndDrop, MonitorSelection,
+        CursorEntered, CursorIcon, CursorLeft, CursorMoved, FileDragAndDrop, Ime, MonitorSelection,
         ReceivedCharacter, Window, WindowMoved, WindowPlugin, WindowPosition,
         WindowResizeConstraints,
     };
@@ -79,6 +79,7 @@ impl Plugin for WindowPlugin {
             .add_event::<CursorEntered>()
             .add_event::<CursorLeft>()
             .add_event::<ReceivedCharacter>()
+            .add_event::<Ime>()
             .add_event::<WindowFocused>()
             .add_event::<WindowScaleFactorChanged>()
             .add_event::<WindowBackendScaleFactorChanged>()

crates/bevy_window/src/window.rs

@@ -167,6 +167,24 @@ pub struct Window {
     pub prevent_default_event_handling: bool,
     /// Stores internal state that isn't directly accessible.
     pub internal: InternalWindowState,
+    /// Should the window use Input Method Editor?
+    ///
+    /// If enabled, the window will receive [`Ime`](crate::Ime) events instead of
+    /// [`ReceivedCharacter`](crate::ReceivedCharacter) or
+    /// [`KeyboardInput`](bevy_input::keyboard::KeyboardInput).
+    ///
+    /// IME should be enabled during text input, but not when you expect to get the exact key pressed.
+    ///
+    ///  ## Platform-specific
+    ///
+    /// - iOS / Android / Web: Unsupported.
+    pub ime_enabled: bool,
+    /// Sets location of IME candidate box in client area coordinates relative to the top left.
+    ///
+    ///  ## Platform-specific
+    ///
+    /// - iOS / Android / Web: Unsupported.
+    pub ime_position: Vec2,
 }
 
 impl Default for Window {
@@ -181,6 +199,8 @@ impl Default for Window {
             internal: Default::default(),
             composite_alpha_mode: Default::default(),
             resize_constraints: Default::default(),
+            ime_enabled: Default::default(),
+            ime_position: Default::default(),
             resizable: true,
             decorations: true,
             transparent: false,

crates/bevy_winit/src/lib.rs

@@ -25,9 +25,10 @@ use bevy_utils::{
     Instant,
 };
 use bevy_window::{
-    CursorEntered, CursorLeft, CursorMoved, FileDragAndDrop, ModifiesWindows, ReceivedCharacter,
-    RequestRedraw, Window, WindowBackendScaleFactorChanged, WindowCloseRequested, WindowCreated,
-    WindowFocused, WindowMoved, WindowResized, WindowScaleFactorChanged,
+    CursorEntered, CursorLeft, CursorMoved, FileDragAndDrop, Ime, ModifiesWindows,
+    ReceivedCharacter, RequestRedraw, Window, WindowBackendScaleFactorChanged,
+    WindowCloseRequested, WindowCreated, WindowFocused, WindowMoved, WindowResized,
+    WindowScaleFactorChanged,
 };
 
 use winit::{
@@ -170,6 +171,7 @@ struct InputEvents<'w> {
     mouse_button_input: EventWriter<'w, MouseButtonInput>,
     mouse_wheel_input: EventWriter<'w, MouseWheel>,
     touch_input: EventWriter<'w, TouchInput>,
+    ime_input: EventWriter<'w, Ime>,
 }
 
 #[derive(SystemParam)]
@@ -555,6 +557,25 @@ pub fn winit_runner(mut app: App) {
                             position,
                         });
                     }
+                    WindowEvent::Ime(event) => match event {
+                        event::Ime::Preedit(value, cursor) => {
+                            input_events.ime_input.send(Ime::Preedit {
+                                window: window_entity,
+                                value,
+                                cursor,
+                            });
+                        }
+                        event::Ime::Commit(value) => input_events.ime_input.send(Ime::Commit {
+                            window: window_entity,
+                            value,
+                        }),
+                        event::Ime::Enabled => input_events.ime_input.send(Ime::Enabled {
+                            window: window_entity,
+                        }),
+                        event::Ime::Disabled => input_events.ime_input.send(Ime::Disabled {
+                            window: window_entity,
+                        }),
+                    },
                     _ => {}
                 }
             }

crates/bevy_winit/src/system.rs

@@ -13,7 +13,7 @@ use bevy_window::{RawHandleWrapper, Window, WindowClosed, WindowCreated};
 use raw_window_handle::{HasRawDisplayHandle, HasRawWindowHandle};
 
 use winit::{
-    dpi::{LogicalSize, PhysicalPosition, PhysicalSize},
+    dpi::{LogicalPosition, LogicalSize, PhysicalPosition, PhysicalSize},
     event_loop::EventLoopWindowTarget,
 };
 
@@ -278,6 +278,17 @@ pub(crate) fn changed_window(
                 );
             }
 
+            if window.ime_enabled != previous.ime_enabled {
+                winit_window.set_ime_allowed(window.ime_enabled);
+            }
+
+            if window.ime_position != previous.ime_position {
+                winit_window.set_ime_position(LogicalPosition::new(
+                    window.ime_position.x,
+                    window.ime_position.y,
+                ));
+            }
+
             info.previous = window.clone();
         }
     }

examples/README.md

@@ -234,6 +234,7 @@ Example | Description
 [Mouse Grab](../examples/input/mouse_grab.rs) | Demonstrates how to grab the mouse, locking the cursor to the app's screen
 [Mouse Input](../examples/input/mouse_input.rs) | Demonstrates handling a mouse button press/release
 [Mouse Input Events](../examples/input/mouse_input_events.rs) | Prints out all mouse events (buttons, movement, etc.)
+[Text Input](../examples/input/text_input.rs) | Simple text input with IME support
 [Touch Input](../examples/input/touch_input.rs) | Displays touch presses, releases, and cancels
 [Touch Input Events](../examples/input/touch_input_events.rs) | Prints out all touch inputs
 

examples/input/text_input.rs

@@ -0,0 +1,201 @@
+//! Simple text input support
+//!
+//! Return creates a new line, backspace removes the last character.
+//! Clicking toggle IME (Input Method Editor) support, but the font used as limited support of characters.
+//! You should change the provided font with another one to test other languages input.
+
+use bevy::{input::keyboard::KeyboardInput, prelude::*};
+
+fn main() {
+    App::new()
+        .add_plugins(DefaultPlugins)
+        .add_startup_system(setup_scene)
+        .add_system(toggle_ime)
+        .add_system(listen_ime_events)
+        .add_system(listen_received_character_events)
+        .add_system(listen_keyboard_input_events)
+        .add_system(bubbling_text)
+        .run();
+}
+
+fn setup_scene(mut commands: Commands, asset_server: Res<AssetServer>) {
+    commands.spawn(Camera2dBundle::default());
+
+    let font = asset_server.load("fonts/FiraMono-Medium.ttf");
+
+    commands.spawn(
+        TextBundle::from_sections([
+            TextSection {
+                value: "IME Enabled: ".to_string(),
+                style: TextStyle {
+                    font: font.clone_weak(),
+                    font_size: 20.0,
+                    color: Color::WHITE,
+                },
+            },
+            TextSection {
+                value: "false\n".to_string(),
+                style: TextStyle {
+                    font: font.clone_weak(),
+                    font_size: 30.0,
+                    color: Color::WHITE,
+                },
+            },
+            TextSection {
+                value: "IME Active: ".to_string(),
+                style: TextStyle {
+                    font: font.clone_weak(),
+                    font_size: 20.0,
+                    color: Color::WHITE,
+                },
+            },
+            TextSection {
+                value: "false\n".to_string(),
+                style: TextStyle {
+                    font: font.clone_weak(),
+                    font_size: 30.0,
+                    color: Color::WHITE,
+                },
+            },
+            TextSection {
+                value: "click to toggle IME, press return to start a new line\n\n".to_string(),
+                style: TextStyle {
+                    font: font.clone_weak(),
+                    font_size: 18.0,
+                    color: Color::WHITE,
+                },
+            },
+            TextSection {
+                value: "".to_string(),
+                style: TextStyle {
+                    font,
+                    font_size: 25.0,
+                    color: Color::WHITE,
+                },
+            },
+        ])
+        .with_style(Style {
+            position_type: PositionType::Absolute,
+            position: UiRect {
+                top: Val::Px(10.0),
+                left: Val::Px(10.0),
+                ..default()
+            },
+            ..default()
+        }),
+    );
+
+    commands.spawn(Text2dBundle {
+        text: Text::from_section(
+            "".to_string(),
+            TextStyle {
+                font: asset_server.load("fonts/FiraMono-Medium.ttf"),
+                font_size: 100.0,
+                color: Color::WHITE,
+            },
+        ),
+        ..default()
+    });
+}
+
+fn toggle_ime(
+    input: Res<Input<MouseButton>>,
+    mut windows: Query<&mut Window>,
+    mut text: Query<&mut Text, With<Node>>,
+) {
+    if input.just_pressed(MouseButton::Left) {
+        let mut window = windows.single_mut();
+
+        window.ime_position = window
+            .cursor_position()
+            .map(|p| Vec2::new(p.x, window.height() - p.y))
+            .unwrap();
+        window.ime_enabled = !window.ime_enabled;
+
+        let mut text = text.single_mut();
+        text.sections[1].value = format!("{}\n", window.ime_enabled);
+    }
+}
+
+#[derive(Component)]
+struct Bubble {
+    timer: Timer,
+}
+
+#[derive(Component)]
+struct ImePreedit;
+
+fn bubbling_text(
+    mut commands: Commands,
+    mut bubbles: Query<(Entity, &mut Transform, &mut Bubble)>,
+    time: Res<Time>,
+) {
+    for (entity, mut transform, mut bubble) in bubbles.iter_mut() {
+        if bubble.timer.tick(time.delta()).just_finished() {
+            commands.entity(entity).despawn();
+        }
+        transform.translation.y += time.delta_seconds() * 100.0;
+    }
+}
+
+fn listen_ime_events(
+    mut events: EventReader<Ime>,
+    mut status_text: Query<&mut Text, With<Node>>,
+    mut edit_text: Query<&mut Text, (Without<Node>, Without<Bubble>)>,
+) {
+    for event in events.iter() {
+        match event {
+            Ime::Preedit { value, cursor, .. } if !cursor.is_none() => {
+                status_text.single_mut().sections[5].value = format!("IME buffer: {value}");
+            }
+            Ime::Preedit { cursor, .. } if cursor.is_none() => {
+                status_text.single_mut().sections[5].value = "".to_string();
+            }
+            Ime::Commit { value, .. } => {
+                edit_text.single_mut().sections[0].value.push_str(value);
+            }
+            Ime::Enabled { .. } => {
+                status_text.single_mut().sections[3].value = "true\n".to_string();
+            }
+            Ime::Disabled { .. } => {
+                status_text.single_mut().sections[3].value = "false\n".to_string();
+            }
+            _ => (),
+        }
+    }
+}
+
+fn listen_received_character_events(
+    mut events: EventReader<ReceivedCharacter>,
+    mut edit_text: Query<&mut Text, (Without<Node>, Without<Bubble>)>,
+) {
+    for event in events.iter() {
+        edit_text.single_mut().sections[0].value.push(event.char);
+    }
+}
+
+fn listen_keyboard_input_events(
+    mut commands: Commands,
+    mut events: EventReader<KeyboardInput>,
+    mut edit_text: Query<(Entity, &mut Text), (Without<Node>, Without<Bubble>)>,
+) {
+    for event in events.iter() {
+        match event.key_code {
+            Some(KeyCode::Return) => {
+                let (entity, text) = edit_text.single();
+                commands.entity(entity).insert(Bubble {
+                    timer: Timer::from_seconds(5.0, TimerMode::Once),
+                });
+
+                commands.spawn(Text2dBundle {
+                    text: Text::from_section("".to_string(), text.sections[0].style.clone()),
+                    ..default()
+                });
+            }
+            Some(KeyCode::Back) => {
+                edit_text.single_mut().1.sections[0].value.pop();
+            }
+            _ => continue,
+        }
+    }
+}