Release v0.2.7: mobile text_input (#13, #46); platform-aware viewer clipboard

Host: accept the official text_input {content} action on any channel and type it on
X11 via spare-keycode Unicode keysyms (verified with xev); log non-CONTROL JSON
action names. Viewer: Windows-type hosts get only the case-10 text change (a rich
format list overwrote it and was never rendered), Mac gets only the rich offer.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
VallierDevandClaude Opus 5.5 committed 2026-10-18 09:41:56 +08:00
1 parent e4a2cc4698
commit 63ebb3e78a
9 files changed
+148 -15

No files matched your search

+11
View File
@@ -15,6 +15,9 @@ pub enum ControlInput {
MouseAbsolute { x: f64, y: f64 },
MouseRelative { dx: i32, dy: i32 },
MouseScroll { dx: i32, dy: i32 },
/// Official Windows host `text_input {content}` (KeyboardMouseRunner::execTextInput): text typed by the
/// mobile clients' own input method. Never logged.
TextInput { text: String },
}
#[derive(Debug, Clone, PartialEq)]
@@ -58,6 +61,11 @@ pub fn decode_control(data: &[u8]) -> Result<ControlMessage, String> {
"mouse_move_relative" => ControlInput::MouseRelative {
dx: integer(&v, "delta_x")?, dy: integer(&v, "delta_y")?,
},
"text_input" => {
let text = v.get("content").and_then(Value::as_str).ok_or("CONTROL field content must be a string")?;
if text.len() > 16 * 1024 { return Err("text_input content exceeds 16 KiB".into()); }
ControlInput::TextInput { text: text.to_owned() }
}
"mouse_scroll" => ControlInput::MouseScroll {
dx: integer(&v, "delta_x")?, dy: integer(&v, "delta_y")?,
},
@@ -137,6 +145,9 @@ mod tests {
#[test]
fn does_not_invent_actions_for_unknown_json() {
assert!(matches!(decode_control(br#"{"action":"unverified"}"#).unwrap(), ControlMessage::Unsupported(_)));
assert_eq!(decode_control("{\"action\":\"text_input\",\"content\":\"你好 a\"}".as_bytes()).unwrap(),
ControlMessage::Input(ControlInput::TextInput { text: "你好 a".into() }));
assert!(decode_control(br#"{"action":"text_input"}"#).is_err());
}
}
+17
View File
@@ -10,9 +10,12 @@ extern "C" {
fn x11_key_event(keycode: libc::c_int, is_press: libc::c_int) -> libc::c_int;
fn x11_release_all() -> libc::c_int;
fn x11_terminal_shortcut_state() -> libc::c_int;
fn x11_type_utf8(text: *const libc::c_char) -> libc::c_int;
}
#[cfg(not(target_os = "linux"))]
unsafe fn x11_terminal_shortcut_state() -> libc::c_int { 0 }
#[cfg(not(target_os = "linux"))]
unsafe fn x11_type_utf8(_: *const libc::c_char) -> libc::c_int { 0 }
#[cfg(not(target_os = "linux"))]
unsafe fn x11_mouse_move(_: libc::c_int, _: libc::c_int) -> libc::c_int { 0 }
@@ -357,6 +360,20 @@ impl InputInjector {
if self.backend==crate::desktop_backend::DesktopBackend::Wayland { crate::wayland::key(symbol,pressed) }
else { unsafe { x11_key_symbol(symbol as libc::c_ulong,i32::from(pressed)) } }
}
/// Mobile input-method text (`text_input`). X11 only for now; other backends log once.
pub fn text_input(&self, text: &str) {
let _serial=self.serial.lock().unwrap();
let Ok(c_text)=std::ffi::CString::new(text.replace('\0',"")) else { return };
if self.backend==crate::desktop_backend::DesktopBackend::X11 {
let typed=unsafe { x11_type_utf8(c_text.as_ptr()) };
log::info!("Desktop text input: chars={} result={}", text.chars().count(), typed);
} else {
static WARNED:std::sync::atomic::AtomicBool=std::sync::atomic::AtomicBool::new(false);
if !WARNED.swap(true,std::sync::atomic::Ordering::Relaxed) {
log::warn!("Mobile text input is not supported on the {:?} backend yet", self.backend);
}
}
}
pub fn key_event(&self, keycode: i32, is_press: bool) {
let _serial=self.serial.lock().unwrap();
if self.backend!=crate::desktop_backend::DesktopBackend::X11 { return; }
+22 -7
View File
@@ -95,7 +95,8 @@ impl CommandBackend {
}
}
#[derive(Default)]
struct SyncState {last:Option<String>,assembly:text::Assembly,rich:rich::State,seq:u64}
// target_mac: Some(true) official Mac host, Some(false) Windows-type host (official or this project), None unknown.
struct SyncState {last:Option<String>,assembly:text::Assembly,rich:rich::State,seq:u64,target_mac:Option<bool>}
impl SyncState {
async fn apply(&mut self,backend:&Backend,value:String)->Result<(),()>{
if backend.write(&value).await? {
@@ -114,7 +115,10 @@ impl SyncState {
let id=self.next();let payload=self.rich.offer_any_host(id,value.clone())?;
let rich=wire::envelope(self.next(),21,&payload);
let text=wire::envelope(self.next(),21,&crate::clipboard_protocol::text_request(self.next(),&value));
self.last=Some(value);Ok(Some(vec![rich,text]))
self.last=Some(value);
// Live-verified: the official Windows host writes case-10 text, then a rich format list replaces it with a
// placeholder it never asks us to render (paste is empty). The Mac host only takes the rich offer.
Ok(Some(match self.target_mac {Some(true)=>vec![rich],Some(false)=>vec![text],None=>vec![rich,text]}))
}
fn receive(&mut self,data:&[u8])->Result<(Vec<Vec<u8>>,Option<String>),&'static str>{
if data.len()>MAX_TEXT*2+1024{return Err("clipboard message exceeds limit")}
@@ -151,23 +155,24 @@ impl SyncState {
async fn send(channel:&RTCDataChannel,message:Vec<u8>)->Result<(),()>{
tokio::time::timeout(Duration::from_secs(2),channel.send(&bytes::Bytes::from(message))).await.map_err(|_|())?.map_err(|_|())?;Ok(())
}
pub struct Bridge {backend:Option<Backend>,pub enabled:bool,tx:Option<mpsc::Sender<Vec<u8>>>,task:Option<tokio::task::JoinHandle<()>>}
pub struct Bridge {backend:Option<Backend>,pub enabled:bool,tx:Option<mpsc::Sender<Vec<u8>>>,task:Option<tokio::task::JoinHandle<()>>,target_mac:Option<bool>}
impl Bridge {
pub fn new(desktop:bool)->Self{
let backend=if desktop{Backend::detect()}else{None};let enabled=backend.is_some();
Self{backend,enabled,tx:None,task:None}
Self{backend,enabled,tx:None,task:None,target_mac:None}
}
#[cfg(feature="viewer-sdl")]
pub fn sdl(tx:mpsc::Sender<ClipboardRequest>)->Self{
let backend=(std::env::var("UU_VIEWER_CLIPBOARD").as_deref()!=Ok("0")).then_some(Backend::Sdl(tx));
Self{enabled:backend.is_some(),backend,tx:None,task:None}
Self{enabled:backend.is_some(),backend,tx:None,task:None,target_mac:None}
}
pub fn set_target_mac(&mut self,mac:Option<bool>){self.target_mac=mac;}
pub fn start(&mut self,channel:&Arc<RTCDataChannel>,control:&Arc<RTCDataChannel>,observed:Arc<Observed>){
let Some(backend)=self.backend.take() else{return};
let Some(backend)=self.backend.take() else{return};let target_mac=self.target_mac;
let (tx,mut rx)=mpsc::channel::<Vec<u8>>(16);self.tx=Some(tx);
let channel=Arc::downgrade(channel);let control=Arc::downgrade(control);
self.task=Some(tokio::spawn(async move{
let mut state=SyncState::default();let mut tick=tokio::time::interval(POLL);
let mut state=SyncState{target_mac,..SyncState::default()};let mut tick=tokio::time::interval(POLL);
let mut retries=0;let mut last_probe=std::time::Instant::now();
tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
loop {
@@ -325,6 +330,16 @@ impl Drop for Bridge{fn drop(&mut self){self.stop();}}
if reverse && received.is_some(){viewer.last=Some(value.clone());assert!(viewer.local(value).unwrap().is_none());}
(received,unavailable)
}
#[test] fn local_change_follows_target_platform(){
for (mac,kinds) in [(Some(true),vec![9]),(Some(false),vec![10]),(None,vec![9,10])]{
let mut viewer=SyncState{target_mac:mac,..SyncState::default()};
let cases:Vec<u32>=viewer.local("text".into()).unwrap().unwrap().iter().map(|m|{
let rpc=decode_wire(m).unwrap().into_iter().find_map(|f|match (f.number,f.value){(21,WireValue::Bytes(b))=>Some(b.to_vec()),_=>None}).unwrap();
decode_wire(&rpc).unwrap().iter().map(|f|f.number).find(|n|*n!=1).unwrap()
}).collect();
assert_eq!(cases,kinds,"{mac:?}");
}
}
#[test] fn local_change_also_sends_windows_text_change_request(){
let mut viewer=SyncState::default();
let messages=viewer.local("windows 中文".into()).unwrap().unwrap();assert_eq!(messages.len(),2);
+5 -5
View File
@@ -93,21 +93,21 @@ async fn desktop_command(args:&[String],code:Option<String>)->Result<(),String>
}else{None};
let creds=credentials()?;
let api=Api::new(auth::UUAuthClient::new(creds.clone()),auth::BASE_URL);
#[cfg(feature="viewer-sdl")]
let account_connection=code.is_none();
let room=match code {
Some(code)=>api.join_share(&args[1],&code).await,
None=>api.join(&args[1],force,true).await,
}.map_err(|e|e.to_string())?;
let target_mac=if shape.capture_type==1 {api.target_is_mac(&room,account_connection.then_some(args[1].as_str())).await} else {Err(viewer::api::Error::UnknownKeyboardPlatform)};
#[cfg(feature="viewer-sdl")]
if let Some((window,_))=desktop.as_mut().filter(|_|!view_only){
let device=account_connection.then_some(args[1].as_str());
window.set_mac(api.target_is_mac(&room,device).await.map_err(|e|e.to_string())?);
window.set_mac(*target_mac.as_ref().map_err(|e|e.to_string())?);
}
#[cfg(feature="viewer-sdl")]
let clipboard=desktop.as_ref().map(|(window,_)|window.clipboard_bridge()).unwrap_or_else(||clipboard::Bridge::new(false));
let mut clipboard=desktop.as_ref().map(|(window,_)|window.clipboard_bridge()).unwrap_or_else(||clipboard::Bridge::new(false));
#[cfg(not(feature="viewer-sdl"))]
let clipboard=clipboard::Bridge::new(shape.capture_type==1);
let mut clipboard=clipboard::Bridge::new(shape.capture_type==1);
clipboard.set_target_mac(target_mac.ok());
let options=Options{timeout:Duration::from_secs(12),legacy_prefix:legacy,ready_policy:policy,clipboard};
let mut session=Session::open(&room,&shape,&creds.device_id,&creds.client_id,options,Default::default()).await?;
#[cfg(feature="viewer-sdl")]
+17 -1
View File
@@ -1041,6 +1041,21 @@ impl WebRtcSession {
if let Ok(txt) = std::str::from_utf8(&msg.data) {
log::debug!("DataChannel [{}] received UTF-8 data: {} bytes", channel_name, txt.len());
if let Ok(val) = serde_json::from_str::<serde_json::Value>(txt) {
if !channel_name.contains("CONTROL") {
// Input JSON on another channel (mobile input-method text was missing from CONTROL, #13/#46):
// record action names only, once per name, and accept text_input wherever it arrives.
if let Some(action) = val.get("action").and_then(serde_json::Value::as_str) {
static SEEN: std::sync::Mutex<Vec<String>> = std::sync::Mutex::new(Vec::new());
let mut seen = SEEN.lock().unwrap();
if seen.len() < 32 && action.len() <= 40 && action.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_') && !seen.iter().any(|s| s == action) {
seen.push(action.to_owned());
log::info!("JSON action on {} channel: action={}", if channel_name.contains("STREAMER") {"STREAMER"} else if channel_name.contains("TEXT") {"TEXT"} else {"other"}, action);
}
}
if let Ok(crate::control_protocol::ControlMessage::Input(crate::control_protocol::ControlInput::TextInput {text})) = crate::control_protocol::decode_control(&msg.data) {
if let Some(session)=message_input_session.as_ref() { session.with_input(|| input_worker.text_input(&text)); }
}
}
if channel_name.contains("CONTROL") {
match crate::control_protocol::decode_control(&msg.data) {
Ok(crate::control_protocol::ControlMessage::Input(event)) => {
@@ -1051,7 +1066,7 @@ impl WebRtcSession {
static RECEIPTS: [std::sync::atomic::AtomicUsize; 3] = [const { std::sync::atomic::AtomicUsize::new(0) }; 3];
let (slot, kind) = match &event {
ControlInput::MouseButton {..} | ControlInput::MouseClick {..} => (0, "button"),
ControlInput::Keyboard {..} | ControlInput::KeyboardClick {..} => (1, "keyboard"),
ControlInput::Keyboard {..} | ControlInput::KeyboardClick {..} | ControlInput::TextInput {..} => (1, "keyboard"),
_ => (2, "motion"),
};
if RECEIPTS[slot].fetch_add(1, std::sync::atomic::Ordering::Relaxed) < 16 {
@@ -1063,6 +1078,7 @@ impl WebRtcSession {
ControlInput::MouseRelative {dx,dy} => input_worker.mouse_relative(*dx,*dy),
ControlInput::MouseScroll {dx,dy} => input_worker.mouse_scroll(*dx,*dy),
ControlInput::Keyboard {key, pressed} => input_worker.windows_key_event(*key, *pressed),
ControlInput::TextInput {text} => input_worker.text_input(text),
ControlInput::KeyboardClick {key} => {input_worker.windows_key_event(*key, true);input_worker.windows_key_event(*key, false);},
ControlInput::MouseButton {button,pressed} => input_worker.wire_mouse_button(*button,*pressed),
ControlInput::MouseClick {button} => {input_worker.wire_mouse_button(*button,true);input_worker.wire_mouse_button(*button,false);},