From 3d4d8d1d64b914c50322f961929685fe5195907a Mon Sep 17 00:00:00 2001 From: wanglz111 <36812003+wanglz111@users.noreply.github.com> Date: Sat, 17 Oct 2026 18:58:57 +0800 Subject: [PATCH] fix(viewer): use native keycodes for Mac targets; ffmpeg 4.4 compatible decoding (#40) * fix(viewer): send native keycodes to Mac targets * fix(viewer): reject unknown keyboard target platforms * fix(viewer): support FFmpeg 4.4 passthrough decoding --- README.en.md | 7 +++ README.md | 5 ++ src/viewer/api.rs | 100 +++++++++++++++++++++++++++++++++++++- src/viewer/decoder_cli.rs | 8 +-- src/viewer/input.rs | 51 ++++++++++++++++++- src/viewer/sdl.rs | 7 +-- src/viewer_main.rs | 7 +++ 7 files changed, 176 insertions(+), 9 deletions(-) diff --git a/README.en.md b/README.en.md index 52bfd0c..5e63451 100644 --- a/README.en.md +++ b/README.en.md @@ -429,6 +429,13 @@ terminal emulator (`TERMINAL` selects one). From source: `sudo apt install libsd The default request stays 4:2:0; `UU_VIEWER_CHROMA=444` requests true color (4:4:4) after CONTROL opens. The host may fall back to 4:2:0, as reported by CaptureSettingSync. FFmpeg converts either chroma format to RGB24 for display. +Mac targets use native macOS keycodes; Windows / Linux targets retain Windows VK. Keys follow physical positions, without +automatically translating Ctrl to Command; on a Mac target the PC Super / Windows key acts as Command. Only verified target +platform values 1 (Windows / this project's Linux host) and 4 (Mac) are accepted; unknown values fail explicitly. Account +connections look up the selected device only when the platform is absent; assistance requires it in the join response. +View-only sessions skip this lookup. Media keys and some JIS-specific keys are not mapped; Mac platform detection for remote +assistance and ISO / JIS keyboards have not been tested live. + **Tested live** (author's setup, not full acceptance): | Feature | Official Windows host | Official Mac host | This project's Linux host | |---|---|---|---| diff --git a/README.md b/README.md index d6a1b30..b49e04c 100644 --- a/README.md +++ b/README.md @@ -393,6 +393,11 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。 默认请求 4:2:0;`UU_VIEWER_CHROMA=444` 在 CONTROL 打开后请求真彩(4:4:4)。主机可能回落到 4:2:0, 以 CaptureSettingSync 回报为准。两种色度均由 FFmpeg 转为 RGB24 显示。 +Mac 目标使用 macOS 原生键码,Windows / Linux 目标保留 Windows VK。键盘按物理键位置转发,不自动把 Ctrl 改成 Command; +控制 Mac 时,PC 键盘的 Super / Win 键对应 Command。仅接受已验证的目标平台值 1(Windows / 本项目 Linux 主机)和 4(Mac), +未知值会报错,不猜测键码。账号连接缺少平台字段时查询目标设备;远程协助必须由连接响应提供平台。只看模式不需要此识别。 +媒体键和部分 JIS 专用键尚未映射;远程协助的 Mac 平台识别及 ISO / JIS 键盘尚未实机验收。 + **已实测**(作者环境,非全面验收): | 功能 | 官方 Windows 被控端 | 官方 Mac 被控端 | 本项目 Linux 主机 | |---|---|---|---| diff --git a/src/viewer/api.rs b/src/viewer/api.rs index e51702b..bc0f11c 100644 --- a/src/viewer/api.rs +++ b/src/viewer/api.rs @@ -14,9 +14,16 @@ pub struct JoinData { #[serde(default)] pub report_server_address: String, #[serde(default)] pub streamer_retry_delta_ms: i32, #[serde(default)] pub international_connect: bool, + #[serde(default)] pub publisher_platform: Option, +} +fn device_platform(data:&Value,device:&str)->Option{ + ["my_binded_devices","others_shared_devices"].into_iter() + .filter_map(|key|data.get(key).and_then(Value::as_array)).flatten() + .find(|d|d.get("device_id").and_then(Value::as_str)==Some(device)) + .and_then(|d|d.get("platform")).and_then(Value::as_i64) } #[derive(Debug, PartialEq, Eq)] -pub enum Error { AuthenticationExpired, Api(i32, String), InvalidResponse, Transport, MissingRoom, InvalidDevice, InvalidAssist } +pub enum Error { AuthenticationExpired, Api(i32, String), InvalidResponse, Transport, MissingRoom, InvalidDevice, InvalidAssist, UnknownKeyboardPlatform } impl std::fmt::Display for Error { // The server's short message is user-facing (e.g. 2002 「被控端正在被远控中,无法发起连接」). fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -27,6 +34,7 @@ impl std::fmt::Display for Error { Error::InvalidAssist => write!(f, "请输入对方的设备 ID 和有效验证码"), Error::InvalidResponse => write!(f, "服务器响应格式无效"), Error::MissingRoom => write!(f, "服务器未返回可用的房间或信令信息"), + Error::UnknownKeyboardPlatform => write!(f, "无法确定目标键盘平台,不能启用控制"), Error::Api(code, _) => write!(f, "连接请求失败(错误码 {code})"), _ => write!(f, "viewer API: {self:?}"), } @@ -154,6 +162,21 @@ impl Api { if status != 200 || body.get("code").and_then(Value::as_i64) != Some(0) { return Err(Error::InvalidResponse); } body.get("data").cloned().ok_or(Error::InvalidResponse) } + pub async fn target_is_mac(&self,room:&JoinData,device:Option<&str>)->Result{ + let platform=match room.publisher_platform { + Some(platform)=>Some(i64::from(platform)), + None=>match device { + Some(device)=>device_platform(&self.device_list().await?,device), + None=>None, + }, + }; + // Observed device/share platforms: 1 = Windows (also our Linux host), 4 = Mac. + match platform { + Some(1)=>Ok(false), + Some(4)=>Ok(true), + _=>Err(Error::UnknownKeyboardPlatform), + } + } /// Method and wire-key mapping remain unverified. Caller must select both explicitly. pub async fn devices(&self, method: reqwest::Method, array_pointer: &str) -> Result, Error> { let (status, body) = self.request(method, "/api/v1/device/list", None).await?; @@ -167,6 +190,16 @@ impl Api { mod tests { use super::*; #[test] + fn target_platform_matches_only_the_selected_device(){ + let data=json!({"my_binded_devices":[{"device_id":"windows","platform":1},{"device_id":"mac","platform":4}], + "others_shared_devices":[{"device_id":"shared-mac","platform":4}]}); + assert_eq!(device_platform(&data,"windows"),Some(1)); + assert_eq!(device_platform(&data,"mac"),Some(4)); + assert_eq!(device_platform(&data,"shared-mac"),Some(4)); + assert_eq!(device_platform(&data,"missing"),None); + assert_eq!(device_platform(&json!({}),"mac"),None); + } + #[test] fn assistance_response_validation_and_safe_user_errors() { let mut fixture:Value=serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap(); fixture["data"]["publisher_device_id"]=json!("synthetic-publisher"); @@ -188,6 +221,11 @@ mod tests { let fixture: Value = serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap(); let data = parse_join(200, fixture.clone()).unwrap(); assert_eq!(data.signaling_list.len(), 1); + assert_eq!(data.publisher_platform,None); + let mut empty=fixture.clone();empty["data"]["publisher_platform"]=Value::Null; + assert_eq!(parse_join(200,empty).unwrap().publisher_platform,None); + let mut mac=fixture.clone();mac["data"]["publisher_platform"]=json!(4); + assert_eq!(parse_join(200,mac).unwrap().publisher_platform,Some(4)); for key in ["report_token", "report_url"] { let mut v = fixture.clone(); v["data"].as_object_mut().unwrap().remove(key); assert!(matches!(parse_join(200,v), Err(Error::InvalidResponse))); @@ -242,6 +280,62 @@ mod http_tests { (url,job) } fn signer()->UUAuthClient {UUAuthClient::new(Credentials{token:"synthetic-auth".into(),sign_key:"synthetic-key".into(),..Default::default()})} + fn room(platform:Option)->JoinData { + let mut room=parse_join(200,serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap()).unwrap(); + room.publisher_platform=platform;room + } + #[tokio::test] + async fn keyboard_platform_uses_room_metadata_without_device_lookup(){ + let api=Api::new(signer(),"http://127.0.0.1:1"); + for device in [None,Some("synthetic-device")] { + assert_eq!(api.target_is_mac(&room(Some(1)),device).await,Ok(false)); + assert_eq!(api.target_is_mac(&room(Some(4)),device).await,Ok(true)); + for platform in [0,-1,2,3,5,999,i32::MIN,i32::MAX] { + assert_eq!(api.target_is_mac(&room(Some(platform)),device).await,Err(Error::UnknownKeyboardPlatform)); + } + } + assert_eq!(api.target_is_mac(&room(None),None).await,Err(Error::UnknownKeyboardPlatform)); + } + #[tokio::test] + async fn keyboard_platform_looks_up_only_the_selected_account_device(){ + for list in ["my_binded_devices","others_shared_devices"] { + for (platform,mac) in [(1,false),(4,true)] { + let data=json!({list:[{"device_id":"unrelated","platform":4}, + {"device_id":"synthetic-device","platform":platform}]}); + let(url,job)=server(vec![(200,json!({"code":0,"data":data}))]).await; + assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Ok(mac)); + let requests=job.await.unwrap();assert_eq!(requests.len(),1); + assert!(requests[0].starts_with("GET /api/v1/device/list ")); + } + } + } + #[tokio::test] + async fn keyboard_platform_rejects_unknown_or_malformed_device_metadata(){ + let mut devices=vec![json!({"device_id":"unrelated","platform":4}),json!({"device_id":"synthetic-device"})]; + for platform in [json!(0),json!(-1),json!(999),json!("4"),json!(true),json!(4.5),json!(null),json!(u64::MAX)] { + devices.push(json!({"device_id":"synthetic-device","platform":platform})); + } + for device in devices { + let data=json!({"my_binded_devices":[device]}); + let(url,job)=server(vec![(200,json!({"code":0,"data":data}))]).await; + assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Err(Error::UnknownKeyboardPlatform)); + assert_eq!(job.await.unwrap().len(),1); + } + } + #[tokio::test] + async fn keyboard_platform_propagates_device_lookup_failures(){ + for (status,body,error) in [ + (422,json!({"code":1120}),Error::AuthenticationExpired), + (503,json!({"code":5000}),Error::InvalidResponse), + (200,json!({"code":0}),Error::InvalidResponse), + ] { + let(url,job)=server(vec![(status,body)]).await; + assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Err(error)); + assert_eq!(job.await.unwrap().len(),1); + } + let api=Api::new(signer(),"http://127.0.0.1:1"); + assert_eq!(api.target_is_mac(&room(None),Some("synthetic-device")).await,Err(Error::Transport)); + } #[tokio::test] async fn signed_join_and_explicit_device_list_use_only_loopback_http() { let fixture=serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap(); @@ -325,6 +419,10 @@ mod strict_tests { for (key,bad) in [("token",json!(false)),("signaling_list",json!([7])),("ws_connect_timeout_ms",json!(2147483648u64)),("streamer_retry_delta_ms",json!(null)),("international_connect",json!(1)),("report_token",json!(null)),("report_url",json!(7)),("report_server_address",json!([]))] { let mut v=fixture.clone();v["data"][key]=bad;assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse))); } + for platform in [json!("4"),json!(true),json!(4.5),json!(2147483648u64),json!(-2147483649i64)] { + let mut v=fixture.clone();v["data"]["publisher_platform"]=platform; + assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse))); + } for key in ["code","msg"] { let mut v=fixture.clone();v.as_object_mut().unwrap().remove(key);assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse))); } diff --git a/src/viewer/decoder_cli.rs b/src/viewer/decoder_cli.rs index 8eb26f6..1852ad6 100644 --- a/src/viewer/decoder_cli.rs +++ b/src/viewer/decoder_cli.rs @@ -1,5 +1,5 @@ //! H.264/H.265 decoding through an external `ffmpeg` process, like the host's encoder pipeline. -//! Needs only the ffmpeg CLI (any recent version), not FFmpeg development libraries. +//! Needs only the ffmpeg CLI, not FFmpeg development libraries. //! Output is scaled into a fixed letterboxed size so every frame has a known byte length. use super::desktop::{Decoder,Frame}; use super::media::Codec; @@ -27,7 +27,8 @@ impl Cli { "-pix_fmt","rgb24", // Raw H.264/H.265 carries no timestamps; constant-rate output dropped almost every frame of // the official Mac host's stream (270 → 2). Pass frames through unchanged. - "-fps_mode","passthrough","-f","rawvideo","pipe:1"]) + // -vsync 0 also supports Ubuntu 22.04's FFmpeg 4.4, which lacks -fps_mode. + "-vsync","0","-f","rawvideo","pipe:1"]) .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) .spawn().map_err(|e|format!("Start ffmpeg decoder: {e}"))?; let input=child.stdin.take().ok_or("ffmpeg stdin missing")?; @@ -91,12 +92,13 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi Err(std::sync::mpsc::RecvTimeoutError::Disconnected)=>break, Err(e)=>panic!("{codec:?} decoder did not finish: {e}"), }} + let status=cli.child.wait().unwrap(); + assert!(status.success(),"{codec:?}: ffmpeg exited with {status}"); assert!(!frames.is_empty(),"{codec:?}: no RGB frames"); for frame in frames { assert_eq!(frame.rgb.len(),32*32*3); assert!(frame.rgb.chunks_exact(3).all(|p|p[0]>220 && p[1]<30 && p[2]<30),"{codec:?}: RGB conversion"); } - assert!(cli.child.wait().unwrap().success()); } } } diff --git a/src/viewer/input.rs b/src/viewer/input.rs index 759f03f..2787289 100644 --- a/src/viewer/input.rs +++ b/src/viewer/input.rs @@ -16,6 +16,25 @@ pub fn windows_vk(scan: i32) -> Option { _=>return None, }) } +pub fn mac_keycode(scan:i32)->Option{ + // Inverted SDL Darwin table (release-2.32.10/src/events/scancodes_darwin.h). + // Live Mac host evidence: Windows VK 49 becomes Space, and VK 8 becomes C. + Some(match scan{ + 4=>0,22=>1,7=>2,9=>3,11=>4,10=>5,29=>6,27=>7,6=>8,25=>9,100=>10,5=>11, + 20=>12,26=>13,8=>14,21=>15,28=>16,23=>17,30=>18,31=>19,32=>20,33=>21, + 35=>22,34=>23,46=>24,38=>25,36=>26,45=>27,37=>28,39=>29,48=>30,18=>31, + 24=>32,47=>33,12=>34,19=>35,40=>36,15=>37,13=>38,52=>39,14=>40,51=>41, + 49=>42,54=>43,56=>44,17=>45,16=>46,55=>47,43=>48,44=>49,53=>50,42=>51, + 41=>53,231=>54,227=>55,225=>56,57=>57,226=>58,224=>59,229=>60,230=>61,228=>62, + 108=>64,99=>65,85=>67,87=>69,83=>71,84=>75,88=>76,86=>78,109=>79,110=>80, + 103=>81,98=>82,89=>83,90=>84,91=>85,92=>86,93=>87,94=>88,95=>89,96=>91,97=>92, + 62=>96,63=>97,64=>98,60=>99,65=>100,66=>101,68=>103,104=>105,107=>106,105=>107, + 70=>105,71=>107,72=>113, + 67=>109,101=>110,69=>111,106=>113,73=>114,74=>115,75=>116,76=>117,61=>118, + 77=>119,59=>120,78=>121,58=>122,80=>123,79=>124,81=>125,82=>126, + _=>return None, + }) +} #[derive(Clone,Copy,Debug,PartialEq)] pub struct Viewport { pub x:f64, pub y:f64, pub width:f64, pub height:f64 } impl Viewport { @@ -30,16 +49,17 @@ impl Viewport { } } #[derive(Default)] -pub struct Input { view_only:bool, keys:BTreeMap, buttons:BTreeSet } +pub struct Input { view_only:bool, mac:bool, keys:BTreeMap, buttons:BTreeSet } pub fn view_only(argument:Option<&str>,environment:Option<&str>)->Result { match argument {None=>Ok(environment==Some("1")),Some("--view-only")=>Ok(true),_=>Err("Expected --view-only".into())} } impl Input { pub fn new(view_only:bool)->Self {Self{view_only,..Default::default()}} + pub fn set_mac(&mut self,mac:bool){self.mac=mac;} pub fn enabled(&self)->bool {!self.view_only} pub fn key(&mut self,scan:i32,down:bool)->Option { if self.view_only {return None} - let vk=windows_vk(scan)?; + let vk=if self.mac{mac_keycode(scan)}else{windows_vk(scan)}?; let changed=if down { if self.keys.contains_key(&scan) {return None} let already=self.keys.values().any(|v|*v==vk);self.keys.insert(scan,vk);!already @@ -65,6 +85,33 @@ impl Input { } #[cfg(test)] mod tests { use super::*; + #[test] fn mac_target_uses_native_codes_and_releases_same_keys(){ + let mut input=Input::default();input.set_mac(true); + for (scan,code) in [(30,18),(39,29),(42,51),(44,49),(6,8),(4,0),(40,36),(43,48), + (80,123),(79,124),(81,125),(82,126),(224,59),(225,56),(226,58),(227,55)]{ + assert_eq!(input.key(scan,true).unwrap()["key"],code); + assert_eq!(input.key(scan,false).unwrap()["key"],code); + } + input.set_mac(false);assert_eq!(input.key(30,true).unwrap()["key"],49); + } + #[test] fn mac_focus_release_keeps_zero_keycode_and_modifiers(){ + let mut input=Input::default();input.set_mac(true); + assert!(input.key(4,true).is_some());assert!(input.key(4,true).is_none()); + input.key(225,true);input.key(227,true); + let release=input.release_all(); + assert_eq!(release,vec![json!({"action":"kbd_release","key":0}), + json!({"action":"kbd_release","key":55}),json!({"action":"kbd_release","key":56})]); + assert!(input.release_all().is_empty());assert!(input.key(4,false).is_none()); + let mut watch=Input::new(true);watch.set_mac(true);assert!(watch.key(4,true).is_none()); + } + #[test] fn mac_function_key_aliases_release_after_last_keyup(){ + for (pc,function,code) in [(70,104,105),(71,105,107),(72,106,113)]{ + let mut input=Input::default();input.set_mac(true); + assert_eq!(input.key(pc,true).unwrap()["key"],code); + assert!(input.key(function,true).is_none());assert!(input.key(pc,false).is_none()); + assert_eq!(input.key(function,false).unwrap()["key"],code); + } + } #[test] fn watch_switch_is_explicit_and_validated(){ assert!(!view_only(None,None).unwrap());assert!(!view_only(None,Some("0")).unwrap()); assert!(view_only(None,Some("1")).unwrap());assert!(view_only(Some("--view-only"),Some("0")).unwrap()); diff --git a/src/viewer/sdl.rs b/src/viewer/sdl.rs index 26a604c..5f126fe 100644 --- a/src/viewer/sdl.rs +++ b/src/viewer/sdl.rs @@ -1,13 +1,14 @@ use super::{desktop::{Frame,FrameSink},input::{Input,Viewport},session::Session}; use sdl2::{event::{Event,WindowEvent},mouse::MouseButton,pixels::PixelFormatEnum,rect::Rect}; -pub struct Window { canvas:sdl2::render::Canvas, events:sdl2::EventPump, _sdl:sdl2::Sdl, frame:(u32,u32), focused:bool, files:bool } +pub struct Window { canvas:sdl2::render::Canvas, events:sdl2::EventPump, _sdl:sdl2::Sdl, frame:(u32,u32), focused:bool, files:bool, mac:bool } impl Window { pub fn new()->Result{ let sdl=sdl2::init()?;let video=sdl.video()?; let window=video.window("Native viewer (experimental)",1280,720).position_centered().resizable().build().map_err(|_|"Window creation failed")?; let canvas=window.into_canvas().build().map_err(|_|"Renderer creation failed")?; - let events=sdl.event_pump()?;Ok(Self{canvas,events,_sdl:sdl,frame:(1920,1080),focused:true,files:false}) + let events=sdl.event_pump()?;Ok(Self{canvas,events,_sdl:sdl,frame:(1920,1080),focused:true,files:false,mac:false}) } + pub fn set_mac(&mut self,mac:bool){self.mac=mac;} fn viewport(&self)->Viewport {Viewport::fit(self.canvas.window().size(),self.frame)} pub fn poll(&mut self,input:&mut Input)->(bool,Vec){ let viewport=self.viewport(); @@ -67,7 +68,7 @@ pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super: let mut command_open=true;let mut requested=false; let mut terminate=tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).map_err(|_|"Cannot watch termination")?; let mut audio=super::audio::Playback::new(&window._sdl,session.peer.audio.clone()); - let mut input=Input::new(view_only);let mut tick=tokio::time::interval(std::time::Duration::from_millis(8)); + let mut input=Input::new(view_only);input.set_mac(window.mac);let mut tick=tokio::time::interval(std::time::Duration::from_millis(8)); let mut heartbeat=tokio::time::interval(std::time::Duration::from_secs(1)); let deadline=tokio::time::sleep(std::time::Duration::from_secs(30));tokio::pin!(deadline); let result=async { diff --git a/src/viewer_main.rs b/src/viewer_main.rs index ff0721d..f1dc916 100644 --- a/src/viewer_main.rs +++ b/src/viewer_main.rs @@ -89,10 +89,17 @@ async fn desktop_command(args:&[String],code:Option)->Result<(),String> }else{None}; let creds=credentials()?; let api=Api::new(auth::UUAuthClient::new(creds.clone()),auth::BASE_URL); + #[cfg(feature="viewer-sdl")] + let account_connection=code.is_none(); let room=match code { Some(code)=>api.join_share(&args[1],&code).await, None=>api.join(&args[1],force,true).await, }.map_err(|e|e.to_string())?; + #[cfg(feature="viewer-sdl")] + if let Some((window,_))=desktop.as_mut().filter(|_|!view_only){ + let device=account_connection.then_some(args[1].as_str()); + window.set_mac(api.target_is_mac(&room,device).await.map_err(|e|e.to_string())?); + } let options=Options{timeout:Duration::from_secs(12),legacy_prefix:legacy,ready_policy:policy}; let mut session=Session::open(&room,&shape,&creds.device_id,&creds.client_id,options,Default::default()).await?; #[cfg(feature="viewer-sdl")]