fix(viewer): use native keycodes for Mac targets; ffmpeg 4.4 compatible decoding (#40)

* fix(viewer): send native keycodes to Mac targets

* fix(viewer): reject unknown keyboard target platforms

* fix(viewer): support FFmpeg 4.4 passthrough decoding
This commit is contained in:
wanglz111 authored and GitHub committed 2026-10-17 18:58:57 +08:00
1 parent 843e4aa342
commit 3d4d8d1d64
7 files changed
+176 -9

No files matched your search

+7
View File
@@ -429,6 +429,13 @@ terminal emulator (`TERMINAL` selects one). From source: `sudo apt install libsd
The default request stays 4:2:0; `UU_VIEWER_CHROMA=444` requests true color (4:4:4) after CONTROL opens. The host may fall back to
4:2:0, as reported by CaptureSettingSync. FFmpeg converts either chroma format to RGB24 for display.
Mac targets use native macOS keycodes; Windows / Linux targets retain Windows VK. Keys follow physical positions, without
automatically translating Ctrl to Command; on a Mac target the PC Super / Windows key acts as Command. Only verified target
platform values 1 (Windows / this project's Linux host) and 4 (Mac) are accepted; unknown values fail explicitly. Account
connections look up the selected device only when the platform is absent; assistance requires it in the join response.
View-only sessions skip this lookup. Media keys and some JIS-specific keys are not mapped; Mac platform detection for remote
assistance and ISO / JIS keyboards have not been tested live.
**Tested live** (author's setup, not full acceptance):
| Feature | Official Windows host | Official Mac host | This project's Linux host |
|---|---|---|---|
+5
View File
@@ -393,6 +393,11 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。
默认请求 4:2:0;`UU_VIEWER_CHROMA=444` 在 CONTROL 打开后请求真彩(4:4:4)。主机可能回落到 4:2:0,
以 CaptureSettingSync 回报为准。两种色度均由 FFmpeg 转为 RGB24 显示。
Mac 目标使用 macOS 原生键码,Windows / Linux 目标保留 Windows VK。键盘按物理键位置转发,不自动把 Ctrl 改成 Command;
控制 Mac 时,PC 键盘的 Super / Win 键对应 Command。仅接受已验证的目标平台值 1(Windows / 本项目 Linux 主机)和 4(Mac),
未知值会报错,不猜测键码。账号连接缺少平台字段时查询目标设备;远程协助必须由连接响应提供平台。只看模式不需要此识别。
媒体键和部分 JIS 专用键尚未映射;远程协助的 Mac 平台识别及 ISO / JIS 键盘尚未实机验收。
**已实测**(作者环境,非全面验收):
| 功能 | 官方 Windows 被控端 | 官方 Mac 被控端 | 本项目 Linux 主机 |
|---|---|---|---|
+99 -1
View File
@@ -14,9 +14,16 @@ pub struct JoinData {
#[serde(default)] pub report_server_address: String,
#[serde(default)] pub streamer_retry_delta_ms: i32,
#[serde(default)] pub international_connect: bool,
#[serde(default)] pub publisher_platform: Option<i32>,
}
fn device_platform(data:&Value,device:&str)->Option<i64>{
["my_binded_devices","others_shared_devices"].into_iter()
.filter_map(|key|data.get(key).and_then(Value::as_array)).flatten()
.find(|d|d.get("device_id").and_then(Value::as_str)==Some(device))
.and_then(|d|d.get("platform")).and_then(Value::as_i64)
}
#[derive(Debug, PartialEq, Eq)]
pub enum Error { AuthenticationExpired, Api(i32, String), InvalidResponse, Transport, MissingRoom, InvalidDevice, InvalidAssist }
pub enum Error { AuthenticationExpired, Api(i32, String), InvalidResponse, Transport, MissingRoom, InvalidDevice, InvalidAssist, UnknownKeyboardPlatform }
impl std::fmt::Display for Error {
// The server's short message is user-facing (e.g. 2002 「被控端正在被远控中,无法发起连接」).
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
@@ -27,6 +34,7 @@ impl std::fmt::Display for Error {
Error::InvalidAssist => write!(f, "请输入对方的设备 ID 和有效验证码"),
Error::InvalidResponse => write!(f, "服务器响应格式无效"),
Error::MissingRoom => write!(f, "服务器未返回可用的房间或信令信息"),
Error::UnknownKeyboardPlatform => write!(f, "无法确定目标键盘平台,不能启用控制"),
Error::Api(code, _) => write!(f, "连接请求失败(错误码 {code})"),
_ => write!(f, "viewer API: {self:?}"),
}
@@ -154,6 +162,21 @@ impl Api {
if status != 200 || body.get("code").and_then(Value::as_i64) != Some(0) { return Err(Error::InvalidResponse); }
body.get("data").cloned().ok_or(Error::InvalidResponse)
}
pub async fn target_is_mac(&self,room:&JoinData,device:Option<&str>)->Result<bool,Error>{
let platform=match room.publisher_platform {
Some(platform)=>Some(i64::from(platform)),
None=>match device {
Some(device)=>device_platform(&self.device_list().await?,device),
None=>None,
},
};
// Observed device/share platforms: 1 = Windows (also our Linux host), 4 = Mac.
match platform {
Some(1)=>Ok(false),
Some(4)=>Ok(true),
_=>Err(Error::UnknownKeyboardPlatform),
}
}
/// Method and wire-key mapping remain unverified. Caller must select both explicitly.
pub async fn devices(&self, method: reqwest::Method, array_pointer: &str) -> Result<Vec<Value>, Error> {
let (status, body) = self.request(method, "/api/v1/device/list", None).await?;
@@ -167,6 +190,16 @@ impl Api {
mod tests {
use super::*;
#[test]
fn target_platform_matches_only_the_selected_device(){
let data=json!({"my_binded_devices":[{"device_id":"windows","platform":1},{"device_id":"mac","platform":4}],
"others_shared_devices":[{"device_id":"shared-mac","platform":4}]});
assert_eq!(device_platform(&data,"windows"),Some(1));
assert_eq!(device_platform(&data,"mac"),Some(4));
assert_eq!(device_platform(&data,"shared-mac"),Some(4));
assert_eq!(device_platform(&data,"missing"),None);
assert_eq!(device_platform(&json!({}),"mac"),None);
}
#[test]
fn assistance_response_validation_and_safe_user_errors() {
let mut fixture:Value=serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap();
fixture["data"]["publisher_device_id"]=json!("synthetic-publisher");
@@ -188,6 +221,11 @@ mod tests {
let fixture: Value = serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap();
let data = parse_join(200, fixture.clone()).unwrap();
assert_eq!(data.signaling_list.len(), 1);
assert_eq!(data.publisher_platform,None);
let mut empty=fixture.clone();empty["data"]["publisher_platform"]=Value::Null;
assert_eq!(parse_join(200,empty).unwrap().publisher_platform,None);
let mut mac=fixture.clone();mac["data"]["publisher_platform"]=json!(4);
assert_eq!(parse_join(200,mac).unwrap().publisher_platform,Some(4));
for key in ["report_token", "report_url"] {
let mut v = fixture.clone(); v["data"].as_object_mut().unwrap().remove(key);
assert!(matches!(parse_join(200,v), Err(Error::InvalidResponse)));
@@ -242,6 +280,62 @@ mod http_tests {
(url,job)
}
fn signer()->UUAuthClient {UUAuthClient::new(Credentials{token:"synthetic-auth".into(),sign_key:"synthetic-key".into(),..Default::default()})}
fn room(platform:Option<i32>)->JoinData {
let mut room=parse_join(200,serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap()).unwrap();
room.publisher_platform=platform;room
}
#[tokio::test]
async fn keyboard_platform_uses_room_metadata_without_device_lookup(){
let api=Api::new(signer(),"http://127.0.0.1:1");
for device in [None,Some("synthetic-device")] {
assert_eq!(api.target_is_mac(&room(Some(1)),device).await,Ok(false));
assert_eq!(api.target_is_mac(&room(Some(4)),device).await,Ok(true));
for platform in [0,-1,2,3,5,999,i32::MIN,i32::MAX] {
assert_eq!(api.target_is_mac(&room(Some(platform)),device).await,Err(Error::UnknownKeyboardPlatform));
}
}
assert_eq!(api.target_is_mac(&room(None),None).await,Err(Error::UnknownKeyboardPlatform));
}
#[tokio::test]
async fn keyboard_platform_looks_up_only_the_selected_account_device(){
for list in ["my_binded_devices","others_shared_devices"] {
for (platform,mac) in [(1,false),(4,true)] {
let data=json!({list:[{"device_id":"unrelated","platform":4},
{"device_id":"synthetic-device","platform":platform}]});
let(url,job)=server(vec![(200,json!({"code":0,"data":data}))]).await;
assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Ok(mac));
let requests=job.await.unwrap();assert_eq!(requests.len(),1);
assert!(requests[0].starts_with("GET /api/v1/device/list "));
}
}
}
#[tokio::test]
async fn keyboard_platform_rejects_unknown_or_malformed_device_metadata(){
let mut devices=vec![json!({"device_id":"unrelated","platform":4}),json!({"device_id":"synthetic-device"})];
for platform in [json!(0),json!(-1),json!(999),json!("4"),json!(true),json!(4.5),json!(null),json!(u64::MAX)] {
devices.push(json!({"device_id":"synthetic-device","platform":platform}));
}
for device in devices {
let data=json!({"my_binded_devices":[device]});
let(url,job)=server(vec![(200,json!({"code":0,"data":data}))]).await;
assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Err(Error::UnknownKeyboardPlatform));
assert_eq!(job.await.unwrap().len(),1);
}
}
#[tokio::test]
async fn keyboard_platform_propagates_device_lookup_failures(){
for (status,body,error) in [
(422,json!({"code":1120}),Error::AuthenticationExpired),
(503,json!({"code":5000}),Error::InvalidResponse),
(200,json!({"code":0}),Error::InvalidResponse),
] {
let(url,job)=server(vec![(status,body)]).await;
assert_eq!(Api::new(signer(),&url).target_is_mac(&room(None),Some("synthetic-device")).await,Err(error));
assert_eq!(job.await.unwrap().len(),1);
}
let api=Api::new(signer(),"http://127.0.0.1:1");
assert_eq!(api.target_is_mac(&room(None),Some("synthetic-device")).await,Err(Error::Transport));
}
#[tokio::test]
async fn signed_join_and_explicit_device_list_use_only_loopback_http() {
let fixture=serde_json::from_str(include_str!("../../tests/fixtures/viewer/join.json")).unwrap();
@@ -325,6 +419,10 @@ mod strict_tests {
for (key,bad) in [("token",json!(false)),("signaling_list",json!([7])),("ws_connect_timeout_ms",json!(2147483648u64)),("streamer_retry_delta_ms",json!(null)),("international_connect",json!(1)),("report_token",json!(null)),("report_url",json!(7)),("report_server_address",json!([]))] {
let mut v=fixture.clone();v["data"][key]=bad;assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse)));
}
for platform in [json!("4"),json!(true),json!(4.5),json!(2147483648u64),json!(-2147483649i64)] {
let mut v=fixture.clone();v["data"]["publisher_platform"]=platform;
assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse)));
}
for key in ["code","msg"] {
let mut v=fixture.clone();v.as_object_mut().unwrap().remove(key);assert!(matches!(parse_join(200,v),Err(Error::InvalidResponse)));
}
+5 -3
View File
@@ -1,5 +1,5 @@
//! H.264/H.265 decoding through an external `ffmpeg` process, like the host's encoder pipeline.
//! Needs only the ffmpeg CLI (any recent version), not FFmpeg development libraries.
//! Needs only the ffmpeg CLI, not FFmpeg development libraries.
//! Output is scaled into a fixed letterboxed size so every frame has a known byte length.
use super::desktop::{Decoder,Frame};
use super::media::Codec;
@@ -27,7 +27,8 @@ impl Cli {
"-pix_fmt","rgb24",
// Raw H.264/H.265 carries no timestamps; constant-rate output dropped almost every frame of
// the official Mac host's stream (270 → 2). Pass frames through unchanged.
"-fps_mode","passthrough","-f","rawvideo","pipe:1"])
// -vsync 0 also supports Ubuntu 22.04's FFmpeg 4.4, which lacks -fps_mode.
"-vsync","0","-f","rawvideo","pipe:1"])
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
.spawn().map_err(|e|format!("Start ffmpeg decoder: {e}"))?;
let input=child.stdin.take().ok_or("ffmpeg stdin missing")?;
@@ -91,12 +92,13 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi
Err(std::sync::mpsc::RecvTimeoutError::Disconnected)=>break,
Err(e)=>panic!("{codec:?} decoder did not finish: {e}"),
}}
let status=cli.child.wait().unwrap();
assert!(status.success(),"{codec:?}: ffmpeg exited with {status}");
assert!(!frames.is_empty(),"{codec:?}: no RGB frames");
for frame in frames {
assert_eq!(frame.rgb.len(),32*32*3);
assert!(frame.rgb.chunks_exact(3).all(|p|p[0]>220 && p[1]<30 && p[2]<30),"{codec:?}: RGB conversion");
}
assert!(cli.child.wait().unwrap().success());
}
}
}
+49 -2
View File
@@ -16,6 +16,25 @@ pub fn windows_vk(scan: i32) -> Option<i32> {
_=>return None,
})
}
pub fn mac_keycode(scan:i32)->Option<i32>{
// Inverted SDL Darwin table (release-2.32.10/src/events/scancodes_darwin.h).
// Live Mac host evidence: Windows VK 49 becomes Space, and VK 8 becomes C.
Some(match scan{
4=>0,22=>1,7=>2,9=>3,11=>4,10=>5,29=>6,27=>7,6=>8,25=>9,100=>10,5=>11,
20=>12,26=>13,8=>14,21=>15,28=>16,23=>17,30=>18,31=>19,32=>20,33=>21,
35=>22,34=>23,46=>24,38=>25,36=>26,45=>27,37=>28,39=>29,48=>30,18=>31,
24=>32,47=>33,12=>34,19=>35,40=>36,15=>37,13=>38,52=>39,14=>40,51=>41,
49=>42,54=>43,56=>44,17=>45,16=>46,55=>47,43=>48,44=>49,53=>50,42=>51,
41=>53,231=>54,227=>55,225=>56,57=>57,226=>58,224=>59,229=>60,230=>61,228=>62,
108=>64,99=>65,85=>67,87=>69,83=>71,84=>75,88=>76,86=>78,109=>79,110=>80,
103=>81,98=>82,89=>83,90=>84,91=>85,92=>86,93=>87,94=>88,95=>89,96=>91,97=>92,
62=>96,63=>97,64=>98,60=>99,65=>100,66=>101,68=>103,104=>105,107=>106,105=>107,
70=>105,71=>107,72=>113,
67=>109,101=>110,69=>111,106=>113,73=>114,74=>115,75=>116,76=>117,61=>118,
77=>119,59=>120,78=>121,58=>122,80=>123,79=>124,81=>125,82=>126,
_=>return None,
})
}
#[derive(Clone,Copy,Debug,PartialEq)]
pub struct Viewport { pub x:f64, pub y:f64, pub width:f64, pub height:f64 }
impl Viewport {
@@ -30,16 +49,17 @@ impl Viewport {
}
}
#[derive(Default)]
pub struct Input { view_only:bool, keys:BTreeMap<i32,i32>, buttons:BTreeSet<i32> }
pub struct Input { view_only:bool, mac:bool, keys:BTreeMap<i32,i32>, buttons:BTreeSet<i32> }
pub fn view_only(argument:Option<&str>,environment:Option<&str>)->Result<bool,String> {
match argument {None=>Ok(environment==Some("1")),Some("--view-only")=>Ok(true),_=>Err("Expected --view-only".into())}
}
impl Input {
pub fn new(view_only:bool)->Self {Self{view_only,..Default::default()}}
pub fn set_mac(&mut self,mac:bool){self.mac=mac;}
pub fn enabled(&self)->bool {!self.view_only}
pub fn key(&mut self,scan:i32,down:bool)->Option<Value> {
if self.view_only {return None}
let vk=windows_vk(scan)?;
let vk=if self.mac{mac_keycode(scan)}else{windows_vk(scan)}?;
let changed=if down {
if self.keys.contains_key(&scan) {return None}
let already=self.keys.values().any(|v|*v==vk);self.keys.insert(scan,vk);!already
@@ -65,6 +85,33 @@ impl Input {
}
#[cfg(test)] mod tests {
use super::*;
#[test] fn mac_target_uses_native_codes_and_releases_same_keys(){
let mut input=Input::default();input.set_mac(true);
for (scan,code) in [(30,18),(39,29),(42,51),(44,49),(6,8),(4,0),(40,36),(43,48),
(80,123),(79,124),(81,125),(82,126),(224,59),(225,56),(226,58),(227,55)]{
assert_eq!(input.key(scan,true).unwrap()["key"],code);
assert_eq!(input.key(scan,false).unwrap()["key"],code);
}
input.set_mac(false);assert_eq!(input.key(30,true).unwrap()["key"],49);
}
#[test] fn mac_focus_release_keeps_zero_keycode_and_modifiers(){
let mut input=Input::default();input.set_mac(true);
assert!(input.key(4,true).is_some());assert!(input.key(4,true).is_none());
input.key(225,true);input.key(227,true);
let release=input.release_all();
assert_eq!(release,vec![json!({"action":"kbd_release","key":0}),
json!({"action":"kbd_release","key":55}),json!({"action":"kbd_release","key":56})]);
assert!(input.release_all().is_empty());assert!(input.key(4,false).is_none());
let mut watch=Input::new(true);watch.set_mac(true);assert!(watch.key(4,true).is_none());
}
#[test] fn mac_function_key_aliases_release_after_last_keyup(){
for (pc,function,code) in [(70,104,105),(71,105,107),(72,106,113)]{
let mut input=Input::default();input.set_mac(true);
assert_eq!(input.key(pc,true).unwrap()["key"],code);
assert!(input.key(function,true).is_none());assert!(input.key(pc,false).is_none());
assert_eq!(input.key(function,false).unwrap()["key"],code);
}
}
#[test] fn watch_switch_is_explicit_and_validated(){
assert!(!view_only(None,None).unwrap());assert!(!view_only(None,Some("0")).unwrap());
assert!(view_only(None,Some("1")).unwrap());assert!(view_only(Some("--view-only"),Some("0")).unwrap());
+4 -3
View File
@@ -1,13 +1,14 @@
use super::{desktop::{Frame,FrameSink},input::{Input,Viewport},session::Session};
use sdl2::{event::{Event,WindowEvent},mouse::MouseButton,pixels::PixelFormatEnum,rect::Rect};
pub struct Window { canvas:sdl2::render::Canvas<sdl2::video::Window>, events:sdl2::EventPump, _sdl:sdl2::Sdl, frame:(u32,u32), focused:bool, files:bool }
pub struct Window { canvas:sdl2::render::Canvas<sdl2::video::Window>, events:sdl2::EventPump, _sdl:sdl2::Sdl, frame:(u32,u32), focused:bool, files:bool, mac:bool }
impl Window {
pub fn new()->Result<Self,String>{
let sdl=sdl2::init()?;let video=sdl.video()?;
let window=video.window("Native viewer (experimental)",1280,720).position_centered().resizable().build().map_err(|_|"Window creation failed")?;
let canvas=window.into_canvas().build().map_err(|_|"Renderer creation failed")?;
let events=sdl.event_pump()?;Ok(Self{canvas,events,_sdl:sdl,frame:(1920,1080),focused:true,files:false})
let events=sdl.event_pump()?;Ok(Self{canvas,events,_sdl:sdl,frame:(1920,1080),focused:true,files:false,mac:false})
}
pub fn set_mac(&mut self,mac:bool){self.mac=mac;}
fn viewport(&self)->Viewport {Viewport::fit(self.canvas.window().size(),self.frame)}
pub fn poll(&mut self,input:&mut Input)->(bool,Vec<serde_json::Value>){
let viewport=self.viewport();
@@ -67,7 +68,7 @@ pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super:
let mut command_open=true;let mut requested=false;
let mut terminate=tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).map_err(|_|"Cannot watch termination")?;
let mut audio=super::audio::Playback::new(&window._sdl,session.peer.audio.clone());
let mut input=Input::new(view_only);let mut tick=tokio::time::interval(std::time::Duration::from_millis(8));
let mut input=Input::new(view_only);input.set_mac(window.mac);let mut tick=tokio::time::interval(std::time::Duration::from_millis(8));
let mut heartbeat=tokio::time::interval(std::time::Duration::from_secs(1));
let deadline=tokio::time::sleep(std::time::Duration::from_secs(30));tokio::pin!(deadline);
let result=async {
+7
View File
@@ -89,10 +89,17 @@ async fn desktop_command(args:&[String],code:Option<String>)->Result<(),String>
}else{None};
let creds=credentials()?;
let api=Api::new(auth::UUAuthClient::new(creds.clone()),auth::BASE_URL);
#[cfg(feature="viewer-sdl")]
let account_connection=code.is_none();
let room=match code {
Some(code)=>api.join_share(&args[1],&code).await,
None=>api.join(&args[1],force,true).await,
}.map_err(|e|e.to_string())?;
#[cfg(feature="viewer-sdl")]
if let Some((window,_))=desktop.as_mut().filter(|_|!view_only){
let device=account_connection.then_some(args[1].as_str());
window.set_mac(api.target_is_mac(&room,device).await.map_err(|e|e.to_string())?);
}
let options=Options{timeout:Duration::from_secs(12),legacy_prefix:legacy,ready_policy:policy};
let mut session=Session::open(&room,&shape,&creds.device_id,&creds.client_id,options,Default::default()).await?;
#[cfg(feature="viewer-sdl")]