From 1bdc0eb910c7ec11e2e23b855447262d0ff3a127 Mon Sep 17 00:00:00 2001 From: AZsaber Date: Sun, 1 Nov 2026 21:11:07 +0800 Subject: [PATCH] fix(viewer): prevent compressed-frame queue overflow at high input rates (#143) --- README.en.md | 2 + README.md | 2 + docs/viewer-video-scheduling.md | 51 +++++++++++ src/viewer/desktop.rs | 149 +++++++++++++++++++++++++++++--- src/viewer/sdl.rs | 15 +--- 5 files changed, 193 insertions(+), 26 deletions(-) create mode 100644 docs/viewer-video-scheduling.md diff --git a/README.en.md b/README.en.md index ef8f269..200a171 100644 --- a/README.en.md +++ b/README.en.md @@ -586,6 +586,8 @@ No official "no desktop" error code was found for headless desktop offers, so th ## Experimental native viewer (uuremote-viewer) +See [SDL viewer video scheduling](docs/viewer-video-scheduling.md). + `uuremote-viewer` is a Linux **viewer/controller**: it connects to and controls other devices on your UU account (official Windows / Mac hosts or this project's Linux host), and can start remote assistance with someone's device ID and code. It is experimental and separate from the host `uuremote-server`; the default host build and behaviour are unchanged. diff --git a/README.md b/README.md index 31e051f..a297146 100644 --- a/README.md +++ b/README.md @@ -514,6 +514,8 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。 ## 实验性原生控制端(uuremote-viewer) +[SDL 控制端视频调度说明](docs/viewer-video-scheduling.md)。 + `uuremote-viewer` 是 Linux 上的**控制端**:用你的 UU 账号连接并控制其他设备(官方 Windows / Mac 被控端,或本项目的 Linux 主机), 也可以凭对方的设备 ID 和验证码发起远程协助。它是实验性功能,与主机 `uuremote-server` 相互独立,默认主机构建和行为不变。 diff --git a/docs/viewer-video-scheduling.md b/docs/viewer-video-scheduling.md new file mode 100644 index 0000000..3f0d536 --- /dev/null +++ b/docs/viewer-video-scheduling.md @@ -0,0 +1,51 @@ +# SDL viewer video scheduling + +## Problem and cause + +The SDL viewer previously consumed at most one compressed access unit per 8 ms +tick: approximately 125 units/s. With one unit per picture, sustained 144 FPS +input can overflow its bounded queue even when decoding keeps up. Lost compressed +units can break reference-frame dependencies, causing repeated decoder resets +and keyframe recovery. Increasing queue capacity alone only delays the overflow. + +## Implementation + +Each tick feeds queued compressed units to the existing decoder in FIFO order, +up to 8 units. A 4 ms elapsed-work budget is checked between calls. The viewer +then presents only the newest completed picture, avoiding redundant presentation +of older decoded pictures. It also polls asynchronous decoder output on idle ticks. + +The compressed queue holds 8 units instead of 3 to absorb short presentation +stalls; processing starts immediately without waiting for a full batch. Actual +compressed-unit loss still resets the decoder and uses the existing keyframe +gate and PLI requests. Decoder interfaces, counters and opt-in diagnostic dumps +remain available; the presentation counter counts pictures actually presented. +There are no new settings or protocol fields. + +## Verification + +The desktop regression tests simulate 8 seconds of 144 units/s input and 125 +processing ticks/s. The old one-unit policy drops 150 units with capacity 3 and +145 with capacity 8. Bounded batching delivers all 1,152 units to a mock decoder +in order without compressed-unit loss or resets, presenting at most once per tick. +Other tests cover newest-picture selection, idle asynchronous output, recovery +after real queue overflow, and count/time budgets leaving work for the next tick. +Budget tests inject elapsed time internally rather than depending on CI timing. + +Run the full viewer checks without test filters or skips: + +```sh +cargo test --locked --features native-viewer,viewer-sdl --bin uuremote-viewer +cargo build --locked --features native-viewer,viewer-sdl --bin uuremote-viewer +``` + +## Limits and scope + +The 4 ms budget cannot interrupt an already-blocked decoder call and does not +bound the subsequent presentation call. Slow decoding or rendering can still +overflow the queue. This adds no hardware decoding and guarantees neither stable +144 FPS nor 144 distinct displayed pictures on a 90 Hz screen. + +Startup timeout, stream-profile synchronization, gamepad/rumble and installation +changes are outside this patch. +Implementation and regression tests were prepared with Codex assistance. diff --git a/src/viewer/desktop.rs b/src/viewer/desktop.rs index 44bbd57..c38eb53 100644 --- a/src/viewer/desktop.rs +++ b/src/viewer/desktop.rs @@ -3,7 +3,7 @@ use std::sync::{Arc, atomic::{AtomicBool,AtomicU32,Ordering}}; #[derive(Clone)] pub struct Desktop { pub units:Latest, pub pli:Arc, pub reset:Arc, pub ssrc:Arc } impl Default for Desktop { - fn default()->Self {Self{units:Latest::new(3),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}} + fn default()->Self {Self{units:Latest::new(8),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}} } /// `content` is the source picture size inside the (possibly padded) RGB canvas; None = full frame. pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec, pub content:Option<(u32,u32)> } @@ -51,12 +51,15 @@ impl KeyGate { } else {false} } } -/// Shows the newest frame a decoder finished between access units. -pub fn drain(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink)->Result<(),String> { - if let Some(frame)=decoder.poll().pop() {STATS[1].fetch_add(1,Ordering::Relaxed);sink.present(&frame)?;} +fn present(sink:&mut dyn FrameSink,frame:&Frame)->Result<(),String> { + sink.present(frame)?; + // Local acceptance aid: UU_VIEWER_DUMP_FRAME=path writes the 5th frame as PPM (owner's own screen). + if STATS[1].fetch_add(1,Ordering::Relaxed)==std::env::var("UU_VIEWER_DUMP_FRAME_INDEX").ok().and_then(|v|v.parse().ok()).unwrap_or(4u64) { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_FRAME") { + let mut ppm=format!("P6\n{} {}\n255\n",frame.width,frame.height).into_bytes();ppm.extend_from_slice(&frame.rgb);let _=std::fs::write(path,ppm); + }} Ok(()) } -pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:super::media::Codec,pli:&AtomicBool)->Result<(),String> { +fn decode(decoder:&mut dyn Decoder,au:&[u8],codec:super::media::Codec,pli:&AtomicBool)->Vec { let units=STATS[0].fetch_add(1,Ordering::Relaxed)+1; // A later session can join a clean stream with no IDR in flight (#124): keep asking until a frame shows. if wants_key_frame(units,STATS[1].load(Ordering::Relaxed)) { @@ -68,16 +71,33 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s use std::io::Write; if let Ok(mut f)=std::fs::OpenOptions::new().create(true).append(true).open(path){let _=f.write_all(au);} }} match decoder.decode(au) { - Ok(frames)=>{for frame in frames { - sink.present(&frame)?; - // Local acceptance aid: UU_VIEWER_DUMP_FRAME=path writes the 5th frame as PPM (owner's own screen). - if STATS[1].fetch_add(1,Ordering::Relaxed)==std::env::var("UU_VIEWER_DUMP_FRAME_INDEX").ok().and_then(|v|v.parse().ok()).unwrap_or(4u64) { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_FRAME") { - let mut ppm=format!("P6\n{} {}\n255\n",frame.width,frame.height).into_bytes();ppm.extend_from_slice(&frame.rgb);let _=std::fs::write(path,ppm); - }} - }Ok(())}, - Err(_)=>{STATS[2].fetch_add(1,Ordering::Relaxed);decoder.reset();pli.store(true,Ordering::Release);Ok(())}, + Ok(frames)=>frames, + Err(_)=>{STATS[2].fetch_add(1,Ordering::Relaxed);decoder.reset();pli.store(true,Ordering::Release);Vec::new()}, } } +/// Feed reference-dependent units in FIFO order, but present only the newest decoded picture. +pub fn pump(desktop:&Desktop,decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,gate:&mut KeyGate)->Result,String> { + let started=std::time::Instant::now(); + pump_with_elapsed(desktop,decoder,sink,gate,||started.elapsed()) +} +fn pump_with_elapsed(desktop:&Desktop,decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,gate:&mut KeyGate,mut elapsed:impl FnMut()->std::time::Duration)->Result,String> { + let mut newest=None;let mut codec=None; + for _ in 0..8 { + let Some(au)=desktop.units.pop() else {break}; + if desktop.reset.swap(false,Ordering::AcqRel) {decoder.reset();gate.arm();} + if !gate.admit(au.codec,&au.data) {desktop.pli.store(true,Ordering::Release);} + else { + decoder.set_codec(au.codec)?;codec=Some(au.codec); + if let Some(frame)=decode(decoder,&au.data,au.codec,&desktop.pli).pop() {newest=Some(frame);} + } + // A soft budget between calls: an in-progress decoder call cannot be interrupted. + if elapsed()>=std::time::Duration::from_millis(4) {break} + } + // Asynchronous decoders may finish a picture even when no new input has arrived. + if let Some(frame)=decoder.poll().pop() {newest=Some(frame);} + if let Some(frame)=newest {present(sink,&frame)?;} + Ok(codec) +} #[cfg(test)] mod tests { #[test] fn key_frames_by_nal_type(){ use crate::viewer::media::Codec; @@ -106,5 +126,106 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s assert_eq!((1..=90).filter(|&n|wants_key_frame(n,0)).count(),3); assert!(!(1..=90).any(|n|wants_key_frame(n,1))); } - #[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert_eq!(s.0,1);} + #[test] fn decode_error_requests_pli_and_recovers(){ + let desktop=Desktop::default();let mut d=Mock(true);let mut s=Sink(0);let mut gate=KeyGate::new(); + enqueue(&desktop,unit(0,false)); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert!(desktop.pli.load(Ordering::Acquire));assert_eq!(s.0,0); + enqueue(&desktop,unit(1,false)); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,1); + } + + use crate::viewer::media::{AccessUnit,Codec}; + use std::time::Duration; + fn unit(id:u32,key:bool)->AccessUnit { + let mut data=vec![0,0,0,1,if key {0x65}else{0x41},0b1001_1000]; + data.extend_from_slice(&id.to_le_bytes());AccessUnit{codec:Codec::H264,data} + } + fn picture(id:u32)->Frame {Frame{width:1,height:1,rgb:id.to_le_bytes()[..3].to_vec(),content:None}} + #[derive(Default)] + struct BatchDecoder {seen:Vec,resets:usize,pending:Vec,extra_picture:bool} + impl Decoder for BatchDecoder { + fn set_codec(&mut self,_:Codec)->Result<(),String>{Ok(())} + fn decode(&mut self,au:&[u8])->Result,String>{ + let id=u32::from_le_bytes(au[6..].try_into().unwrap());self.seen.push(id); + let mut frames=vec![picture(id)];if self.extra_picture {frames.push(picture(id+100));}Ok(frames) + } + fn reset(&mut self){self.resets+=1;self.pending.clear();} + fn poll(&mut self)->Vec{std::mem::take(&mut self.pending)} + } + #[derive(Default)] + struct Pictures(Vec); + impl FrameSink for Pictures { + fn present(&mut self,frame:&Frame)->Result<(),String>{assert_eq!(frame.rgb.len(),3);self.0.push(u32::from_le_bytes([frame.rgb[0],frame.rgb[1],frame.rgb[2],0]));Ok(())} + } + // Match the RTP reader's overflow signal without a network or wall-clock sleeps. + fn enqueue(desktop:&Desktop,au:AccessUnit)->bool { + let dropped=desktop.units.push(au); + if dropped {desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release);} + dropped + } + #[test] fn one_unit_per_tick_overflows_even_with_a_larger_queue(){ + for capacity in [3,8] { + let q=Latest::new(capacity);let mut arrived=0;let mut dropped=0; + // 1,000 ticks at 125 Hz = 8 seconds; exactly 1,152 units arrive at 144 Hz. + for tick in 1..=1000 { + let due=tick*144/125; + while arrived>()); + assert_eq!(d.resets,0);assert!(desktop.units.pop().is_none());assert!(!desktop.reset.load(Ordering::Acquire)); + } + #[test] fn batch_presents_only_the_latest_picture_and_polls_when_idle(){ + let desktop=Desktop::default();let mut d=BatchDecoder{extra_picture:true,..Default::default()};let mut s=Pictures::default();let mut gate=KeyGate::new(); + for id in 0..3 {assert!(!enqueue(&desktop,unit(id,false)));} + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.seen,vec![0,1,2]);assert_eq!(s.0,vec![102]); + d.pending=vec![picture(103),picture(104)]; + assert_eq!(pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(),None); + assert_eq!(s.0,vec![102,104]); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,vec![102,104]); + enqueue(&desktop,unit(105,false));d.pending=vec![picture(206)]; + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,vec![102,104,206]); + } + #[test] fn actual_queue_loss_still_resets_and_waits_for_a_key_frame(){ + let desktop=Desktop::default();let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new(); + for id in 0..8 {assert!(!enqueue(&desktop,unit(id,false)));} + assert!(enqueue(&desktop,unit(8,false))); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.resets,1);assert!(d.seen.is_empty());assert!(s.0.is_empty());assert!(desktop.pli.load(Ordering::Acquire)); + enqueue(&desktop,unit(9,false));enqueue(&desktop,unit(10,true));enqueue(&desktop,unit(11,false)); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.resets,1);assert_eq!(d.seen,vec![10,11]);assert_eq!(s.0,vec![11]); + } + #[test] fn batch_count_limit_leaves_remaining_units_for_the_next_tick(){ + let desktop=Desktop{units:Latest::new(12),..Default::default()};let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new(); + for id in 0..12 {assert!(!enqueue(&desktop,unit(id,false)));} + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.seen,(0..8).collect::>());assert_eq!(s.0,vec![7]); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.seen,(0..12).collect::>());assert_eq!(s.0,vec![7,11]); + } + #[test] fn elapsed_budget_is_checked_between_calls_and_preserves_the_queue(){ + let desktop=Desktop::default();let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new(); + for id in 0..4 {enqueue(&desktop,unit(id,false));} + let mut times=[Duration::from_millis(3),Duration::from_millis(4)].into_iter(); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||times.next().unwrap()).unwrap(); + assert_eq!(d.seen,vec![0,1]);assert_eq!(s.0,vec![1]); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::from_millis(20)).unwrap(); + assert_eq!(d.seen,vec![0,1,2]);assert_eq!(s.0,vec![1,2]); + pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(); + assert_eq!(d.seen,vec![0,1,2,3]);assert_eq!(s.0,vec![1,2,3]); + } } diff --git a/src/viewer/sdl.rs b/src/viewer/sdl.rs index 13448e1..d991097 100644 --- a/src/viewer/sdl.rs +++ b/src/viewer/sdl.rs @@ -551,18 +551,9 @@ pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super: if session.peer.ready(super::rtc::ReadyPolicy::EchoOnly,false){for m in messages{session.peer.send_input(&m).await?;}} else {input.release_all();} if quit{return Ok(())} - // At most one AU per iteration keeps local event polling responsive. - if let Some(au)=session.peer.desktop.units.pop(){ - if session.peer.desktop.reset.swap(false,std::sync::atomic::Ordering::AcqRel){decoder.reset();key_gate.arm();} - if !key_gate.admit(au.codec,&au.data) {session.peer.desktop.pli.store(true,std::sync::atomic::Ordering::Release);super::desktop::drain(decoder,window)?;} - else { - decoder.set_codec(au.codec)?; - codec_name=if au.codec==super::media::Codec::H265 {"H.265"} else {"H.264"}; - super::desktop::display(decoder,window,&au.data,au.codec,&session.peer.desktop.pli)?; - }} - // A picture can finish after its access unit was written; show it without waiting - // for the next one (a single keystroke may change the remote screen only once). - else {super::desktop::drain(decoder,window)?;} + if let Some(codec)=super::desktop::pump(&session.peer.desktop,decoder,window,&mut key_gate)? { + codec_name=if codec==super::media::Codec::H265 {"H.265"} else {"H.264"}; + } } }} }.await;