fix(viewer): prevent compressed-frame queue overflow at high input rates (#143)

This commit is contained in:
AZsaber authored and GitHub committed 2026-11-01 21:11:07 +08:00
1 parent 33861082dc
commit 1bdc0eb910
5 files changed
+193 -26

No files matched your search

+2
View File
@@ -586,6 +586,8 @@ No official "no desktop" error code was found for headless desktop offers, so th
## Experimental native viewer (uuremote-viewer)
See [SDL viewer video scheduling](docs/viewer-video-scheduling.md).
`uuremote-viewer` is a Linux **viewer/controller**: it connects to and controls other devices on your UU account (official Windows / Mac
hosts or this project's Linux host), and can start remote assistance with someone's device ID and code. It is experimental and
separate from the host `uuremote-server`; the default host build and behaviour are unchanged.
+2
View File
@@ -514,6 +514,8 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。
## 实验性原生控制端(uuremote-viewer)
[SDL 控制端视频调度说明](docs/viewer-video-scheduling.md)。
`uuremote-viewer` 是 Linux 上的**控制端**:用你的 UU 账号连接并控制其他设备(官方 Windows / Mac 被控端,或本项目的 Linux 主机),
也可以凭对方的设备 ID 和验证码发起远程协助。它是实验性功能,与主机 `uuremote-server` 相互独立,默认主机构建和行为不变。
+51
View File
@@ -0,0 +1,51 @@
# SDL viewer video scheduling
## Problem and cause
The SDL viewer previously consumed at most one compressed access unit per 8 ms
tick: approximately 125 units/s. With one unit per picture, sustained 144 FPS
input can overflow its bounded queue even when decoding keeps up. Lost compressed
units can break reference-frame dependencies, causing repeated decoder resets
and keyframe recovery. Increasing queue capacity alone only delays the overflow.
## Implementation
Each tick feeds queued compressed units to the existing decoder in FIFO order,
up to 8 units. A 4 ms elapsed-work budget is checked between calls. The viewer
then presents only the newest completed picture, avoiding redundant presentation
of older decoded pictures. It also polls asynchronous decoder output on idle ticks.
The compressed queue holds 8 units instead of 3 to absorb short presentation
stalls; processing starts immediately without waiting for a full batch. Actual
compressed-unit loss still resets the decoder and uses the existing keyframe
gate and PLI requests. Decoder interfaces, counters and opt-in diagnostic dumps
remain available; the presentation counter counts pictures actually presented.
There are no new settings or protocol fields.
## Verification
The desktop regression tests simulate 8 seconds of 144 units/s input and 125
processing ticks/s. The old one-unit policy drops 150 units with capacity 3 and
145 with capacity 8. Bounded batching delivers all 1,152 units to a mock decoder
in order without compressed-unit loss or resets, presenting at most once per tick.
Other tests cover newest-picture selection, idle asynchronous output, recovery
after real queue overflow, and count/time budgets leaving work for the next tick.
Budget tests inject elapsed time internally rather than depending on CI timing.
Run the full viewer checks without test filters or skips:
```sh
cargo test --locked --features native-viewer,viewer-sdl --bin uuremote-viewer
cargo build --locked --features native-viewer,viewer-sdl --bin uuremote-viewer
```
## Limits and scope
The 4 ms budget cannot interrupt an already-blocked decoder call and does not
bound the subsequent presentation call. Slow decoding or rendering can still
overflow the queue. This adds no hardware decoding and guarantees neither stable
144 FPS nor 144 distinct displayed pictures on a 90 Hz screen.
Startup timeout, stream-profile synchronization, gamepad/rumble and installation
changes are outside this patch.
Implementation and regression tests were prepared with Codex assistance.
+135 -14
View File
@@ -3,7 +3,7 @@ use std::sync::{Arc, atomic::{AtomicBool,AtomicU32,Ordering}};
#[derive(Clone)]
pub struct Desktop { pub units:Latest<super::media::AccessUnit>, pub pli:Arc<AtomicBool>, pub reset:Arc<AtomicBool>, pub ssrc:Arc<AtomicU32> }
impl Default for Desktop {
fn default()->Self {Self{units:Latest::new(3),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}}
fn default()->Self {Self{units:Latest::new(8),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}}
}
/// `content` is the source picture size inside the (possibly padded) RGB canvas; None = full frame.
pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec<u8>, pub content:Option<(u32,u32)> }
@@ -51,12 +51,15 @@ impl KeyGate {
} else {false}
}
}
/// Shows the newest frame a decoder finished between access units.
pub fn drain(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink)->Result<(),String> {
if let Some(frame)=decoder.poll().pop() {STATS[1].fetch_add(1,Ordering::Relaxed);sink.present(&frame)?;}
fn present(sink:&mut dyn FrameSink,frame:&Frame)->Result<(),String> {
sink.present(frame)?;
// Local acceptance aid: UU_VIEWER_DUMP_FRAME=path writes the 5th frame as PPM (owner's own screen).
if STATS[1].fetch_add(1,Ordering::Relaxed)==std::env::var("UU_VIEWER_DUMP_FRAME_INDEX").ok().and_then(|v|v.parse().ok()).unwrap_or(4u64) { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_FRAME") {
let mut ppm=format!("P6\n{} {}\n255\n",frame.width,frame.height).into_bytes();ppm.extend_from_slice(&frame.rgb);let _=std::fs::write(path,ppm);
}}
Ok(())
}
pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:super::media::Codec,pli:&AtomicBool)->Result<(),String> {
fn decode(decoder:&mut dyn Decoder,au:&[u8],codec:super::media::Codec,pli:&AtomicBool)->Vec<Frame> {
let units=STATS[0].fetch_add(1,Ordering::Relaxed)+1;
// A later session can join a clean stream with no IDR in flight (#124): keep asking until a frame shows.
if wants_key_frame(units,STATS[1].load(Ordering::Relaxed)) {
@@ -68,16 +71,33 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s
use std::io::Write; if let Ok(mut f)=std::fs::OpenOptions::new().create(true).append(true).open(path){let _=f.write_all(au);}
}}
match decoder.decode(au) {
Ok(frames)=>{for frame in frames {
sink.present(&frame)?;
// Local acceptance aid: UU_VIEWER_DUMP_FRAME=path writes the 5th frame as PPM (owner's own screen).
if STATS[1].fetch_add(1,Ordering::Relaxed)==std::env::var("UU_VIEWER_DUMP_FRAME_INDEX").ok().and_then(|v|v.parse().ok()).unwrap_or(4u64) { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_FRAME") {
let mut ppm=format!("P6\n{} {}\n255\n",frame.width,frame.height).into_bytes();ppm.extend_from_slice(&frame.rgb);let _=std::fs::write(path,ppm);
}}
}Ok(())},
Err(_)=>{STATS[2].fetch_add(1,Ordering::Relaxed);decoder.reset();pli.store(true,Ordering::Release);Ok(())},
Ok(frames)=>frames,
Err(_)=>{STATS[2].fetch_add(1,Ordering::Relaxed);decoder.reset();pli.store(true,Ordering::Release);Vec::new()},
}
}
/// Feed reference-dependent units in FIFO order, but present only the newest decoded picture.
pub fn pump(desktop:&Desktop,decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,gate:&mut KeyGate)->Result<Option<super::media::Codec>,String> {
let started=std::time::Instant::now();
pump_with_elapsed(desktop,decoder,sink,gate,||started.elapsed())
}
fn pump_with_elapsed(desktop:&Desktop,decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,gate:&mut KeyGate,mut elapsed:impl FnMut()->std::time::Duration)->Result<Option<super::media::Codec>,String> {
let mut newest=None;let mut codec=None;
for _ in 0..8 {
let Some(au)=desktop.units.pop() else {break};
if desktop.reset.swap(false,Ordering::AcqRel) {decoder.reset();gate.arm();}
if !gate.admit(au.codec,&au.data) {desktop.pli.store(true,Ordering::Release);}
else {
decoder.set_codec(au.codec)?;codec=Some(au.codec);
if let Some(frame)=decode(decoder,&au.data,au.codec,&desktop.pli).pop() {newest=Some(frame);}
}
// A soft budget between calls: an in-progress decoder call cannot be interrupted.
if elapsed()>=std::time::Duration::from_millis(4) {break}
}
// Asynchronous decoders may finish a picture even when no new input has arrived.
if let Some(frame)=decoder.poll().pop() {newest=Some(frame);}
if let Some(frame)=newest {present(sink,&frame)?;}
Ok(codec)
}
#[cfg(test)] mod tests {
#[test] fn key_frames_by_nal_type(){
use crate::viewer::media::Codec;
@@ -106,5 +126,106 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s
assert_eq!((1..=90).filter(|&n|wants_key_frame(n,0)).count(),3);
assert!(!(1..=90).any(|n|wants_key_frame(n,1)));
}
#[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert_eq!(s.0,1);}
#[test] fn decode_error_requests_pli_and_recovers(){
let desktop=Desktop::default();let mut d=Mock(true);let mut s=Sink(0);let mut gate=KeyGate::new();
enqueue(&desktop,unit(0,false));
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert!(desktop.pli.load(Ordering::Acquire));assert_eq!(s.0,0);
enqueue(&desktop,unit(1,false));
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,1);
}
use crate::viewer::media::{AccessUnit,Codec};
use std::time::Duration;
fn unit(id:u32,key:bool)->AccessUnit {
let mut data=vec![0,0,0,1,if key {0x65}else{0x41},0b1001_1000];
data.extend_from_slice(&id.to_le_bytes());AccessUnit{codec:Codec::H264,data}
}
fn picture(id:u32)->Frame {Frame{width:1,height:1,rgb:id.to_le_bytes()[..3].to_vec(),content:None}}
#[derive(Default)]
struct BatchDecoder {seen:Vec<u32>,resets:usize,pending:Vec<Frame>,extra_picture:bool}
impl Decoder for BatchDecoder {
fn set_codec(&mut self,_:Codec)->Result<(),String>{Ok(())}
fn decode(&mut self,au:&[u8])->Result<Vec<Frame>,String>{
let id=u32::from_le_bytes(au[6..].try_into().unwrap());self.seen.push(id);
let mut frames=vec![picture(id)];if self.extra_picture {frames.push(picture(id+100));}Ok(frames)
}
fn reset(&mut self){self.resets+=1;self.pending.clear();}
fn poll(&mut self)->Vec<Frame>{std::mem::take(&mut self.pending)}
}
#[derive(Default)]
struct Pictures(Vec<u32>);
impl FrameSink for Pictures {
fn present(&mut self,frame:&Frame)->Result<(),String>{assert_eq!(frame.rgb.len(),3);self.0.push(u32::from_le_bytes([frame.rgb[0],frame.rgb[1],frame.rgb[2],0]));Ok(())}
}
// Match the RTP reader's overflow signal without a network or wall-clock sleeps.
fn enqueue(desktop:&Desktop,au:AccessUnit)->bool {
let dropped=desktop.units.push(au);
if dropped {desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release);}
dropped
}
#[test] fn one_unit_per_tick_overflows_even_with_a_larger_queue(){
for capacity in [3,8] {
let q=Latest::new(capacity);let mut arrived=0;let mut dropped=0;
// 1,000 ticks at 125 Hz = 8 seconds; exactly 1,152 units arrive at 144 Hz.
for tick in 1..=1000 {
let due=tick*144/125;
while arrived<due {dropped+=usize::from(q.push(arrived));arrived+=1;}
assert!(q.pop().is_some());
}
assert_eq!(arrived,1152);assert_eq!(dropped,152-(capacity-1));
}
}
#[test] fn batches_keep_up_with_144_units_per_second_in_fifo_order(){
let desktop=Desktop::default();let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new();let mut arrived=0;
for tick in 1..=1000 {
let due=tick*144/125;
while arrived<due {assert!(!enqueue(&desktop,unit(arrived,false)));arrived+=1;}
let before=s.0.len();
assert_eq!(pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(),Some(Codec::H264));
assert_eq!(s.0.len(),before+1);assert_eq!(s.0.last(),Some(&(due-1)));
}
assert_eq!(arrived,1152);assert_eq!(d.seen,(0..1152).collect::<Vec<_>>());
assert_eq!(d.resets,0);assert!(desktop.units.pop().is_none());assert!(!desktop.reset.load(Ordering::Acquire));
}
#[test] fn batch_presents_only_the_latest_picture_and_polls_when_idle(){
let desktop=Desktop::default();let mut d=BatchDecoder{extra_picture:true,..Default::default()};let mut s=Pictures::default();let mut gate=KeyGate::new();
for id in 0..3 {assert!(!enqueue(&desktop,unit(id,false)));}
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.seen,vec![0,1,2]);assert_eq!(s.0,vec![102]);
d.pending=vec![picture(103),picture(104)];
assert_eq!(pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap(),None);
assert_eq!(s.0,vec![102,104]);
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,vec![102,104]);
enqueue(&desktop,unit(105,false));d.pending=vec![picture(206)];
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();assert_eq!(s.0,vec![102,104,206]);
}
#[test] fn actual_queue_loss_still_resets_and_waits_for_a_key_frame(){
let desktop=Desktop::default();let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new();
for id in 0..8 {assert!(!enqueue(&desktop,unit(id,false)));}
assert!(enqueue(&desktop,unit(8,false)));
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.resets,1);assert!(d.seen.is_empty());assert!(s.0.is_empty());assert!(desktop.pli.load(Ordering::Acquire));
enqueue(&desktop,unit(9,false));enqueue(&desktop,unit(10,true));enqueue(&desktop,unit(11,false));
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.resets,1);assert_eq!(d.seen,vec![10,11]);assert_eq!(s.0,vec![11]);
}
#[test] fn batch_count_limit_leaves_remaining_units_for_the_next_tick(){
let desktop=Desktop{units:Latest::new(12),..Default::default()};let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new();
for id in 0..12 {assert!(!enqueue(&desktop,unit(id,false)));}
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.seen,(0..8).collect::<Vec<_>>());assert_eq!(s.0,vec![7]);
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.seen,(0..12).collect::<Vec<_>>());assert_eq!(s.0,vec![7,11]);
}
#[test] fn elapsed_budget_is_checked_between_calls_and_preserves_the_queue(){
let desktop=Desktop::default();let mut d=BatchDecoder::default();let mut s=Pictures::default();let mut gate=KeyGate::new();
for id in 0..4 {enqueue(&desktop,unit(id,false));}
let mut times=[Duration::from_millis(3),Duration::from_millis(4)].into_iter();
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||times.next().unwrap()).unwrap();
assert_eq!(d.seen,vec![0,1]);assert_eq!(s.0,vec![1]);
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::from_millis(20)).unwrap();
assert_eq!(d.seen,vec![0,1,2]);assert_eq!(s.0,vec![1,2]);
pump_with_elapsed(&desktop,&mut d,&mut s,&mut gate,||Duration::ZERO).unwrap();
assert_eq!(d.seen,vec![0,1,2,3]);assert_eq!(s.0,vec![1,2,3]);
}
}
+3 -12
View File
@@ -551,18 +551,9 @@ pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super:
if session.peer.ready(super::rtc::ReadyPolicy::EchoOnly,false){for m in messages{session.peer.send_input(&m).await?;}}
else {input.release_all();}
if quit{return Ok(())}
// At most one AU per iteration keeps local event polling responsive.
if let Some(au)=session.peer.desktop.units.pop(){
if session.peer.desktop.reset.swap(false,std::sync::atomic::Ordering::AcqRel){decoder.reset();key_gate.arm();}
if !key_gate.admit(au.codec,&au.data) {session.peer.desktop.pli.store(true,std::sync::atomic::Ordering::Release);super::desktop::drain(decoder,window)?;}
else {
decoder.set_codec(au.codec)?;
codec_name=if au.codec==super::media::Codec::H265 {"H.265"} else {"H.264"};
super::desktop::display(decoder,window,&au.data,au.codec,&session.peer.desktop.pli)?;
}}
// A picture can finish after its access unit was written; show it without waiting
// for the next one (a single keystroke may change the remote screen only once).
else {super::desktop::drain(decoder,window)?;}
if let Some(codec)=super::desktop::pump(&session.peer.desktop,decoder,window,&mut key_gate)? {
codec_name=if codec==super::media::Codec::H265 {"H.265"} else {"H.264"};
}
}
}}
}.await;