viewer: map pointer input to the content rectangle of the padded canvas (#107)

The CLI decoder pads every frame into a fixed 1920x1080 canvas so rawvideo frames have a known byte length. Input mapping and the SDL viewport treated that padded canvas as the picture, so non-16:9 hosts (official Macs are usually 16:10) had the pillarbox bars inside the 0..1 range: the remote pointer lagged inward and the error grew towards the screen edges.

Parse the coded picture size from ffmpeg's one-time "Video:" stream line, carry it on Frame.content, and render/map only the content sub-rectangle. 16:9 streams are unchanged; UU_VIEWER_SIZE keeps working.
This commit is contained in:
liuyanghejerry authored and GitHub committed 2026-10-24 21:41:48 +08:00
1 parent bcde9f967c
commit 287b21a622
5 files changed
+84 -14

No files matched your search

+1 -1
View File
@@ -37,7 +37,7 @@ impl Decoder for Ffmpeg {
let mut rgb=av::frame::Video::empty();scaler.run(&decoded,&mut rgb).map_err(|_|"Frame conversion failed")?;
let mut data=Vec::with_capacity(w as usize*h as usize*3);
for row in 0..h as usize {let start=row*rgb.stride(0);data.extend_from_slice(&rgb.data(0)[start..start+w as usize*3]);}
frames.push(Frame{width:w,height:h,rgb:data});
frames.push(Frame{width:w,height:h,rgb:data,content:None});
} Ok(frames)
}
}
+46 -8
View File
@@ -1,13 +1,16 @@
//! H.264/H.265 decoding through an external `ffmpeg` process, like the host's encoder pipeline.
//! Needs only the ffmpeg CLI, not FFmpeg development libraries.
//! Output is scaled into a fixed letterboxed size so every frame has a known byte length.
//! Output is scaled into a fixed letterboxed canvas so every frame has a known byte length; the
//! source size from ffmpeg's stream line tells the window which sub-rectangle is picture content.
use super::desktop::{Decoder,Frame};
use super::media::Codec;
use std::io::{Read,Write};
use std::io::{BufRead,Read,Write};
use std::process::{Child,ChildStdin,Command,Stdio};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64,Ordering};
use std::sync::mpsc::{Receiver,TryRecvError};
pub struct Cli { child:Child, input:Option<ChildStdin>, codec:Codec, frames:Receiver<Vec<u8>>, width:u32, height:u32 }
pub struct Cli { child:Child, input:Option<ChildStdin>, codec:Codec, frames:Receiver<Vec<u8>>, width:u32, height:u32, source:Arc<AtomicU64> }
impl Cli {
/// `UU_VIEWER_SIZE=WxH` picks the decoded size (default 1920x1080); `UU_VIEWER_FFMPEG` the binary.
@@ -23,16 +26,27 @@ impl Cli {
let filter=format!("scale={width}:{height}:force_original_aspect_ratio=decrease,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2");
let passthrough=passthrough_args(&ffmpeg);
let mut child=Command::new(&ffmpeg)
.args(["-hide_banner","-loglevel","error","-nostdin","-threads","1","-probesize","32","-analyzeduration","0",
.args(["-hide_banner","-loglevel","info","-nostats","-nostdin","-threads","1","-probesize","32","-analyzeduration","0",
"-fflags","nobuffer","-flags","low_delay","-f",codec.demuxer(),"-i","pipe:0","-an","-vf",&filter,
"-pix_fmt","rgb24",
// Raw H.264/H.265 carries no timestamps; constant-rate output dropped almost every frame of
// the official Mac host's stream (270 → 2). Pass frames through unchanged.
passthrough.0,passthrough.1,"-f","rawvideo","pipe:1"])
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped())
.spawn().map_err(|e|format!("Start ffmpeg decoder: {e}"))?;
let input=child.stdin.take().ok_or("ffmpeg stdin missing")?;
let mut output=child.stdout.take().ok_or("ffmpeg stdout missing")?;
// Source picture size from the one-time stream line; 0 until parsed. Only the numbers are kept.
let source=Arc::new(AtomicU64::new(0));
if let Some(stderr)=child.stderr.take() {
let source=source.clone();
let _=std::thread::Builder::new().name("uu-viewer-ffmpeg-info".into()).spawn(move||{
for line in std::io::BufReader::new(stderr).lines().map_while(Result::ok) {
if source.load(Ordering::Relaxed)!=0 {continue}
if let Some((w,h))=parse_video_size(&line) {source.store((u64::from(w)<<32)|u64::from(h),Ordering::Relaxed);}
}
});
}
let size=width as usize*height as usize*3;
// Bounded: a slow window drops decoded frames instead of growing memory.
let (tx,frames)=std::sync::mpsc::sync_channel(2);
@@ -43,10 +57,23 @@ impl Cli {
match tx.try_send(frame) { Ok(())|Err(std::sync::mpsc::TrySendError::Full(_))=>{}, Err(_)=>break }
}
}).map_err(|e|format!("Decoder reader: {e}"))?;
Ok(Self{child,input:Some(input),codec,frames,width,height})
Ok(Self{child,input:Some(input),codec,frames,width,height,source})
}
}
/// `30000/1001`-style or display strings are irrelevant here: the first WxH on a `Video:` line is the
/// coded picture size, which is the aspect the scale+pad filter letterboxes into the canvas.
fn parse_video_size(line:&str)->Option<(u32,u32)> {
let rest=line.split_once("Video:")?.1;
rest.split(|c:char|c==','||c==' '||c=='('||c==')'||c=='['||c==']').find_map(|token|{
let token=token.trim_matches(|c:char|!c.is_ascii_digit()&&c!='x');
let (w,h)=token.split_once('x')?;
if w.is_empty()||h.is_empty()||!w.bytes().all(|b|b.is_ascii_digit())||!h.bytes().all(|b|b.is_ascii_digit()) {return None}
let (w,h)=(w.parse::<u32>().ok()?,h.parse::<u32>().ok()?);
((16..=16384).contains(&w)&&(16..=16384).contains(&h)).then_some((w,h))
})
}
/// FFmpeg ≥ 5.1 has `-fps_mode`; FFmpeg 8+ removed `-vsync`, and Ubuntu 22.04's 4.4 lacks `-fps_mode`.
/// Probe once per binary with a one-frame null run instead of parsing version strings.
fn passthrough_args(ffmpeg:&str)->(&'static str,&'static str) {
@@ -75,7 +102,11 @@ impl Decoder for Cli {
let mut out=Vec::new();
loop {
match self.frames.try_recv() {
Ok(rgb)=>out.push(Frame{width:self.width,height:self.height,rgb}),
Ok(rgb)=>{
let source=self.source.load(Ordering::Relaxed);
let content=(source!=0).then(||((source>>32) as u32,source as u32));
out.push(Frame{width:self.width,height:self.height,rgb,content})
},
Err(TryRecvError::Empty)=>break,
Err(TryRecvError::Disconnected)=>return Err("ffmpeg decoder exited".into()),
}
@@ -115,7 +146,7 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi
// Exercise the same streaming process and RGB conversion as SDL, then drain at EOF.
let mut frames=cli.decode(data).unwrap();drop(cli.input.take());
loop {match cli.frames.recv_timeout(std::time::Duration::from_secs(5)) {
Ok(rgb)=>frames.push(Frame{width:32,height:32,rgb}),
Ok(rgb)=>frames.push(Frame{width:32,height:32,rgb,content:None}),
Err(std::sync::mpsc::RecvTimeoutError::Disconnected)=>break,
Err(e)=>panic!("{codec:?} decoder did not finish: {e}"),
}}
@@ -128,4 +159,11 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi
}
}
}
#[test] fn source_size_is_read_from_the_ffmpeg_stream_line(){
assert_eq!(super::parse_video_size(" Stream #0:0: Video: h264 (High), yuv420p(progressive), 3024x1964, 25 fps"),Some((3024,1964)));
assert_eq!(super::parse_video_size("Stream #0:0: Video: hevc, yuv420p, 1920x1080 [SAR 1:1 DAR 16:9]"),Some((1920,1080)));
assert_eq!(super::parse_video_size("Stream #0:0: Audio: opus, 48000 Hz, stereo"),None);
assert_eq!(super::parse_video_size("frame= 1 fps=0.0 q=0.0"),None);
assert_eq!(super::parse_video_size("Stream #0:0: Video: h264, 1920xsynth"),None);
}
}
+3 -2
View File
@@ -5,7 +5,8 @@ pub struct Desktop { pub units:Latest<super::media::AccessUnit>, pub pli:Arc<Ato
impl Default for Desktop {
fn default()->Self {Self{units:Latest::new(3),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}}
}
pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec<u8> }
/// `content` is the source picture size inside the (possibly padded) RGB canvas; None = full frame.
pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec<u8>, pub content:Option<(u32,u32)> }
pub trait Decoder { fn set_codec(&mut self,codec:super::media::Codec)->Result<(),String>; fn decode(&mut self, au:&[u8])->Result<Vec<Frame>,String>; fn reset(&mut self); }
pub trait FrameSink { fn present(&mut self,frame:&Frame)->Result<(),String>; }
/// Counts only (access units, presented frames, decode errors) for the periodic stats line.
@@ -29,7 +30,7 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s
}
#[cfg(test)] mod tests {
use super::*;
struct Mock(bool);impl Decoder for Mock {fn set_codec(&mut self,_:crate::viewer::media::Codec)->Result<(),String>{Ok(())} fn decode(&mut self,_:&[u8])->Result<Vec<Frame>,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3]}])}} fn reset(&mut self){self.0=false;}}
struct Mock(bool);impl Decoder for Mock {fn set_codec(&mut self,_:crate::viewer::media::Codec)->Result<(),String>{Ok(())} fn decode(&mut self,_:&[u8])->Result<Vec<Frame>,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3],content:None}])}} fn reset(&mut self){self.0=false;}}
struct Sink(usize);impl FrameSink for Sink{fn present(&mut self,f:&Frame)->Result<(),String>{assert_eq!(f.rgb.len(),3);self.0+=1;Ok(())}}
#[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert_eq!(s.0,1);}
}
+26
View File
@@ -35,6 +35,15 @@ pub fn mac_keycode(scan:i32)->Option<i32>{
_=>return None,
})
}
/// The content rectangle inside a decoder canvas padded to a fixed size (ffmpeg's scale+pad).
/// Mirrors `force_original_aspect_ratio=decrease,pad=…:(ow-iw)/2:(oh-ih)/2` so pointer input
/// ignores the padding instead of compressing the valid area.
pub fn content_rect(canvas:(u32,u32),content:(u32,u32))->Option<(f64,f64,f64,f64)> {
if canvas.0==0||canvas.1==0||content.0==0||content.1==0 {return None}
let scale=(canvas.0 as f64/content.0 as f64).min(canvas.1 as f64/content.1 as f64);
let(w,h)=(content.0 as f64*scale,content.1 as f64*scale);
Some(((canvas.0 as f64-w)/2.,(canvas.1 as f64-h)/2.,w,h))
}
#[derive(Clone,Copy,Debug,PartialEq)]
pub struct Viewport { pub x:f64, pub y:f64, pub width:f64, pub height:f64 }
impl Viewport {
@@ -138,4 +147,21 @@ assert!(i.release_all().is_empty());
assert_eq!(v.motion(1010.,1000.).unwrap()["abs_x"],1.0);
assert!(v.motion(f64::NAN,0.).is_none());
}
#[test] fn padded_canvas_maps_only_its_content_rectangle(){
// 16:10 picture inside the default 16:9 canvas: 1728x1080 centred, 96 px pillarbox each side.
assert_eq!(content_rect((1920,1080),(1280,800)),Some((96.,0.,1728.,1080.)));
assert_eq!(content_rect((1920,1080),(1920,1080)),Some((0.,0.,1920.,1080.)));
// 4:3 picture: 1440x1080 centred, 240 px each side.
assert_eq!(content_rect((1920,1080),(640,480)),Some((240.,0.,1440.,1080.)));
assert_eq!(content_rect((0,1080),(640,480)),None);
// The content rectangle's edges are the screen edges, not the canvas edges.
let (x,y,w,h)=content_rect((1920,1080),(1280,800)).unwrap();
let v=Viewport::fit((1920,1080),(1280,800));
assert_eq!((x,y,w,h),(v.x,v.y,v.width,v.height));
let abs=|px:f64|v.motion(px,540.).unwrap();
assert_eq!(abs(96.)["abs_x"],0.0);
assert_eq!(abs(960.)["abs_x"],0.5);
assert_eq!(abs(1824.)["abs_x"],1.0);
assert_eq!(abs(0.)["abs_x"],0.0); // padding is clamped to the screen edge
}
}
+8 -3
View File
@@ -89,12 +89,17 @@ fn translate_actions(events:impl IntoIterator<Item=Event>,input:&mut Input,viewp
fn button(b:MouseButton)->Option<i32>{match b{MouseButton::Left=>Some(1),MouseButton::Right=>Some(2),MouseButton::Middle=>Some(4),_=>None}}
impl FrameSink for Window {
fn present(&mut self,frame:&Frame)->Result<(),String>{
self.last_frame=Some(Frame{width:frame.width,height:frame.height,rgb:frame.rgb.clone()});
self.frame=(frame.width,frame.height);let v=self.viewport();let creator=self.canvas.texture_creator();
self.last_frame=Some(Frame{width:frame.width,height:frame.height,rgb:frame.rgb.clone(),content:frame.content});
// Pad bars belong to the decoded canvas, not the remote picture: viewport and input use the
// content aspect, and only the content sub-rectangle of the texture is drawn.
let content=frame.content.unwrap_or((frame.width,frame.height));
self.frame=content;let v=self.viewport();let creator=self.canvas.texture_creator();
let mut texture=creator.create_texture_streaming(PixelFormatEnum::RGB24,frame.width,frame.height).map_err(|_|"Texture creation failed")?;
texture.update(None,&frame.rgb,frame.width as usize*3).map_err(|_|"Texture update failed")?;
self.canvas.set_draw_color(sdl2::pixels::Color::BLACK);self.canvas.clear();
self.canvas.copy(&texture,None,Some(Rect::new(v.x as i32,v.y as i32,v.width as u32,v.height as u32)))?;self.canvas.present();Ok(())
let src=super::input::content_rect((frame.width,frame.height),content)
.map(|(x,y,w,h)|Rect::new(x.round() as i32,y.round() as i32,w.round().max(1.) as u32,h.round().max(1.) as u32));
self.canvas.copy(&texture,src,Some(Rect::new(v.x.round() as i32,v.y.round() as i32,v.width.round().max(1.) as u32,v.height.round().max(1.) as u32)))?;self.canvas.present();Ok(())
}
}
pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super::desktop::Decoder,view_only:bool,device:&str)->Result<(),String>{