mirror of
https://github.com/VallierDev/uu-remote-linux.git
synced 2026-11-16 08:50:40 +01:00
viewer: map pointer input to the content rectangle of the padded canvas (#107)
The CLI decoder pads every frame into a fixed 1920x1080 canvas so rawvideo frames have a known byte length. Input mapping and the SDL viewport treated that padded canvas as the picture, so non-16:9 hosts (official Macs are usually 16:10) had the pillarbox bars inside the 0..1 range: the remote pointer lagged inward and the error grew towards the screen edges. Parse the coded picture size from ffmpeg's one-time "Video:" stream line, carry it on Frame.content, and render/map only the content sub-rectangle. 16:9 streams are unchanged; UU_VIEWER_SIZE keeps working.
This commit is contained in:
1 parent
bcde9f967c
commit
287b21a622
5 files changed
+84
-14
No files matched your search
@@ -37,7 +37,7 @@ impl Decoder for Ffmpeg {
|
||||
let mut rgb=av::frame::Video::empty();scaler.run(&decoded,&mut rgb).map_err(|_|"Frame conversion failed")?;
|
||||
let mut data=Vec::with_capacity(w as usize*h as usize*3);
|
||||
for row in 0..h as usize {let start=row*rgb.stride(0);data.extend_from_slice(&rgb.data(0)[start..start+w as usize*3]);}
|
||||
frames.push(Frame{width:w,height:h,rgb:data});
|
||||
frames.push(Frame{width:w,height:h,rgb:data,content:None});
|
||||
} Ok(frames)
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,16 @@
|
||||
//! H.264/H.265 decoding through an external `ffmpeg` process, like the host's encoder pipeline.
|
||||
//! Needs only the ffmpeg CLI, not FFmpeg development libraries.
|
||||
//! Output is scaled into a fixed letterboxed size so every frame has a known byte length.
|
||||
//! Output is scaled into a fixed letterboxed canvas so every frame has a known byte length; the
|
||||
//! source size from ffmpeg's stream line tells the window which sub-rectangle is picture content.
|
||||
use super::desktop::{Decoder,Frame};
|
||||
use super::media::Codec;
|
||||
use std::io::{Read,Write};
|
||||
use std::io::{BufRead,Read,Write};
|
||||
use std::process::{Child,ChildStdin,Command,Stdio};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64,Ordering};
|
||||
use std::sync::mpsc::{Receiver,TryRecvError};
|
||||
|
||||
pub struct Cli { child:Child, input:Option<ChildStdin>, codec:Codec, frames:Receiver<Vec<u8>>, width:u32, height:u32 }
|
||||
pub struct Cli { child:Child, input:Option<ChildStdin>, codec:Codec, frames:Receiver<Vec<u8>>, width:u32, height:u32, source:Arc<AtomicU64> }
|
||||
|
||||
impl Cli {
|
||||
/// `UU_VIEWER_SIZE=WxH` picks the decoded size (default 1920x1080); `UU_VIEWER_FFMPEG` the binary.
|
||||
@@ -23,16 +26,27 @@ impl Cli {
|
||||
let filter=format!("scale={width}:{height}:force_original_aspect_ratio=decrease,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2");
|
||||
let passthrough=passthrough_args(&ffmpeg);
|
||||
let mut child=Command::new(&ffmpeg)
|
||||
.args(["-hide_banner","-loglevel","error","-nostdin","-threads","1","-probesize","32","-analyzeduration","0",
|
||||
.args(["-hide_banner","-loglevel","info","-nostats","-nostdin","-threads","1","-probesize","32","-analyzeduration","0",
|
||||
"-fflags","nobuffer","-flags","low_delay","-f",codec.demuxer(),"-i","pipe:0","-an","-vf",&filter,
|
||||
"-pix_fmt","rgb24",
|
||||
// Raw H.264/H.265 carries no timestamps; constant-rate output dropped almost every frame of
|
||||
// the official Mac host's stream (270 → 2). Pass frames through unchanged.
|
||||
passthrough.0,passthrough.1,"-f","rawvideo","pipe:1"])
|
||||
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null())
|
||||
.stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped())
|
||||
.spawn().map_err(|e|format!("Start ffmpeg decoder: {e}"))?;
|
||||
let input=child.stdin.take().ok_or("ffmpeg stdin missing")?;
|
||||
let mut output=child.stdout.take().ok_or("ffmpeg stdout missing")?;
|
||||
// Source picture size from the one-time stream line; 0 until parsed. Only the numbers are kept.
|
||||
let source=Arc::new(AtomicU64::new(0));
|
||||
if let Some(stderr)=child.stderr.take() {
|
||||
let source=source.clone();
|
||||
let _=std::thread::Builder::new().name("uu-viewer-ffmpeg-info".into()).spawn(move||{
|
||||
for line in std::io::BufReader::new(stderr).lines().map_while(Result::ok) {
|
||||
if source.load(Ordering::Relaxed)!=0 {continue}
|
||||
if let Some((w,h))=parse_video_size(&line) {source.store((u64::from(w)<<32)|u64::from(h),Ordering::Relaxed);}
|
||||
}
|
||||
});
|
||||
}
|
||||
let size=width as usize*height as usize*3;
|
||||
// Bounded: a slow window drops decoded frames instead of growing memory.
|
||||
let (tx,frames)=std::sync::mpsc::sync_channel(2);
|
||||
@@ -43,10 +57,23 @@ impl Cli {
|
||||
match tx.try_send(frame) { Ok(())|Err(std::sync::mpsc::TrySendError::Full(_))=>{}, Err(_)=>break }
|
||||
}
|
||||
}).map_err(|e|format!("Decoder reader: {e}"))?;
|
||||
Ok(Self{child,input:Some(input),codec,frames,width,height})
|
||||
Ok(Self{child,input:Some(input),codec,frames,width,height,source})
|
||||
}
|
||||
}
|
||||
|
||||
/// `30000/1001`-style or display strings are irrelevant here: the first WxH on a `Video:` line is the
|
||||
/// coded picture size, which is the aspect the scale+pad filter letterboxes into the canvas.
|
||||
fn parse_video_size(line:&str)->Option<(u32,u32)> {
|
||||
let rest=line.split_once("Video:")?.1;
|
||||
rest.split(|c:char|c==','||c==' '||c=='('||c==')'||c=='['||c==']').find_map(|token|{
|
||||
let token=token.trim_matches(|c:char|!c.is_ascii_digit()&&c!='x');
|
||||
let (w,h)=token.split_once('x')?;
|
||||
if w.is_empty()||h.is_empty()||!w.bytes().all(|b|b.is_ascii_digit())||!h.bytes().all(|b|b.is_ascii_digit()) {return None}
|
||||
let (w,h)=(w.parse::<u32>().ok()?,h.parse::<u32>().ok()?);
|
||||
((16..=16384).contains(&w)&&(16..=16384).contains(&h)).then_some((w,h))
|
||||
})
|
||||
}
|
||||
|
||||
/// FFmpeg ≥ 5.1 has `-fps_mode`; FFmpeg 8+ removed `-vsync`, and Ubuntu 22.04's 4.4 lacks `-fps_mode`.
|
||||
/// Probe once per binary with a one-frame null run instead of parsing version strings.
|
||||
fn passthrough_args(ffmpeg:&str)->(&'static str,&'static str) {
|
||||
@@ -75,7 +102,11 @@ impl Decoder for Cli {
|
||||
let mut out=Vec::new();
|
||||
loop {
|
||||
match self.frames.try_recv() {
|
||||
Ok(rgb)=>out.push(Frame{width:self.width,height:self.height,rgb}),
|
||||
Ok(rgb)=>{
|
||||
let source=self.source.load(Ordering::Relaxed);
|
||||
let content=(source!=0).then(||((source>>32) as u32,source as u32));
|
||||
out.push(Frame{width:self.width,height:self.height,rgb,content})
|
||||
},
|
||||
Err(TryRecvError::Empty)=>break,
|
||||
Err(TryRecvError::Disconnected)=>return Err("ffmpeg decoder exited".into()),
|
||||
}
|
||||
@@ -115,7 +146,7 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi
|
||||
// Exercise the same streaming process and RGB conversion as SDL, then drain at EOF.
|
||||
let mut frames=cli.decode(data).unwrap();drop(cli.input.take());
|
||||
loop {match cli.frames.recv_timeout(std::time::Duration::from_secs(5)) {
|
||||
Ok(rgb)=>frames.push(Frame{width:32,height:32,rgb}),
|
||||
Ok(rgb)=>frames.push(Frame{width:32,height:32,rgb,content:None}),
|
||||
Err(std::sync::mpsc::RecvTimeoutError::Disconnected)=>break,
|
||||
Err(e)=>panic!("{codec:?} decoder did not finish: {e}"),
|
||||
}}
|
||||
@@ -128,4 +159,11 @@ impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.chi
|
||||
}
|
||||
}
|
||||
}
|
||||
#[test] fn source_size_is_read_from_the_ffmpeg_stream_line(){
|
||||
assert_eq!(super::parse_video_size(" Stream #0:0: Video: h264 (High), yuv420p(progressive), 3024x1964, 25 fps"),Some((3024,1964)));
|
||||
assert_eq!(super::parse_video_size("Stream #0:0: Video: hevc, yuv420p, 1920x1080 [SAR 1:1 DAR 16:9]"),Some((1920,1080)));
|
||||
assert_eq!(super::parse_video_size("Stream #0:0: Audio: opus, 48000 Hz, stereo"),None);
|
||||
assert_eq!(super::parse_video_size("frame= 1 fps=0.0 q=0.0"),None);
|
||||
assert_eq!(super::parse_video_size("Stream #0:0: Video: h264, 1920xsynth"),None);
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,8 @@ pub struct Desktop { pub units:Latest<super::media::AccessUnit>, pub pli:Arc<Ato
|
||||
impl Default for Desktop {
|
||||
fn default()->Self {Self{units:Latest::new(3),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}}
|
||||
}
|
||||
pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec<u8> }
|
||||
/// `content` is the source picture size inside the (possibly padded) RGB canvas; None = full frame.
|
||||
pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec<u8>, pub content:Option<(u32,u32)> }
|
||||
pub trait Decoder { fn set_codec(&mut self,codec:super::media::Codec)->Result<(),String>; fn decode(&mut self, au:&[u8])->Result<Vec<Frame>,String>; fn reset(&mut self); }
|
||||
pub trait FrameSink { fn present(&mut self,frame:&Frame)->Result<(),String>; }
|
||||
/// Counts only (access units, presented frames, decode errors) for the periodic stats line.
|
||||
@@ -29,7 +30,7 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:s
|
||||
}
|
||||
#[cfg(test)] mod tests {
|
||||
use super::*;
|
||||
struct Mock(bool);impl Decoder for Mock {fn set_codec(&mut self,_:crate::viewer::media::Codec)->Result<(),String>{Ok(())} fn decode(&mut self,_:&[u8])->Result<Vec<Frame>,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3]}])}} fn reset(&mut self){self.0=false;}}
|
||||
struct Mock(bool);impl Decoder for Mock {fn set_codec(&mut self,_:crate::viewer::media::Codec)->Result<(),String>{Ok(())} fn decode(&mut self,_:&[u8])->Result<Vec<Frame>,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3],content:None}])}} fn reset(&mut self){self.0=false;}}
|
||||
struct Sink(usize);impl FrameSink for Sink{fn present(&mut self,f:&Frame)->Result<(),String>{assert_eq!(f.rgb.len(),3);self.0+=1;Ok(())}}
|
||||
#[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert_eq!(s.0,1);}
|
||||
}
|
||||
@@ -35,6 +35,15 @@ pub fn mac_keycode(scan:i32)->Option<i32>{
|
||||
_=>return None,
|
||||
})
|
||||
}
|
||||
/// The content rectangle inside a decoder canvas padded to a fixed size (ffmpeg's scale+pad).
|
||||
/// Mirrors `force_original_aspect_ratio=decrease,pad=…:(ow-iw)/2:(oh-ih)/2` so pointer input
|
||||
/// ignores the padding instead of compressing the valid area.
|
||||
pub fn content_rect(canvas:(u32,u32),content:(u32,u32))->Option<(f64,f64,f64,f64)> {
|
||||
if canvas.0==0||canvas.1==0||content.0==0||content.1==0 {return None}
|
||||
let scale=(canvas.0 as f64/content.0 as f64).min(canvas.1 as f64/content.1 as f64);
|
||||
let(w,h)=(content.0 as f64*scale,content.1 as f64*scale);
|
||||
Some(((canvas.0 as f64-w)/2.,(canvas.1 as f64-h)/2.,w,h))
|
||||
}
|
||||
#[derive(Clone,Copy,Debug,PartialEq)]
|
||||
pub struct Viewport { pub x:f64, pub y:f64, pub width:f64, pub height:f64 }
|
||||
impl Viewport {
|
||||
@@ -138,4 +147,21 @@ assert!(i.release_all().is_empty());
|
||||
assert_eq!(v.motion(1010.,1000.).unwrap()["abs_x"],1.0);
|
||||
assert!(v.motion(f64::NAN,0.).is_none());
|
||||
}
|
||||
#[test] fn padded_canvas_maps_only_its_content_rectangle(){
|
||||
// 16:10 picture inside the default 16:9 canvas: 1728x1080 centred, 96 px pillarbox each side.
|
||||
assert_eq!(content_rect((1920,1080),(1280,800)),Some((96.,0.,1728.,1080.)));
|
||||
assert_eq!(content_rect((1920,1080),(1920,1080)),Some((0.,0.,1920.,1080.)));
|
||||
// 4:3 picture: 1440x1080 centred, 240 px each side.
|
||||
assert_eq!(content_rect((1920,1080),(640,480)),Some((240.,0.,1440.,1080.)));
|
||||
assert_eq!(content_rect((0,1080),(640,480)),None);
|
||||
// The content rectangle's edges are the screen edges, not the canvas edges.
|
||||
let (x,y,w,h)=content_rect((1920,1080),(1280,800)).unwrap();
|
||||
let v=Viewport::fit((1920,1080),(1280,800));
|
||||
assert_eq!((x,y,w,h),(v.x,v.y,v.width,v.height));
|
||||
let abs=|px:f64|v.motion(px,540.).unwrap();
|
||||
assert_eq!(abs(96.)["abs_x"],0.0);
|
||||
assert_eq!(abs(960.)["abs_x"],0.5);
|
||||
assert_eq!(abs(1824.)["abs_x"],1.0);
|
||||
assert_eq!(abs(0.)["abs_x"],0.0); // padding is clamped to the screen edge
|
||||
}
|
||||
}
|
||||
+8
-3
@@ -89,12 +89,17 @@ fn translate_actions(events:impl IntoIterator<Item=Event>,input:&mut Input,viewp
|
||||
fn button(b:MouseButton)->Option<i32>{match b{MouseButton::Left=>Some(1),MouseButton::Right=>Some(2),MouseButton::Middle=>Some(4),_=>None}}
|
||||
impl FrameSink for Window {
|
||||
fn present(&mut self,frame:&Frame)->Result<(),String>{
|
||||
self.last_frame=Some(Frame{width:frame.width,height:frame.height,rgb:frame.rgb.clone()});
|
||||
self.frame=(frame.width,frame.height);let v=self.viewport();let creator=self.canvas.texture_creator();
|
||||
self.last_frame=Some(Frame{width:frame.width,height:frame.height,rgb:frame.rgb.clone(),content:frame.content});
|
||||
// Pad bars belong to the decoded canvas, not the remote picture: viewport and input use the
|
||||
// content aspect, and only the content sub-rectangle of the texture is drawn.
|
||||
let content=frame.content.unwrap_or((frame.width,frame.height));
|
||||
self.frame=content;let v=self.viewport();let creator=self.canvas.texture_creator();
|
||||
let mut texture=creator.create_texture_streaming(PixelFormatEnum::RGB24,frame.width,frame.height).map_err(|_|"Texture creation failed")?;
|
||||
texture.update(None,&frame.rgb,frame.width as usize*3).map_err(|_|"Texture update failed")?;
|
||||
self.canvas.set_draw_color(sdl2::pixels::Color::BLACK);self.canvas.clear();
|
||||
self.canvas.copy(&texture,None,Some(Rect::new(v.x as i32,v.y as i32,v.width as u32,v.height as u32)))?;self.canvas.present();Ok(())
|
||||
let src=super::input::content_rect((frame.width,frame.height),content)
|
||||
.map(|(x,y,w,h)|Rect::new(x.round() as i32,y.round() as i32,w.round().max(1.) as u32,h.round().max(1.) as u32));
|
||||
self.canvas.copy(&texture,src,Some(Rect::new(v.x.round() as i32,v.y.round() as i32,v.width.round().max(1.) as u32,v.height.round().max(1.) as u32)))?;self.canvas.present();Ok(())
|
||||
}
|
||||
}
|
||||
pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super::desktop::Decoder,view_only:bool,device:&str)->Result<(),String>{
|
||||
|
||||
Reference in new issue
Block a user