diff --git a/README.en.md b/README.en.md index ed0c1cd..61c1fec 100644 --- a/README.en.md +++ b/README.en.md @@ -425,6 +425,9 @@ terminal emulator (`TERMINAL` selects one). From source: `sudo apt install libsd `assist CONNECT_ID` (the code is read from one stdin line, never an argument). See `uuremote-viewer --help`. - Ctrl+Alt+F in the desktop window opens the files window on the same connection. `UU_VIEWER_AUDIO=0` disables sound, `UU_VIEWER_CLIPBOARD=0` the clipboard, `UU_VIEWER_VIEW_ONLY=1` input; `UU_VIEWER_FFMPEG` selects the ffmpeg binary. +- Desktop offers H.264 / H.265; the host chooses the codec. `UU_VIEWER_CODEC=h264` limits both SDP and decoder capabilities to H.264. + The default request stays 4:2:0; `UU_VIEWER_CHROMA=444` requests true color (4:4:4) after CONTROL opens. The host may fall back to + 4:2:0, as reported by CaptureSettingSync. FFmpeg converts either chroma format to RGB24 for display. H.265 / 4:4:4 are **not tested live**. **Tested live** (author's setup, not full acceptance): | Feature | Official Windows host | Official Mac host | This project's Linux host | @@ -439,5 +442,5 @@ terminal emulator (`TERMINAL` selects one). From source: `sudo apt install libsd | Reboot, shutdown | — | — | ✅ (VM) | | Wayland desktop window and clipboard (Hyprland) | — | — | ✅ (VM) | -"—" = not tested yet. Only use reboot/shutdown on machines you can restart. **Not tested live**: GNOME/KDE Wayland windows and clipboard, H.265 and 4:4:4 (only H.264 4:2:0 is decoded), multiple monitors. Logs carry field +"—" = not tested yet. Only use reboot/shutdown on machines you can restart. **Not tested live**: GNOME/KDE Wayland windows and clipboard, H.265 and 4:4:4, multiple monitors. Logs carry field numbers, sizes and states only, never keystrokes, clipboard or file contents, codes or credentials. diff --git a/README.md b/README.md index e31a7a3..5c3265b 100644 --- a/README.md +++ b/README.md @@ -389,6 +389,9 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。 `assist CONNECT_ID`(验证码从 stdin 读取一行,不能作为参数传入)。`uuremote-viewer --help` 查看全部参数。 - 桌面窗口中按 Ctrl+Alt+F 打开文件窗口(复用同一连接)。`UU_VIEWER_AUDIO=0` 关闭声音,`UU_VIEWER_CLIPBOARD=0` 关闭剪贴板, `UU_VIEWER_VIEW_ONLY=1` 只看不控,`UU_VIEWER_FFMPEG` 指定 ffmpeg 路径。 +- 桌面同时提供 H.264 / H.265,由主机协商选择;`UU_VIEWER_CODEC=h264` 将 SDP 和解码能力限制为 H.264。 + 默认请求 4:2:0;`UU_VIEWER_CHROMA=444` 在 CONTROL 打开后请求真彩(4:4:4)。主机可能回落到 4:2:0, + 以 CaptureSettingSync 回报为准。两种色度均由 FFmpeg 转为 RGB24 显示。H.265 / 4:4:4 **尚未实机测试**。 **已实测**(作者环境,非全面验收): | 功能 | 官方 Windows 被控端 | 官方 Mac 被控端 | 本项目 Linux 主机 | @@ -404,4 +407,4 @@ xfce4-session xfce4-panel xfdesktop xfwm4 gnome-terminal`。 | Wayland 下的桌面窗口与剪贴板(Hyprland) | — | — | ✅(虚拟机) | 「—」表示尚未测试。重启/关机只应在可以重启的机器上使用。**未实测**:GNOME/KDE Wayland 下的窗口与剪贴板、 -H.265 与 4:4:4(目前只解码 H.264 4:2:0)、多显示器。日志只记录字段号、大小和状态,不记录键盘、剪贴板、文件内容、验证码或凭据。 +H.265 与 4:4:4、多显示器。日志只记录字段号、大小和状态,不记录键盘、剪贴板、文件内容、验证码或凭据。 diff --git a/src/viewer/decoder.rs b/src/viewer/decoder.rs index 3e40f0a..daa32a4 100644 --- a/src/viewer/decoder.rs +++ b/src/viewer/decoder.rs @@ -1,29 +1,36 @@ //! Uses the same optional FFmpeg dependency as the host inproc-av feature. use super::desktop::{Decoder,Frame}; use ffmpeg_next as av; -pub struct Ffmpeg { decoder:av::decoder::Video } +use super::media::Codec; +pub struct Ffmpeg { decoder:av::decoder::Video, codec:Codec } impl Ffmpeg { pub fn new()->Result { + Self::with_codec(Codec::H264) + } + fn with_codec(kind:Codec)->Result{ av::init().map_err(|_|"FFmpeg initialization failed")?; // Library diagnostics can contain bitstream data; suppress them globally in this viewer process. av::log::set_level(av::log::Level::Quiet); - let codec=av::decoder::find(av::codec::Id::H264).ok_or("H264 decoder unavailable")?; - let decoder=av::codec::context::Context::new().decoder().open_as(codec).and_then(|d|d.video()).map_err(|_|"H264 decoder initialization failed")?; - Ok(Self{decoder}) + let codec=av::decoder::find(match kind {Codec::H264=>av::codec::Id::H264,Codec::H265=>av::codec::Id::HEVC}).ok_or("Video decoder unavailable")?; + let decoder=av::codec::context::Context::new().decoder().open_as(codec).and_then(|d|d.video()).map_err(|_|"Video decoder initialization failed")?; + Ok(Self{decoder,codec:kind}) } } impl Decoder for Ffmpeg { + fn set_codec(&mut self,codec:Codec)->Result<(),String>{ + if self.codec!=codec {*self=Self::with_codec(codec)?;}Ok(()) + } fn reset(&mut self){self.decoder.flush();} fn decode(&mut self,au:&[u8])->Result,String>{ - self.decoder.send_packet(&av::Packet::copy(au)).map_err(|_|"H264 packet rejected")?; + self.decoder.send_packet(&av::Packet::copy(au)).map_err(|_|"Video packet rejected")?; let mut frames=Vec::new();let mut decoded=av::frame::Video::empty(); loop { match self.decoder.receive_frame(&mut decoded) { Ok(())=>{}, Err(av::Error::Other{errno}) if errno==libc::EAGAIN=>break, - Err(_)=>return Err("H264 decode failed".into()), + Err(_)=>return Err("Video decode failed".into()), } - if decoded.is_corrupt(){return Err("Corrupt H264 frame".into())} + if decoded.is_corrupt(){return Err("Corrupt video frame".into())} let (w,h)=(decoded.width(),decoded.height()); if w==0 || h==0 || w>8192 || h>8192 {return Err("Decoded dimensions exceed limit".into())} let mut scaler=av::software::scaling::context::Context::get(decoded.format(),w,h,av::format::Pixel::RGB24,w,h,av::software::scaling::flag::Flags::BILINEAR).map_err(|_|"Frame conversion unavailable")?; diff --git a/src/viewer/decoder_cli.rs b/src/viewer/decoder_cli.rs index 2f3d457..8eb26f6 100644 --- a/src/viewer/decoder_cli.rs +++ b/src/viewer/decoder_cli.rs @@ -1,12 +1,13 @@ -//! H.264 decoding through an external `ffmpeg` process, like the host's encoder pipeline. +//! H.264/H.265 decoding through an external `ffmpeg` process, like the host's encoder pipeline. //! Needs only the ffmpeg CLI (any recent version), not FFmpeg development libraries. //! Output is scaled into a fixed letterboxed size so every frame has a known byte length. use super::desktop::{Decoder,Frame}; +use super::media::Codec; use std::io::{Read,Write}; use std::process::{Child,ChildStdin,Command,Stdio}; use std::sync::mpsc::{Receiver,TryRecvError}; -pub struct Cli { child:Child, input:ChildStdin, frames:Receiver>, width:u32, height:u32 } +pub struct Cli { child:Child, input:Option, codec:Codec, frames:Receiver>, width:u32, height:u32 } impl Cli { /// `UU_VIEWER_SIZE=WxH` picks the decoded size (default 1920x1080); `UU_VIEWER_FFMPEG` the binary. @@ -15,13 +16,16 @@ impl Cli { .and_then(|v|{let (w,h)=v.split_once('x')?;Some((w.parse().ok()?,h.parse().ok()?))}) .filter(|&(w,h):&(u32,u32)|(16..=8192).contains(&w)&&(16..=8192).contains(&h)&&w%2==0&&h%2==0) .unwrap_or((1920,1080)); + Self::with_codec(Codec::H264,width,height) + } + fn with_codec(codec:Codec,width:u32,height:u32)->Result { let ffmpeg=std::env::var("UU_VIEWER_FFMPEG").unwrap_or_else(|_|"ffmpeg".into()); let filter=format!("scale={width}:{height}:force_original_aspect_ratio=decrease,pad={width}:{height}:(ow-iw)/2:(oh-ih)/2"); let mut child=Command::new(ffmpeg) .args(["-hide_banner","-loglevel","error","-nostdin","-threads","1","-probesize","32","-analyzeduration","0", - "-fflags","nobuffer","-flags","low_delay","-f","h264","-i","pipe:0","-an","-vf",&filter, + "-fflags","nobuffer","-flags","low_delay","-f",codec.demuxer(),"-i","pipe:0","-an","-vf",&filter, "-pix_fmt","rgb24", - // Raw H.264 carries no timestamps; constant-rate output dropped almost every frame of + // Raw H.264/H.265 carries no timestamps; constant-rate output dropped almost every frame of // the official Mac host's stream (270 → 2). Pass frames through unchanged. "-fps_mode","passthrough","-f","rawvideo","pipe:1"]) .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::null()) @@ -38,14 +42,18 @@ impl Cli { match tx.try_send(frame) { Ok(())|Err(std::sync::mpsc::TrySendError::Full(_))=>{}, Err(_)=>break } } }).map_err(|e|format!("Decoder reader: {e}"))?; - Ok(Self{child,input,frames,width,height}) + Ok(Self{child,input:Some(input),codec,frames,width,height}) } } impl Decoder for Cli { + fn set_codec(&mut self,codec:Codec)->Result<(),String>{ + if self.codec!=codec {*self=Self::with_codec(codec,self.width,self.height)?;}Ok(()) + } fn reset(&mut self) {} fn decode(&mut self,au:&[u8])->Result,String> { - self.input.write_all(au).and_then(|_|self.input.flush()).map_err(|_|"ffmpeg decoder exited")?; + let input=self.input.as_mut().ok_or("ffmpeg input closed")?; + input.write_all(au).and_then(|_|input.flush()).map_err(|_|"ffmpeg decoder exited")?; let mut out=Vec::new(); loop { match self.frames.try_recv() { @@ -59,3 +67,36 @@ impl Decoder for Cli { } impl Drop for Cli { fn drop(&mut self) { let _=self.child.kill(); let _=self.child.wait(); } } + +#[cfg(test)] mod tests { + use super::*; + // Six synthetic red IDRs per fixture; no captured desktop or external encoder dependency. + #[test] fn ffmpeg_decodes_h264_and_hevc_420_and_444_to_rgb24(){ + match Command::new("ffmpeg").arg("-version").stdout(Stdio::null()).stderr(Stdio::null()).status(){ + Err(e) if e.kind()==std::io::ErrorKind::NotFound=>{eprintln!("skip: ffmpeg is not on PATH");return}, + result=>assert!(result.expect("ffmpeg availability").success()), + } + let mut cli=Cli::with_codec(Codec::H265,32,32).unwrap(); + for (codec,data) in [ + (Codec::H264,&include_bytes!("../../tests/fixtures/viewer/red-420.h264")[..]), + (Codec::H265,&include_bytes!("../../tests/fixtures/viewer/red-420.hevc")[..]), + (Codec::H264,&include_bytes!("../../tests/fixtures/viewer/red-444.h264")[..]), + (Codec::H265,&include_bytes!("../../tests/fixtures/viewer/red-444.hevc")[..]), + ] { + cli.set_codec(codec).unwrap(); + // Exercise the same streaming process and RGB conversion as SDL, then drain at EOF. + let mut frames=cli.decode(data).unwrap();drop(cli.input.take()); + loop {match cli.frames.recv_timeout(std::time::Duration::from_secs(5)) { + Ok(rgb)=>frames.push(Frame{width:32,height:32,rgb}), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected)=>break, + Err(e)=>panic!("{codec:?} decoder did not finish: {e}"), + }} + assert!(!frames.is_empty(),"{codec:?}: no RGB frames"); + for frame in frames { + assert_eq!(frame.rgb.len(),32*32*3); + assert!(frame.rgb.chunks_exact(3).all(|p|p[0]>220 && p[1]<30 && p[2]<30),"{codec:?}: RGB conversion"); + } + assert!(cli.child.wait().unwrap().success()); + } + } +} diff --git a/src/viewer/desktop.rs b/src/viewer/desktop.rs index 6604867..8deed95 100644 --- a/src/viewer/desktop.rs +++ b/src/viewer/desktop.rs @@ -1,19 +1,19 @@ use super::media::Latest; use std::sync::{Arc, atomic::{AtomicBool,AtomicU32,Ordering}}; #[derive(Clone)] -pub struct Desktop { pub units:Latest>, pub pli:Arc, pub reset:Arc, pub ssrc:Arc } +pub struct Desktop { pub units:Latest, pub pli:Arc, pub reset:Arc, pub ssrc:Arc } impl Default for Desktop { fn default()->Self {Self{units:Latest::new(3),pli:Arc::new(AtomicBool::new(false)),reset:Arc::new(AtomicBool::new(false)),ssrc:Arc::new(AtomicU32::new(0))}} } pub struct Frame { pub width:u32, pub height:u32, pub rgb:Vec } -pub trait Decoder { fn decode(&mut self, au:&[u8])->Result,String>; fn reset(&mut self); } +pub trait Decoder { fn set_codec(&mut self,codec:super::media::Codec)->Result<(),String>; fn decode(&mut self, au:&[u8])->Result,String>; fn reset(&mut self); } pub trait FrameSink { fn present(&mut self,frame:&Frame)->Result<(),String>; } /// Counts only (access units, presented frames, decode errors) for the periodic stats line. pub static STATS:[std::sync::atomic::AtomicU64;3]=[std::sync::atomic::AtomicU64::new(0),std::sync::atomic::AtomicU64::new(0),std::sync::atomic::AtomicU64::new(0)]; -pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],pli:&AtomicBool)->Result<(),String> { +pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],codec:super::media::Codec,pli:&AtomicBool)->Result<(),String> { STATS[0].fetch_add(1,Ordering::Relaxed); // Local diagnostic: UU_VIEWER_DUMP_H264=path appends the first 300 access units (owner's own stream). - if STATS[0].load(Ordering::Relaxed)<=300 { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_H264") { + if codec==super::media::Codec::H264 && STATS[0].load(Ordering::Relaxed)<=300 { if let Some(path)=std::env::var_os("UU_VIEWER_DUMP_H264") { use std::io::Write; if let Ok(mut f)=std::fs::OpenOptions::new().create(true).append(true).open(path){let _=f.write_all(au);} }} match decoder.decode(au) { @@ -29,7 +29,7 @@ pub fn display(decoder:&mut dyn Decoder,sink:&mut dyn FrameSink,au:&[u8],pli:&At } #[cfg(test)] mod tests { use super::*; - struct Mock(bool);impl Decoder for Mock {fn decode(&mut self,_:&[u8])->Result,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3]}])}} fn reset(&mut self){self.0=false;}} + struct Mock(bool);impl Decoder for Mock {fn set_codec(&mut self,_:crate::viewer::media::Codec)->Result<(),String>{Ok(())} fn decode(&mut self,_:&[u8])->Result,String>{if self.0{Err("synthetic".into())}else{Ok(vec![Frame{width:1,height:1,rgb:vec![0;3]}])}} fn reset(&mut self){self.0=false;}} struct Sink(usize);impl FrameSink for Sink{fn present(&mut self,f:&Frame)->Result<(),String>{assert_eq!(f.rgb.len(),3);self.0+=1;Ok(())}} - #[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],&p).unwrap();assert_eq!(s.0,1);} + #[test] fn decode_error_requests_pli_and_recovers(){let mut d=Mock(true);let mut s=Sink(0);let p=AtomicBool::new(false);display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert!(p.load(Ordering::Acquire));display(&mut d,&mut s,&[],crate::viewer::media::Codec::H264,&p).unwrap();assert_eq!(s.0,1);} } diff --git a/src/viewer/media.rs b/src/viewer/media.rs index 0216f6c..8e23de7 100644 --- a/src/viewer/media.rs +++ b/src/viewer/media.rs @@ -1,19 +1,32 @@ -//! Bounded RFC 6184 packetization-mode 1 receiver. No media contents in logs. +//! Bounded RFC 6184 / RFC 7798 non-interleaved receivers (no DONL). No media contents in logs. use std::{collections::VecDeque, sync::{Arc, Mutex}}; use std::time::{Duration,Instant}; +#[derive(Clone,Copy,Debug,Default,PartialEq,Eq)] +pub enum Codec { #[default] H264, H265 } +impl Codec { + pub fn from_mime(mime:&str)->Option { + if mime.eq_ignore_ascii_case("video/H264"){Some(Self::H264)} + else if mime.eq_ignore_ascii_case("video/H265"){Some(Self::H265)}else{None} + } + pub fn demuxer(self)->&'static str {match self {Self::H264=>"h264",Self::H265=>"hevc"}} +} +pub struct AccessUnit {pub codec:Codec,pub data:Vec} const MAX_AU: usize = 8 * 1024 * 1024; #[derive(Clone)] pub struct Packet { pub seq: u16, pub timestamp: u32, pub marker: bool, pub payload: Vec } #[derive(Default)] pub struct Output { pub units: Vec>, pub pli: bool } pub struct Receiver { + codec: Codec, next: Option, pending: Vec, timestamp: Option, - au: Vec, fragment: Option, damaged: bool, gap_since:Option, last_packet:Instant, + au: Vec, fragment: Option, damaged: bool, gap_since:Option, last_packet:Instant, } impl Default for Receiver { - fn default() -> Self { Self { next: None, pending: Vec::new(), timestamp: None, au: Vec::new(), fragment: None, damaged: false, gap_since:None,last_packet:Instant::now() } } + fn default() -> Self { Self { codec: Codec::H264, next: None, pending: Vec::new(), timestamp: None, au: Vec::new(), fragment: None, damaged: false, gap_since:None,last_packet:Instant::now() } } } impl Receiver { + pub fn new(codec:Codec)->Self {Self{codec,..Self::default()}} + pub fn codec(&self)->Codec {self.codec} pub fn push(&mut self, p: Packet) -> Output { if p.payload.len()>65536 {return Output{units:Vec::new(),pli:true}} let next = *self.next.get_or_insert(p.seq); @@ -67,6 +80,7 @@ impl Receiver { self.au.extend_from_slice(bytes); Ok(()) } fn nal(&mut self, p: &[u8]) -> Result<(), ()> { + if self.codec==Codec::H265 {return self.hevc(p)} let h = *p.first().ok_or(())?; if h & 0x80 != 0 { return Err(()) } match h & 31 { @@ -86,14 +100,47 @@ impl Receiver { let header = (h & 0xe0) | kind; if start { if self.fragment.is_some() { return Err(()) } - self.append(&[0,0,0,1,header])?; self.fragment = Some(header); - } else if self.fragment != Some(header) { return Err(()) } + self.append(&[0,0,0,1,header])?; self.fragment = Some(u16::from(header)); + } else if self.fragment != Some(u16::from(header)) { return Err(()) } self.append(&p[2..])?; if end { self.fragment = None; } Ok(()) } _ => Err(()), } } + fn hevc_header(p:&[u8])->Result { + if p.len()<2 || p[0]&0x80!=0 || p[1]&7==0 {return Err(())} + Ok(u16::from_be_bytes([p[0],p[1]])) + } + fn hevc(&mut self,p:&[u8])->Result<(),()> { + Self::hevc_header(p)?; + match (p[0]>>1)&63 { + 0..=47 if self.fragment.is_none()=>{self.append(&[0,0,0,1])?;self.append(p)}, + 48 if self.fragment.is_none()=>{ + let mut rest=&p[2..];if rest.is_empty(){return Err(())} + while !rest.is_empty() { + if rest.len()<2 {return Err(())} + let n=u16::from_be_bytes([rest[0],rest[1]]) as usize;rest=&rest[2..]; + if n<2 || rest.len()>1)&63>=48 {return Err(())} + self.append(&[0,0,0,1])?;self.append(&rest[..n])?;rest=&rest[n..]; + }Ok(()) + }, + 49 if p.len()>3=>{ + let f=p[2];let kind=f&63;let start=f&128!=0;let end=f&64!=0; + if kind>=48 || (start&&end){return Err(())} + // RFC 7798 4.4.3: preserve LayerId and TID, replacing only FU's NAL type. + let header=[(p[0]&0x81)|(kind<<1),p[1]];let key=u16::from_be_bytes(header); + if start { + if self.fragment.is_some(){return Err(())} + self.append(&[0,0,0,1])?;self.append(&header)?;self.fragment=Some(key); + }else if self.fragment!=Some(key){return Err(())} + self.append(&p[3..])?;if end{self.fragment=None;}Ok(()) + }, + _=>Err(()), + } + } } /// Drop the oldest item on overflow; never block the RTP reader on a decoder/UI. pub struct Latest(Arc>>, usize); @@ -136,3 +183,46 @@ mod tests { let q=Latest::new(2);assert!(!q.push(1));q.push(2);assert!(q.push(3));assert_eq!(q.pop(),Some(2));assert_eq!(q.pop(),Some(3)); } } + +#[cfg(test)] mod hevc_tests { + use super::*; + fn packet(seq:u16,ts:u32,marker:bool,payload:Vec)->Packet {Packet{seq,timestamp:ts,marker,payload}} + fn sample()->Vec{ + let mut au=vec![0,0,0,1,64,1,9,0,0,0,1,66,1,8,0,0,0,1,68,1,7,0,0,0,1,0x27,0x2b]; + au.extend((0..2501).map(|i|(i%251+1)as u8));au + } + #[test] fn host_payloader_round_trip_single_fu_and_reorder(){ + for mtu in [4,5,7,1190,9000] { + let au=sample();let payloads=crate::hevc::payload(mtu,&au).unwrap();let n=payloads.len(); + let mut packets:Vec<_>=payloads.into_iter().enumerate().map(|(i,b)|packet(65534u16.wrapping_add(i as u16),10,i+1==n,b.to_vec())).collect(); + // Start at the first sequence, then reorder adjacent packets across wraparound. + for i in (1..n.saturating_sub(1)).step_by(2){packets.swap(i,i+1);} + let mut r=Receiver::new(Codec::H265);let mut units=Vec::new(); + for p in packets {let o=r.push(p);assert!(!o.pli);units.extend(o.units);} + assert_eq!(units,vec![au]); + } + } + #[test] fn aggregation_and_malformed_packets(){ + let ap=vec![96,1,0,3,64,1,9,0,3,38,1,7]; + let mut r=Receiver::new(Codec::H265); + assert_eq!(r.push(packet(0,0,true,ap)).units,vec![vec![0,0,0,1,64,1,9,0,0,0,1,38,1,7]]); + for payload in [vec![],vec![64],vec![128,1],vec![64,0],vec![96,1],vec![96,1,0,4,38,1,7], + vec![96,1,0,2,96,1],vec![98,1,0xc1,3],vec![98,1,0x41,3],vec![98,1,0xb0,3],vec![100,1,7]] { + let o=Receiver::new(Codec::H265).push(packet(1,1,true,payload));assert!(o.pli);assert!(o.units.is_empty()); + } + } + #[test] fn loss_duplicates_incomplete_tail_and_header_mismatch(){ + let payloads=crate::hevc::payload(1190,&sample()).unwrap(); + let mut r=Receiver::new(Codec::H265); + // Parameter sets followed by a fragmented IDR; lose its middle fragment. + for (i,p) in payloads.iter().enumerate().take(4){r.push(packet(i as u16,1,false,p.to_vec()));} + let tail=packet(5,1,true,payloads[5].to_vec()); + assert!(r.push(tail.clone()).units.is_empty());assert!(r.push(tail).units.is_empty()); + let o=r.expire_at(Instant::now()+Duration::from_millis(50));assert!(o.pli);assert!(o.units.is_empty()); + assert_eq!(r.push(packet(6,2,true,vec![38,1,9])).units.len(),1); + r.push(packet(7,3,false,vec![98,1,0x93,7])); + assert!(r.expire_at(Instant::now()+Duration::from_millis(201)).pli); + let mut r=Receiver::new(Codec::H265);r.push(packet(0,0,false,vec![98,1,0x93,7])); + let o=r.push(packet(1,0,true,vec![98,2,0x53,8]));assert!(o.pli);assert!(o.units.is_empty()); + } +} diff --git a/src/viewer/mod.rs b/src/viewer/mod.rs index c9367f8..e559402 100644 --- a/src/viewer/mod.rs +++ b/src/viewer/mod.rs @@ -12,7 +12,7 @@ pub mod audio; pub mod input; pub mod desktop; #[cfg(feature="viewer-ffmpeg")] pub mod decoder; -#[cfg(feature="viewer-sdl")] pub mod decoder_cli; +#[cfg(any(test,feature="viewer-sdl"))] pub mod decoder_cli; #[cfg(uu_gtk)] pub mod ui; #[cfg(uu_gtk)] pub mod tools_ui; #[cfg(feature="viewer-sdl")] pub mod sdl; diff --git a/src/viewer/rtc.rs b/src/viewer/rtc.rs index a3b2938..ec750a7 100644 --- a/src/viewer/rtc.rs +++ b/src/viewer/rtc.rs @@ -1,6 +1,6 @@ use super::wire::{self, Shape, CHANNELS}; use serde_json::{json, Value}; -use std::sync::{Arc,atomic::{AtomicBool,AtomicU64,Ordering}}; +use std::sync::{Arc,atomic::{AtomicBool,AtomicU32,AtomicU64,Ordering}}; use tokio::sync::mpsc; use webrtc::{api::{APIBuilder,media_engine::MediaEngine,setting_engine::SettingEngine}, peer_connection::{RTCPeerConnection,configuration::RTCConfiguration,sdp::session_description::RTCSessionDescription,policy::ice_transport_policy::RTCIceTransportPolicy}, ice_transport::{ice_server::RTCIceServer,ice_candidate::RTCIceCandidateInit}, data_channel::{RTCDataChannel,data_channel_init::RTCDataChannelInit}, rtp_transceiver::{rtp_codec::RTPCodecType,rtp_transceiver_direction::RTCRtpTransceiverDirection,RTCRtpTransceiverInit}}; @@ -34,6 +34,8 @@ pub struct Observed { pub control_messages:AtomicU64, pub other_messages:AtomicU64, pub media_packets:AtomicU64, + pub encoder_codecs:AtomicU32, + pub chroma:AtomicU32, } pub struct Peer { clipboard:super::clipboard::Bridge, @@ -49,17 +51,21 @@ pub struct Peer { ice_id:String, remote_client:String, } +fn media_engine(shape:&Shape)->Result{ + use webrtc::rtp_transceiver::rtp_codec::{RTCRtpCodecParameters,RTCRtpCodecCapability}; + let mut engine=MediaEngine::default(); + let mut codecs=vec![(RTPCodecType::Video,"video/H264",90000,0,"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",102), + (RTPCodecType::Audio,"audio/opus",48000,2,"minptime=10;useinbandfec=1",111)]; + // RFC 7798 single-layer non-interleaved mode: no sprop-max-don-diff, hence no DONL. + if shape.decoders.iter().any(|d|d.codec==2){codecs.insert(0,(RTPCodecType::Video,"video/H265",90000,0,"",104));} + for (kind,mime,rate,channels,fmtp,payload) in codecs { + engine.register_codec(RTCRtpCodecParameters{capability:RTCRtpCodecCapability{mime_type:mime.into(),clock_rate:rate,channels,sdp_fmtp_line:fmtp.into(),..Default::default()},payload_type:payload,..Default::default()},kind).map_err(|_|"Codec initialization failed")?; + }Ok(engine) +} impl Peer { pub async fn new(config:RTCConfiguration, shape:&Shape, app_id:&str, ice_id:&str, remote_client:&str, outbound:mpsc::Sender, settings:SettingEngine)->Result { shape.validate_session()?; - let mut engine=MediaEngine::default(); - use webrtc::rtp_transceiver::rtp_codec::{RTCRtpCodecParameters,RTCRtpCodecCapability}; - for (kind,mime,rate,channels,fmtp,payload) in [ - (RTPCodecType::Video,"video/H264",90000,0,"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",102), - (RTPCodecType::Audio,"audio/opus",48000,2,"minptime=10;useinbandfec=1",111), - ] { - engine.register_codec(RTCRtpCodecParameters{capability:RTCRtpCodecCapability{mime_type:mime.into(),clock_rate:rate,channels,sdp_fmtp_line:fmtp.into(),..Default::default()},payload_type:payload,..Default::default()},kind).map_err(|_|"Codec initialization failed")?; - } + let mut engine=media_engine(shape)?; // UU RTP metadata the official Mac viewer offers (src/rtp_metadata.rs on the host side). for uri in ["http://www.webrtc.org/experiments/rtp-hdrext/video-capture-index","http://www.webrtc.org/experiments/rtp-hdrext/video-frame-is-new-frame"] { engine.register_header_extension(webrtc::rtp_transceiver::rtp_codec::RTCRtpHeaderExtensionCapability{uri:uri.into()},RTPCodecType::Video,None).map_err(|_|"RTP header extension registration failed")?; @@ -104,9 +110,9 @@ impl Peer { let video=track.kind()==RTPCodecType::Video; log::info!("Viewer track: kind={} mime={} payload_type={}",track.kind(),track.codec().capability.mime_type,track.payload_type()); let opus=track.kind()==RTPCodecType::Audio && track.codec().capability.mime_type.eq_ignore_ascii_case("audio/opus"); - if video && !track.codec().capability.mime_type.eq_ignore_ascii_case("video/H264") {return} + if video && super::media::Codec::from_mime(&track.codec().capability.mime_type).is_none() {return} if video {desktop.ssrc.store(track.ssrc(),Ordering::Release);} - let mut receiver=super::media::Receiver::default(); + let mut receiver=super::media::Receiver::new(super::media::Codec::from_mime(&track.codec().capability.mime_type).unwrap_or_default()); let mut tick=tokio::time::interval(std::time::Duration::from_millis(40)); let mut last_pli=std::time::Instant::now()-std::time::Duration::from_secs(1); loop { @@ -115,6 +121,12 @@ impl Peer { Ok((p,_))=>{ counters.media_packets.fetch_add(1,Ordering::Relaxed); if !video {if opus {audio.push(p.header.sequence_number,&p.payload);}continue} + // TrackRemote updates codec from the negotiated payload type on each read. + let Some(codec)=super::media::Codec::from_mime(&track.codec().capability.mime_type) else{continue}; + if receiver.codec()!=codec { + receiver=super::media::Receiver::new(codec); + desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release); + } receiver.push(super::media::Packet{seq:p.header.sequence_number,timestamp:p.header.timestamp,marker:p.header.marker,payload:p.payload.to_vec()}) }, Err(_)=>break, @@ -122,7 +134,7 @@ impl Peer { _=tick.tick()=>{if !video {continue} receiver.expire()}, }; if output.pli {desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release);} - for au in output.units {if desktop.units.push(au) {desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release);}} + for au in output.units {if desktop.units.push(super::media::AccessUnit{codec:receiver.codec(),data:au}) {desktop.reset.store(true,Ordering::Release);desktop.pli.store(true,Ordering::Release);}} if desktop.pli.load(Ordering::Acquire) && last_pli.elapsed()>=std::time::Duration::from_millis(250) { desktop.pli.swap(false,Ordering::AcqRel); if let Some(pc)=weak_pc.upgrade() { @@ -160,9 +172,13 @@ impl Peer { for (index,channel) in channels.iter().enumerate() { if index==0 { let weak=Arc::downgrade(&channel);let state=observed.clone(); + let chroma444=shape.capture_type==1 && std::env::var("UU_VIEWER_CHROMA").is_ok_and(|v|v=="444"); channel.on_open(Box::new(move ||Box::pin(async move { state.control_open.store(true,Ordering::Release); - if let Some(channel)=weak.upgrade(){let _=channel.send(&bytes::Bytes::from(wire::echo_request_with_clipboard(1,clipboard_enabled))).await;} + if let Some(channel)=weak.upgrade(){ + let _=channel.send(&bytes::Bytes::from(wire::echo_request_with_clipboard(1,clipboard_enabled))).await; + if chroma444 {let _=channel.send(&bytes::Bytes::from(wire::request_chroma444())).await;} + } }))); } let closed=tools.clone();let state=observed.clone(); @@ -174,6 +190,11 @@ impl Peer { let control_for_echo=if index==0 {Arc::downgrade(&channel)} else {std::sync::Weak::new()}; channel.on_message(Box::new(move |message| {let state=state.clone();let inbox=inbox.clone();let control_for_echo=control_for_echo.clone();let clipboard_tx=clipboard_tx.clone();Box::pin(async move { if index==0 { + let (codecs,chroma)=wire::media_control(&message.data); + if let Some(codecs)=codecs {state.encoder_codecs.store(codecs,Ordering::Release);} + if let Some(chroma)=chroma { + if state.chroma.swap(chroma,Ordering::AcqRel)!=chroma {log::info!("Viewer capture sync: chroma={chroma}");} + } let count=state.control_messages.fetch_add(1,Ordering::Relaxed); if wire::echo_matches(&message.data,1) {state.clipboard_supported.store(wire::echo_clipboard_supported(&message.data),Ordering::Release);state.echo_received.store(true,Ordering::Release);} if let Some(request)=wire::echo_request_seq(&message.data) { @@ -313,3 +334,17 @@ pub fn with_max_message_size(sdp:&str)->String{ assert_eq!(with_max_message_size("m=application 9\r\na=sctp-port:5000\r\na=mid:0\r\n"),"m=application 9\r\na=sctp-port:5000\r\na=max-message-size:262144\r\na=mid:0\r\n"); assert_eq!(with_max_message_size("a=max-message-size:1\r\n"),"a=max-message-size:1\r\n"); } + +#[cfg(test)] #[tokio::test] async fn offers_both_codecs_or_h264_only(){ + for force in [false,true] { + let shape:Shape=serde_json::from_str(include_str!("../../tests/fixtures/viewer/session-shape.json")).unwrap(); + let shape=shape.video_preferences(force); + let api=APIBuilder::new().with_media_engine(media_engine(&shape).unwrap()).build(); + let pc=api.new_peer_connection(RTCConfiguration::default()).await.unwrap(); + pc.add_transceiver_from_kind(RTPCodecType::Video,Some(RTCRtpTransceiverInit{direction:RTCRtpTransceiverDirection::Recvonly,send_encodings:vec![]})).await.unwrap(); + let offer=pc.create_offer(None).await.unwrap(); + assert!(offer.sdp.contains("a=rtpmap:102 H264/90000")); + assert_eq!(offer.sdp.contains("a=rtpmap:104 H265/90000"),!force); + assert!(!offer.sdp.contains("sprop-max-don-diff"));pc.close().await.unwrap(); + } +} diff --git a/src/viewer/sdl.rs b/src/viewer/sdl.rs index 3facc05..26a604c 100644 --- a/src/viewer/sdl.rs +++ b/src/viewer/sdl.rs @@ -106,7 +106,8 @@ pub async fn run(session:&mut Session,window:&mut Window,decoder:&mut dyn super: // At most one AU per iteration keeps local event polling responsive. if let Some(au)=session.peer.desktop.units.pop(){ if session.peer.desktop.reset.swap(false,std::sync::atomic::Ordering::AcqRel){decoder.reset();} - super::desktop::display(decoder,window,&au,&session.peer.desktop.pli)?;} + decoder.set_codec(au.codec)?; + super::desktop::display(decoder,window,&au.data,au.codec,&session.peer.desktop.pli)?;} } }} }.await; diff --git a/src/viewer/session.rs b/src/viewer/session.rs index 2a6c73e..32522e4 100644 --- a/src/viewer/session.rs +++ b/src/viewer/session.rs @@ -8,6 +8,8 @@ pub struct Options {pub timeout:Duration,pub legacy_prefix:bool,pub ready_policy pub struct Session {pub peer:Peer,signal:Signal,candidates:mpsc::Receiver,policy:ReadyPolicy} impl Session { pub async fn open(room:&JoinData,shape:&Shape,device:&str,client:&str,options:Options,settings:SettingEngine)->Result { + shape.validate_session()?; + let shape=shape.video_preferences(std::env::var("UU_VIEWER_CODEC").is_ok_and(|v|v.eq_ignore_ascii_case("h264"))); shape.validate_session()?; let app=wire::uuid();let ice=hex::encode(rand::random::<[u8;16]>()); let timeout=if room.ws_connect_timeout_ms>0 {Duration::from_millis(room.ws_connect_timeout_ms as u64)}else{options.timeout}; @@ -31,7 +33,7 @@ impl Session { let remote=result.get("client_id").and_then(Value::as_str).filter(|s|!s.is_empty()).ok_or("Control acknowledgement missing peer route")?; let config=rtc::configuration(&result)?; let(tx,rx)=mpsc::channel(256); - let peer=Peer::new(config,shape,&app,&ice,remote,tx,settings).await?; + let peer=Peer::new(config,&shape,&app,&ice,remote,tx,settings).await?; // The official host drops an offer whose ice_id is not yet in its connection table // (libstreamer 0xc3080); give it time to process be-controlled first. let delay=std::env::var("UU_VIEWER_OFFER_DELAY_MS").ok().and_then(|v|v.parse().ok()).unwrap_or(0u64).min(10_000); diff --git a/src/viewer/tests.rs b/src/viewer/tests.rs index 752f80c..acb5928 100644 --- a/src/viewer/tests.rs +++ b/src/viewer/tests.rs @@ -130,7 +130,7 @@ async fn complete_session_uses_loopback_signaling_and_virtual_rtc_through_echo() Some("offer")=> { let offered=body["sdp"].as_str().unwrap(); assert!(offered.contains("H264/90000") && offered.contains("opus/48000")); - assert!(!offered.contains("H265") && !offered.contains("VP8")); + assert!(offered.contains("H265/90000") && !offered.contains("VP8")); assert_eq!(offered.matches("a=recvonly").count(),2); server_host.set_remote_description(RTCSessionDescription::offer(offered.into()).unwrap()).await.unwrap(); let answer=server_host.create_answer(None).await.unwrap();let sdp=answer.sdp.clone();let mut gathered=server_host.gathering_complete_promise().await;server_host.set_local_description(answer).await.unwrap(); @@ -169,7 +169,7 @@ async fn complete_session_uses_loopback_signaling_and_virtual_rtc_through_echo() let heartbeat=super::input::Input::heartbeat();session.peer.send_input(&heartbeat).await.unwrap();assert_eq!(input_rx.recv().await.unwrap(),heartbeat); let packet=|seq,ts,marker,payload:&[u8]|webrtc::rtp::packet::Packet{header:webrtc::rtp::header::Header{version:2,sequence_number:seq,timestamp:ts,marker,..Default::default()},payload:bytes::Bytes::copy_from_slice(payload)}; track.write_rtp(&packet(9,1,true,&[0x65,1])).await.unwrap(); - loop {if let Some(au)=session.peer.desktop.units.pop(){assert_eq!(au,vec![0,0,0,1,0x65,1]);break}tokio::time::sleep(Duration::from_millis(10)).await;} + loop {if let Some(au)=session.peer.desktop.units.pop(){assert_eq!(au.codec,super::media::Codec::H264);assert_eq!(au.data,vec![0,0,0,1,0x65,1]);break}tokio::time::sleep(Duration::from_millis(10)).await;} track.write_rtp(&packet(10,2,false,&[0x7c,0x85,1])).await.unwrap(); track.write_rtp(&packet(12,2,true,&[0x7c,0x45,3])).await.unwrap(); loop {let (packets,_)=sender.read_rtcp().await.unwrap();if packets.iter().any(|p|p.as_any().is::()){break}} @@ -220,7 +220,7 @@ async fn opus_and_video_tracks_are_received_independently_in_memory() { let video=Arc::new(TrackLocalStaticRTP::new(RTCRtpCodecCapability{mime_type:"video/H264".into(),clock_rate:90000,sdp_fmtp_line:"level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f".into(),..Default::default()},"video".into(),"test".into())); host.add_track(audio.clone() as Arc).await.unwrap();host.add_track(video.clone() as Arc).await.unwrap(); let(tx,_rx)=mpsc::channel(256); - let peer=Peer::new(RTCConfiguration::default(),&shape(),"app","ice","host",tx,viewer_settings).await.unwrap(); + let peer=Peer::new(RTCConfiguration::default(),&shape().video_preferences(false),"app","ice","host",tx,viewer_settings).await.unwrap(); let mut gathered=peer.pc.gathering_complete_promise().await;peer.offer().await.unwrap();gathered.recv().await; let offer=peer.pc.local_description().await.unwrap();assert!(offer.sdp.contains("m=audio") && offer.sdp.contains("opus/48000/2")); host.set_remote_description(offer).await.unwrap();let answer=host.create_answer(None).await.unwrap(); @@ -233,7 +233,7 @@ async fn opus_and_video_tracks_are_received_independently_in_memory() { audio.write_rtp(&packet(&[0xf8,0xff,0xfe])).await.unwrap();video.write_rtp(&packet(&[0x65,1])).await.unwrap(); tokio::time::sleep(Duration::from_millis(10)).await; if let Some(data)=peer.audio.pop(){assert_eq!(data,vec![0xf8,0xff,0xfe]);heard=true;} - if let Some(data)=peer.desktop.units.pop(){assert_eq!(data,vec![0,0,0,1,0x65,1]);seen=true;} + if let Some(data)=peer.desktop.units.pop(){assert_eq!(data.codec,super::media::Codec::H264);assert_eq!(data.data,vec![0,0,0,1,0x65,1]);seen=true;} if heard && seen {break} } assert!(heard && seen);peer.close().await;host.close().await.unwrap();router.lock().await.stop().await.unwrap(); @@ -287,3 +287,55 @@ async fn desktop_file_window_reattaches_without_replacing_peer(){ peer.close().await;host.close().await.unwrap();router.lock().await.stop().await.unwrap(); }).await.expect("desktop file reuse timed out"); } + +#[tokio::test] +async fn hevc_track_uses_negotiated_payload_type_in_memory(){ + tokio::time::timeout(Duration::from_secs(10),async { + use super::media::Codec; + use webrtc::{track::track_local::{track_local_static_rtp::TrackLocalStaticRTP,TrackLocal,TrackLocalWriter}, + rtp_transceiver::rtp_codec::{RTCRtpCodecCapability,RTCRtpCodecParameters,RTPCodecType}}; + let(viewer_settings,host_settings,router)=virtual_network().await; + // The host uses a different local PT; the answer binds its track to the viewer offer. + let mut engine=MediaEngine::default(); + let capability=RTCRtpCodecCapability{mime_type:"video/H265".into(),clock_rate:90000,..Default::default()}; + engine.register_codec(RTCRtpCodecParameters{capability:capability.clone(),payload_type:126,..Default::default()},RTPCodecType::Video).unwrap(); + let host=Arc::new(APIBuilder::new().with_media_engine(engine).with_setting_engine(host_settings).build().new_peer_connection(RTCConfiguration::default()).await.unwrap()); + let video=Arc::new(TrackLocalStaticRTP::new(capability,"video".into(),"test".into())); + host.add_track(video.clone() as Arc).await.unwrap(); + host.on_data_channel(Box::new(|channel|Box::pin(async move { + if channel.label()=="CONTROL_DATA_CHANNEL" { + let weak=Arc::downgrade(&channel); + channel.on_open(Box::new(move||Box::pin(async move { + if let Some(channel)=weak.upgrade(){ + let mut caps=Vec::new();for codec in [1,2]{let mut cap=Vec::new();wire::int(3,codec,&mut cap);wire::bytes(1,&cap,&mut caps);} + channel.send(&bytes::Bytes::from(wire::envelope(2,18,&caps))).await.unwrap(); + // Model a host that declines 4:4:4 and encodes 4:2:0. + channel.send(&bytes::Bytes::from(wire::envelope(3,25,&[24,1]))).await.unwrap(); + } + }))); + } + }))); + let(tx,_rx)=mpsc::channel(256); + let peer=Peer::new(RTCConfiguration::default(),&shape().video_preferences(false),"app","ice","host",tx,viewer_settings).await.unwrap(); + let mut gathered=peer.pc.gathering_complete_promise().await;peer.offer().await.unwrap();gathered.recv().await; + host.set_remote_description(peer.pc.local_description().await.unwrap()).await.unwrap(); + let answer=host.create_answer(None).await.unwrap(); + assert!(answer.sdp.contains("H265/90000"));assert!(!answer.sdp.contains("H264/90000")); + let mut gathered=host.gathering_complete_promise().await;host.set_local_description(answer).await.unwrap();gathered.recv().await; + peer.pc.set_remote_description(host.local_description().await.unwrap()).await.unwrap(); + while peer.pc.connection_state()!=webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Connected {tokio::time::sleep(Duration::from_millis(10)).await;} + let mut au=vec![0,0,0,1,64,1,9,0,0,0,1,38,1];au.extend(vec![7;200]); + let payloads=crate::hevc::payload(80,&au).unwrap();let mut seq=0;let mut seen=false; + for timestamp in 0..100 { + for (i,payload) in payloads.iter().enumerate(){ + video.write_rtp(&webrtc::rtp::packet::Packet{header:webrtc::rtp::header::Header{version:2,sequence_number:seq,timestamp:timestamp*3000,marker:i+1==payloads.len(),..Default::default()},payload:payload.clone()}).await.unwrap();seq=seq.wrapping_add(1); + } + tokio::time::sleep(Duration::from_millis(10)).await; + if let Some(unit)=peer.desktop.units.pop(){assert_eq!(unit.codec,Codec::H265);assert_eq!(unit.data,au);seen=true;} + if seen && peer.observed.chroma.load(Ordering::Acquire)==1 {break} + } + assert!(seen);assert_eq!(peer.observed.encoder_codecs.load(Ordering::Acquire),3); + assert_eq!(peer.observed.chroma.load(Ordering::Acquire),1); + peer.close().await;host.close().await.unwrap();router.lock().await.stop().await.unwrap(); + }).await.expect("in-memory H265 timed out"); +} diff --git a/src/viewer/wire.rs b/src/viewer/wire.rs index d38a968..2ae3e40 100644 --- a/src/viewer/wire.rs +++ b/src/viewer/wire.rs @@ -52,14 +52,29 @@ impl Shape { } pub fn validate_session(&self)->Result<(),String> { self.validate()?; - // H264/420 receive only. streamer_data video_codec_capability is sent by the official Mac - // viewer (8 entries observed live), so H264/420 entries are allowed there too. - if self.decoders.iter().any(|d|d.codec!=1 || d.chroma!=1) || self.codecs.iter().any(|c|c.video_codec!=1 || c.chroma_sampling!=1) - || (self.capture_type==1 && (self.chroma!=1 || self.decoders.is_empty())) { - return Err("Sessions require H264/420 decoder and codec capabilities".into()); + if self.codecs.iter().any(|c|![1,2].contains(&c.video_codec) || ![1,3].contains(&c.chroma_sampling)) + || (self.capture_type==1 && self.decoders.is_empty()) { + return Err("Sessions require supported decoder and codec capabilities".into()); } Ok(()) } + /// Advertise software decoding of both 8-bit chroma formats. Keep caller size/fps limits. + /// The explicit chroma request is sent on CONTROL, so initial capture stays 4:2:0. + pub fn video_preferences(&self,h264_only:bool)->Self { + let mut shape=self.clone(); + if shape.capture_type==1 { + shape.chroma=1; + shape.decoders.clear(); + for codec in if h264_only {&[1][..]}else{&[2,1][..]} { + let base=self.decoders.iter().find(|d|d.codec==*codec).or_else(||self.decoders.first()); + if let Some(base)=base {for chroma in [1,3] { + let mut d=base.clone();d.codec=*codec;d.chroma=chroma;shape.decoders.push(d); + }} + } + } + if h264_only {shape.decoders.retain(|d|d.codec==1);shape.codecs.retain(|c|c.video_codec==1);} + shape + } pub fn connect_options(&self, device:&str) -> Result,String> { self.validate()?; let mut out=Vec::new(); int(1,self.capture_type.into(),&mut out); @@ -181,3 +196,54 @@ mod tests { assert!(!echo_matches(&echo_request(7),7)); assert!(!echo_matches(&[255],7)); } } + +/// CaptureSettingRequest field 7 = ChromaFormat 3, wrapped in RPC 21/case 2. +pub fn request_chroma444()->Vec{ + let mut header=Vec::new();int(1,2_000_000,&mut header); + let mut setting=Vec::new();int(7,3,&mut setting); + let mut rpc=Vec::new();bytes(1,&header,&mut rpc);bytes(2,&setting,&mut rpc); + envelope(2_000_000,21,&rpc) +} +/// Message 18 is an encoder capability LIST, not the chosen codec; RTP's negotiated PT is authoritative. +/// Message 25 reports actual encoded chroma, which may fall back to 4:2:0. +pub fn media_control(data:&[u8])->(Option,Option){ + let mut codecs=None;let mut chroma=None; + if let Ok(fields)=decode_wire(data){for field in fields { + if let WireValue::Bytes(body)=field.value {match field.number { + 18=>{if let Ok(caps)=decode_wire(body){let mut mask=0;for cap in caps {if let (1,WireValue::Bytes(cap))=(cap.number,cap.value){ + match super::tools::uint(cap,3){1=>mask|=1,2=>mask|=2,_=>{}} + }}codecs=Some(mask);}}, + 25=>{let value=super::tools::uint(body,3);if [1,3].contains(&value){chroma=Some(value as u32);}}, + _=>{}, + }} + }}(codecs,chroma) +} + +#[cfg(test)] mod video_tests { + use super::*; + #[test] fn decoder_caps_cover_both_codecs_and_chroma_and_force_h264(){ + let original:Shape=serde_json::from_str(include_str!("../../tests/fixtures/viewer/session-shape.json")).unwrap(); + for force in [false,true] { + let s=original.video_preferences(force);s.validate_session().unwrap();assert_eq!(s.chroma,1); + let options=s.connect_options("synthetic-device").unwrap(); + let mut caps=Vec::new(); + for f in decode_wire(&options).unwrap(){if let (4,WireValue::Bytes(cap))=(f.number,f.value){ + assert_eq!(super::super::tools::uint(cap,1),30); + assert_eq!(super::super::tools::uint(cap,3),1280);assert_eq!(super::super::tools::uint(cap,4),720); + caps.push((super::super::tools::uint(cap,2),super::super::tools::uint(cap,5))); + }} + assert_eq!(caps,if force{vec![(1,1),(1,3)]}else{vec![(2,1),(2,3),(1,1),(1,3)]}); + assert!(s.video_preferences(force).connect_options("synthetic-device").unwrap()==options); + } + } + #[test] fn chroma_request_and_actual_sync_are_independent_of_encoder_list(){ + let request=request_chroma444();let rpc=super::super::tools::blob(&request,21).unwrap(); + let setting=super::super::tools::blob(rpc,2).unwrap();assert_eq!(super::super::tools::uint(setting,7),3); + assert_eq!(decode_wire(setting).unwrap().len(),1); + let mut caps=Vec::new();for codec in [1,2] {let mut cap=Vec::new();int(3,codec,&mut cap);bytes(1,&cap,&mut caps);} + let mut msg=envelope(1,18,&caps);let mut sync=Vec::new();int(3,1,&mut sync);bytes(25,&sync,&mut msg); + assert_eq!(media_control(&msg),(Some(3),Some(1))); + assert_eq!(media_control(&envelope(2,25,&[24,3])),(None,Some(3))); + assert_eq!(media_control(&[255]),(None,None)); + } +} diff --git a/src/viewer_main.rs b/src/viewer_main.rs index af11ebe..ff0721d 100644 --- a/src/viewer_main.rs +++ b/src/viewer_main.rs @@ -4,6 +4,7 @@ #[path = "config.rs"] mod config; #[path = "protocol.rs"] mod protocol; mod viewer; +#[cfg(test)] #[path="hevc.rs"] mod hevc; use viewer::clipboard; #[path="clipboard_protocol.rs"] mod clipboard_protocol; #[path="clipboard_rich.rs"] mod clipboard_rich; @@ -12,7 +13,7 @@ use viewer::clipboard; #[cfg(test)] #[path="initialization.rs"] mod initialization; use viewer::{api::Api,rtc::ReadyPolicy,session::{Options,Session},wire::Shape}; use std::time::Duration; -const HELP:&str="Experimental native viewer.\n uuremote-viewer ui (GTK window: sign-in, devices, sessions) Desktop UI requires viewer-sdl (decodes with the ffmpeg CLI, or FFmpeg 6.x libraries with viewer-ffmpeg).\n uuremote-viewer power DEVICE reboot|shutdown\n uuremote-viewer terminal DEVICE [SESSION]\n uuremote-viewer files DEVICE ls PATH | get REMOTE LOCAL | put LOCAL REMOTE\n uuremote-viewer portmap DEVICE LOCAL_PORT HOST PORT\n uuremote-viewer files-ui|portmap-ui|terminal-ui DEVICE NAME (GTK tool windows)\n uuremote-viewer login [--no-gui]\n uuremote-viewer devices METHOD ARRAY_POINTER ID_POINTER NAME_POINTER\n uuremote-viewer assist CONNECT_ID (验证码从 stdin 读取一行)\n uuremote-viewer connect DEVICE true|false SHAPE_JSON echo|manual raw|legacy [--view-only]\nAudio: UU_VIEWER_AUDIO=0 disables playback. UU_VIEWER_VIEW_ONLY=1 disables desktop input.\nDevice-list method/JSON pointers and compatibility shape require verified operator configuration.\nQR login reuses the sibling uuremote-server login command. Never run login in a captured log.\nEcho readiness is transport observation only; no sendVideoTrack RPC is emitted."; +const HELP:&str="Experimental native viewer.\n uuremote-viewer ui (GTK window: sign-in, devices, sessions) Desktop UI requires viewer-sdl (decodes with the ffmpeg CLI, or FFmpeg 6.x libraries with viewer-ffmpeg).\n uuremote-viewer power DEVICE reboot|shutdown\n uuremote-viewer terminal DEVICE [SESSION]\n uuremote-viewer files DEVICE ls PATH | get REMOTE LOCAL | put LOCAL REMOTE\n uuremote-viewer portmap DEVICE LOCAL_PORT HOST PORT\n uuremote-viewer files-ui|portmap-ui|terminal-ui DEVICE NAME (GTK tool windows)\n uuremote-viewer login [--no-gui]\n uuremote-viewer devices METHOD ARRAY_POINTER ID_POINTER NAME_POINTER\n uuremote-viewer assist CONNECT_ID (验证码从 stdin 读取一行)\n uuremote-viewer connect DEVICE true|false SHAPE_JSON echo|manual raw|legacy [--view-only]\nVideo: UU_VIEWER_CODEC=h264 forces H.264; UU_VIEWER_CHROMA=444 requests true color (default 4:2:0).\nAudio: UU_VIEWER_AUDIO=0 disables playback. UU_VIEWER_VIEW_ONLY=1 disables desktop input.\nDevice-list method/JSON pointers and compatibility shape require verified operator configuration.\nQR login reuses the sibling uuremote-server login command. Never run login in a captured log.\nEcho readiness is transport observation only; no sendVideoTrack RPC is emitted."; const DEFAULT_SHAPE:&str=r#"{"capture_type":1,"platform":3,"client_version":"4.42.0","width":1920,"height":1080,"fps":60,"chroma":1, "decoders":[{"fps":60,"codec":1,"width":3840,"height":2160,"chroma":1}],"display_modes":[[1920,1080,60]], "displays":[{"id":1,"fps":60,"type":0,"hdr":0}], diff --git a/tests/fixtures/viewer/README.md b/tests/fixtures/viewer/README.md new file mode 100644 index 0000000..3903862 --- /dev/null +++ b/tests/fixtures/viewer/README.md @@ -0,0 +1,15 @@ +The `red-{420,444}.{h264,hevc}` fixtures contain six synthetic 32x32 red IDRs, +not desktop captures. Their decoded pixel formats are yuv420p and yuv444p. +They let the CLI decoder test run without requiring encoders at test time. + +Generated with FFmpeg (substitute `420` / `444` for `CHROMA`): + +```sh +ffmpeg -f lavfi -i color=c=red:s=32x32:r=10 -frames:v 6 -pix_fmt yuvCHROMAp \ + -c:v libx264 -threads 1 -preset ultrafast -tune zerolatency -g 1 \ + -f h264 red-CHROMA.h264 +ffmpeg -f lavfi -i color=c=red:s=32x32:r=10 -frames:v 6 -pix_fmt yuvCHROMAp \ + -c:v libx265 -threads 1 -preset ultrafast -tune zerolatency -g 1 \ + -x265-params pools=none:frame-threads=1:log-level=none:info=0 \ + -f hevc red-CHROMA.hevc +``` diff --git a/tests/fixtures/viewer/red-420.h264 b/tests/fixtures/viewer/red-420.h264 new file mode 100644 index 0000000..b72d007 Binary files /dev/null and b/tests/fixtures/viewer/red-420.h264 differ diff --git a/tests/fixtures/viewer/red-420.hevc b/tests/fixtures/viewer/red-420.hevc new file mode 100644 index 0000000..e5e647f Binary files /dev/null and b/tests/fixtures/viewer/red-420.hevc differ diff --git a/tests/fixtures/viewer/red-444.h264 b/tests/fixtures/viewer/red-444.h264 new file mode 100644 index 0000000..4f8fbc9 Binary files /dev/null and b/tests/fixtures/viewer/red-444.h264 differ diff --git a/tests/fixtures/viewer/red-444.hevc b/tests/fixtures/viewer/red-444.hevc new file mode 100644 index 0000000..7db8279 Binary files /dev/null and b/tests/fixtures/viewer/red-444.hevc differ