mirror of
https://github.com/VallierDev/uu-remote-linux.git
synced 2026-11-16 16:50:41 +01:00
Encode desktop video in-process instead of an ffmpeg child
Capture (x11grab) and H.264 encoding (h264_nvenc / libx264) now run in this process through ffmpeg-next, with the same codec options as the old ffmpeg CLI. Each encoder packet is one access unit, fed to the existing DependencyGate. A PLI or FIR from the viewer forces the next frame to be an IDR. Wayland and portal capture still use the child-process path. Set UU_VIDEO_PIPELINE=process to fall back to the old ffmpeg child. desktop-video-probe accepts UU_PROBE_FRAMES to lengthen the sample. Measured with desktop-video-probe on ros2go-gpu (RTX 4090 D, Xvfb 1920x1080 @60fps, 600 frames, mostly static desktop). CPU is the sum of the probe process and its children, sampled from /proc once per second. Probe only: no WebRTC send, no audio. encoder pipeline CPU (%) wall (s) bitrate (kbps) nvenc process ~138 10.17 5356 nvenc in-process ~43 10.34 5176 libx264 process ~135 10.09 5073 libx264 in-process ~72 10.08 4986 All four outputs decode without errors and start with an IDR. Not covered: audio still runs its own ffmpeg child (src/audio.rs), PLI to IDR has not been checked against a live viewer, and the CI jobs need the libav*-dev and libclang-dev packages (added to ci.yml, release.yml, AGENTS.md and the READMEs).
This commit is contained in:
1 parent
078ca026a5
commit
bf0aa6879a
11 files changed
+363
-22
No files matched your search
@@ -0,0 +1,155 @@
|
||||
//! In-process X11 capture and H.264 encoding through libavdevice/libavcodec.
|
||||
//! It replaces the `ffmpeg` child process with the same codec options, so the viewer
|
||||
//! still receives the same Annex-B access units (AUD, SPS/PPS, IDR, P).
|
||||
use std::ffi::CString;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Once};
|
||||
use std::time::{Duration, Instant};
|
||||
use ffmpeg_next as ff;
|
||||
use ff::format::{format::Format, Pixel};
|
||||
use ff::util::picture;
|
||||
use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
pub struct Config {
|
||||
pub display: String,
|
||||
pub input: (u32, u32),
|
||||
pub output: Option<(u32, u32)>,
|
||||
pub fps: u32,
|
||||
pub gop: u32,
|
||||
pub cq: u32,
|
||||
pub target_bps: i64,
|
||||
pub max_bps: i64,
|
||||
pub buffer_bits: i64,
|
||||
pub gpu: bool,
|
||||
}
|
||||
|
||||
/// `UU_VIDEO_PIPELINE=process` keeps the original ffmpeg child-process path.
|
||||
pub fn disabled() -> bool {
|
||||
std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process")
|
||||
}
|
||||
|
||||
fn text(error: ff::Error) -> String { error.to_string() }
|
||||
|
||||
fn ensure_init() -> Result<(), String> {
|
||||
static INIT: Once = Once::new();
|
||||
let mut result = Ok(());
|
||||
INIT.call_once(|| {
|
||||
if let Err(error) = ff::init() { result = Err(text(error)); }
|
||||
ff::device::register_all();
|
||||
});
|
||||
result
|
||||
}
|
||||
|
||||
/// Sends one access unit per encoder packet. Returns `Ok(false)` when the receiver is gone.
|
||||
fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender<Vec<u8>>) -> Result<bool, String> {
|
||||
loop {
|
||||
match encoder.receive_packet(packet) {
|
||||
Ok(()) => {
|
||||
let data = packet.data().unwrap_or_default().to_vec();
|
||||
if units.blocking_send(data).is_err() { return Ok(false); }
|
||||
}
|
||||
Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => return Ok(true),
|
||||
Err(error) => return Err(text(error)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR.
|
||||
pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>, idr: Arc<AtomicBool>) -> Result<(), String> {
|
||||
ensure_init()?;
|
||||
let name = CString::new("x11grab").map_err(|e| e.to_string())?;
|
||||
let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) };
|
||||
if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); }
|
||||
let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) });
|
||||
let mut capture_options = Dictionary::new();
|
||||
capture_options.set("framerate", &config.fps.to_string());
|
||||
capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1));
|
||||
// The viewer draws its own pointer; do not bake a second one into the video.
|
||||
capture_options.set("draw_mouse", "0");
|
||||
let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input();
|
||||
|
||||
let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?;
|
||||
let index = stream.index();
|
||||
let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?;
|
||||
|
||||
let (width, height) = config.output.unwrap_or(config.input);
|
||||
let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" })
|
||||
.ok_or("H.264 encoder is not available in this FFmpeg build")?;
|
||||
// Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one.
|
||||
let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?;
|
||||
context.set_width(width);
|
||||
context.set_height(height);
|
||||
context.set_format(Pixel::YUV420P);
|
||||
context.set_time_base(Rational::new(1, config.fps as i32));
|
||||
context.set_frame_rate(Some(Rational::new(config.fps as i32, 1)));
|
||||
context.set_gop(config.gop);
|
||||
context.set_max_b_frames(0);
|
||||
context.set_bit_rate(config.target_bps as usize);
|
||||
context.set_max_bit_rate(config.max_bps as usize);
|
||||
// The CLI's -maxrate/-bufsize map to these raw fields, which the encoder wrappers read.
|
||||
unsafe {
|
||||
let raw = context.as_mut_ptr();
|
||||
(*raw).rc_max_rate = config.max_bps;
|
||||
(*raw).rc_buffer_size = config.buffer_bits as i32;
|
||||
}
|
||||
let cq = config.cq.to_string();
|
||||
let gop = config.gop.to_string();
|
||||
let mut options = Dictionary::new();
|
||||
if config.gpu {
|
||||
for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"),
|
||||
("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] {
|
||||
options.set(key, value);
|
||||
}
|
||||
options.set("cq", &cq);
|
||||
} else {
|
||||
let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}");
|
||||
for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "baseline"), ("threads", "4")] {
|
||||
options.set(key, value);
|
||||
}
|
||||
options.set("crf", &cq);
|
||||
options.set("x264-params", &x264);
|
||||
}
|
||||
let mut encoder = context.open_as_with(codec, options).map_err(text)?;
|
||||
|
||||
let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None;
|
||||
let mut decoded = frame::Video::empty();
|
||||
let mut yuv = frame::Video::new(Pixel::YUV420P, width, height);
|
||||
let mut packet = Packet::empty();
|
||||
let started = Instant::now();
|
||||
let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps));
|
||||
let mut last_at: Option<Duration> = None;
|
||||
let mut last_slot: i64 = -1;
|
||||
for (stream, input_packet) in input.packets() {
|
||||
if stop.load(Ordering::Relaxed) { return Ok(()); }
|
||||
if stream.index() != index { continue; }
|
||||
decoder.send_packet(&input_packet).map_err(text)?;
|
||||
match decoder.receive_frame(&mut decoded) {
|
||||
Ok(()) => {}
|
||||
Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => continue,
|
||||
Err(error) => return Err(text(error)),
|
||||
}
|
||||
// x11grab catches up after a stall by grabbing back-to-back. A frame arriving within half
|
||||
// a frame interval of the previous one is part of that burst and is dropped, as the old
|
||||
// `-fps_mode vfr` did. Normal jitter stays well above that threshold.
|
||||
let now = started.elapsed();
|
||||
if last_at.is_some_and(|previous| now - previous < frame_interval / 2) { continue; }
|
||||
last_at = Some(now);
|
||||
let slot = ((now.as_micros() * u128::from(config.fps) + 500_000) / 1_000_000) as i64;
|
||||
let slot = slot.max(last_slot + 1);
|
||||
last_slot = slot;
|
||||
|
||||
let key = (decoded.format(), decoded.width(), decoded.height());
|
||||
if scaler.as_ref().map(|(_, k)| *k) != Some(key) {
|
||||
let context = scaling::Context::get(key.0, key.1, key.2, Pixel::YUV420P, width, height, scaling::Flags::BILINEAR).map_err(text)?;
|
||||
scaler = Some((context, key));
|
||||
}
|
||||
if let Some((context, _)) = scaler.as_mut() { context.run(&decoded, &mut yuv).map_err(text)?; }
|
||||
|
||||
yuv.set_pts(Some(slot));
|
||||
yuv.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None });
|
||||
encoder.send_frame(&yuv).map_err(text)?;
|
||||
if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); }
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -21,6 +21,7 @@ mod signaling;
|
||||
mod terminal;
|
||||
mod webrtc;
|
||||
mod video;
|
||||
mod inproc_encoder;
|
||||
mod capture_setting;
|
||||
mod audio;
|
||||
mod system_actions;
|
||||
|
||||
+55
-12
@@ -1,4 +1,5 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::process::Command;
|
||||
@@ -99,23 +100,51 @@ impl EncoderTask {
|
||||
}
|
||||
}
|
||||
|
||||
struct StopOnDrop(Arc<AtomicBool>);
|
||||
impl Drop for StopOnDrop { fn drop(&mut self) { self.0.store(true, std::sync::atomic::Ordering::Relaxed); } }
|
||||
|
||||
async fn read_encoder(
|
||||
size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
|
||||
latest: mpsc::Sender<Arc<CompleteFrame>>,
|
||||
settings: VideoSettings, sequence: Arc<std::sync::atomic::AtomicU64>,
|
||||
settings: VideoSettings, sequence: Arc<std::sync::atomic::AtomicU64>, idr: Arc<AtomicBool>,
|
||||
) -> Result<(), String> {
|
||||
// The sequence is shared so a CPU fallback or settings restart continues it (skip statistics stay correct).
|
||||
if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings).await; }
|
||||
match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings).await {
|
||||
if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await; }
|
||||
match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings,idr.clone()).await {
|
||||
Ok(())=>Ok(()),
|
||||
Err(error) if !latest.is_closed()=>{
|
||||
log::warn!("NVENC stream unavailable; switching scoped encoder to CPU: {}",error);
|
||||
read_encoder_with(size,display,portal,latest,false,sequence,settings).await
|
||||
read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await
|
||||
},
|
||||
Err(error)=>Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
/// Capture and encode inside this process; each packet is one access unit.
|
||||
async fn read_inprocess(
|
||||
config: crate::inproc_encoder::Config, latest: mpsc::Sender<Arc<CompleteFrame>>,
|
||||
sequence: Arc<std::sync::atomic::AtomicU64>, idr: Arc<AtomicBool>,
|
||||
) -> Result<(), String> {
|
||||
let (units_tx, mut units) = mpsc::channel::<Vec<u8>>(4);
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
// If this future is dropped (reader task aborted), the blocking encoder stops at its next packet.
|
||||
let _stop_on_drop = StopOnDrop(stop.clone());
|
||||
let job = tokio::task::spawn_blocking({
|
||||
let stop = stop.clone();
|
||||
move || crate::inproc_encoder::run(config, units_tx, stop, idr)
|
||||
});
|
||||
let mut gate = DependencyGate::default();
|
||||
while let Some(data) = units.recv().await {
|
||||
let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
|
||||
gate.submit(&latest, Arc::new(CompleteFrame { sequence, completed_at: Instant::now(), data }));
|
||||
if latest.is_closed() { break; }
|
||||
}
|
||||
// Dropping the receiver also unblocks the encoder if it is waiting to send.
|
||||
stop.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
drop(units);
|
||||
job.await.map_err(|e| format!("Encoder task: {e}"))?
|
||||
}
|
||||
|
||||
/// Rate-control arguments: constant quality capped at 30M (自动), or capped at the viewer's
|
||||
/// chosen bitrate. The VBV buffer stays at most 500k so a cap never adds queueing latency.
|
||||
fn rate_args(settings: &VideoSettings) -> (String, String) {
|
||||
@@ -126,10 +155,22 @@ fn rate_args(settings: &VideoSettings) -> (String, String) {
|
||||
|
||||
async fn read_encoder_with(
|
||||
size: (u32,u32),display:String,portal:Option<Arc<crate::portal::PortalSession>>,latest:mpsc::Sender<Arc<CompleteFrame>>,gpu:bool,sequence:Arc<std::sync::atomic::AtomicU64>,
|
||||
mut settings: VideoSettings,
|
||||
mut settings: VideoSettings, idr: Arc<AtomicBool>,
|
||||
)->Result<(),String>{
|
||||
let size = format!("{}x{}", size.0, size.1);
|
||||
let wayland=display.starts_with("wayland:");
|
||||
if !wayland && portal.is_none() && !crate::inproc_encoder::disabled() {
|
||||
let max_kbps = settings.max_kbps().unwrap_or(30_000) as i64;
|
||||
let config = crate::inproc_encoder::Config {
|
||||
display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps),
|
||||
gop: (u64::from(settings.fps) * GOP / FPS).max(1) as u32, cq: settings.cq() as u32,
|
||||
target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000,
|
||||
max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu,
|
||||
};
|
||||
log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process",
|
||||
display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"});
|
||||
return read_inprocess(config, latest, sequence, idr).await;
|
||||
}
|
||||
let size = format!("{}x{}", size.0, size.1);
|
||||
// The separate wayland-capture process produces frames at its own fixed rate.
|
||||
if wayland && portal.is_none() { settings.fps = FPS as u32; }
|
||||
let fps = settings.fps.to_string();
|
||||
@@ -256,11 +297,13 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option<Arc<crate::portal
|
||||
};
|
||||
let (tx,mut rx)=mpsc::channel(4);
|
||||
let sequence=Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let _reader=EncoderTask(tokio::spawn(read_encoder(size,display,portal,tx,VideoSettings::default(),sequence)));
|
||||
let _reader=EncoderTask(tokio::spawn(read_encoder(size,display,portal,tx,VideoSettings::default(),sequence,Arc::new(AtomicBool::new(false)))));
|
||||
let mut count=0;
|
||||
let mut bytes=0;
|
||||
tokio::time::timeout(Duration::from_secs(15),async {
|
||||
while count<5 {
|
||||
// UU_PROBE_FRAMES lengthens the sample for baseline measurements; default is 5 frames.
|
||||
let wanted:u64=std::env::var("UU_PROBE_FRAMES").ok().and_then(|v|v.parse().ok()).unwrap_or(5);
|
||||
tokio::time::timeout(Duration::from_secs(15+wanted/30),async {
|
||||
while count<wanted {
|
||||
let frame=rx.recv().await.ok_or("Video probe encoder ended")?;
|
||||
if count==0 && !is_independent_frame(&frame.data) { return Err("Video probe did not begin with IDR".to_owned()); }
|
||||
if let Some(file)=file.as_mut() { file.write_all(&frame.data).map_err(|e|e.to_string())?; }
|
||||
@@ -273,7 +316,7 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option<Arc<crate::portal
|
||||
}
|
||||
|
||||
pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalStaticSample>, size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
|
||||
mut settings: tokio::sync::watch::Receiver<VideoSettings>) -> Result<(), String> {
|
||||
mut settings: tokio::sync::watch::Receiver<VideoSettings>, idr: Arc<AtomicBool>) -> Result<(), String> {
|
||||
loop {
|
||||
if transport_closed(&pc) { return Ok(()); }
|
||||
if pc.connection_state() == RTCPeerConnectionState::Connected
|
||||
@@ -284,7 +327,7 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
|
||||
let (latest, mut receiver) = mpsc::channel(4);
|
||||
let sequence = Arc::new(std::sync::atomic::AtomicU64::new(0));
|
||||
let current = *settings.borrow_and_update();
|
||||
let mut reader = EncoderTask(tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone())));
|
||||
let mut reader = EncoderTask(tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone())));
|
||||
let mut previous = None;
|
||||
let mut timeline = RtpTimeline { fps: u64::from(current.fps), ..Default::default() };
|
||||
let mut settings_open = true;
|
||||
@@ -305,7 +348,7 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
|
||||
// still queued from the old encoder remain a valid chain before it.
|
||||
let next = *settings.borrow_and_update();
|
||||
timeline.fps = u64::from(next.fps);
|
||||
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone())).await;
|
||||
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await;
|
||||
},
|
||||
frame = receiver.recv() => {
|
||||
// A closed queue means the reader ended; its JoinHandle carries the reason.
|
||||
|
||||
+8
-1
@@ -184,8 +184,11 @@ impl WebRtcSession {
|
||||
"video_0".to_owned(),
|
||||
));
|
||||
|
||||
// Set by PLI/FIR; the desktop encoder answers with an IDR on its next frame.
|
||||
let idr_request = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
||||
let sender = pc.add_track(video_track.clone()).await
|
||||
.map_err(|e| format!("Add video track failed: {}", e))?;
|
||||
let idr_on_feedback = idr_request.clone();
|
||||
tokio::spawn(async move {
|
||||
let mut feedback = [0u8; 1500];
|
||||
let mut reports = 0usize;
|
||||
@@ -193,6 +196,10 @@ impl WebRtcSession {
|
||||
while let Ok((packets, _)) = sender.read(&mut feedback).await {
|
||||
for packet in packets {
|
||||
feedback_counts.observe(packet.as_any());
|
||||
if packet.as_any().is::<webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication>()
|
||||
|| packet.as_any().is::<webrtc::rtcp::payload_feedbacks::full_intra_request::FullIntraRequest>() {
|
||||
idr_on_feedback.store(true, std::sync::atomic::Ordering::Release);
|
||||
}
|
||||
if let Some(report) = packet.as_any().downcast_ref::<webrtc::rtcp::receiver_report::ReceiverReport>() {
|
||||
reports += 1;
|
||||
if reports == 1 || reports % 10 == 0 {
|
||||
@@ -261,7 +268,7 @@ impl WebRtcSession {
|
||||
// File/terminal/port-mapping connections carry no desktop video.
|
||||
if desktop {
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx).await {
|
||||
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, idr_request).await {
|
||||
log::error!("Live desktop stream stopped: {}", e);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in new issue
Block a user