Encode desktop video in-process instead of an ffmpeg child

Capture (x11grab) and H.264 encoding (h264_nvenc / libx264) now run in
this process through ffmpeg-next, with the same codec options as the old
ffmpeg CLI. Each encoder packet is one access unit, fed to the existing
DependencyGate. A PLI or FIR from the viewer forces the next frame to be
an IDR. Wayland and portal capture still use the child-process path.

Set UU_VIDEO_PIPELINE=process to fall back to the old ffmpeg child.
desktop-video-probe accepts UU_PROBE_FRAMES to lengthen the sample.

Measured with desktop-video-probe on ros2go-gpu (RTX 4090 D, Xvfb
1920x1080 @60fps, 600 frames, mostly static desktop). CPU is the sum of
the probe process and its children, sampled from /proc once per second.
Probe only: no WebRTC send, no audio.

  encoder     pipeline     CPU (%)    wall (s)   bitrate (kbps)
  nvenc       process      ~138       10.17      5356
  nvenc       in-process   ~43        10.34      5176
  libx264     process      ~135       10.09      5073
  libx264     in-process   ~72        10.08      4986

All four outputs decode without errors and start with an IDR.

Not covered: audio still runs its own ffmpeg child (src/audio.rs), PLI to
IDR has not been checked against a live viewer, and the CI jobs need the
libav*-dev and libclang-dev packages (added to ci.yml, release.yml,
AGENTS.md and the READMEs).
This commit is contained in:
ferstar authored and VallierDev committed 2026-10-15 07:43:21 +08:00
1 parent 078ca026a5
commit bf0aa6879a
11 files changed
+363 -22

No files matched your search

+2 -2
View File
@@ -14,7 +14,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev
libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo test --locked
@@ -28,7 +28,7 @@ jobs:
sudo apt-get update
sudo apt-get install -y build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \
libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev
libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- run: cargo test --locked --features wayland,portal
+1 -1
View File
@@ -17,7 +17,7 @@ jobs:
run: |
sudo apt-get update
sudo apt-get install -y build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev
libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev
- uses: dtolnay/rust-toolchain@stable
- run: cargo test --release --locked
- run: cargo build --release --locked
+1 -1
View File
@@ -36,7 +36,7 @@ Dependencies (Ubuntu/Debian):
```sh
sudo apt install build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev ffmpeg
libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev ffmpeg
# --features wayland: libwayland-dev libxkbcommon-dev wl-clipboard
# --features wayland,portal: + libglib2.0-dev libpipewire-0.3-dev
```
Generated
+137 -3
View File
@@ -212,6 +212,26 @@ dependencies = [
"serde",
]
[[package]]
name = "bindgen"
version = "0.64.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4"
dependencies = [
"bitflags 1.3.2",
"cexpr",
"clang-sys",
"lazy_static",
"lazycell",
"peeking_take_while",
"proc-macro2",
"quote",
"regex",
"rustc-hash 1.1.0",
"shlex 1.3.0",
"syn 1.0.109",
]
[[package]]
name = "bitflags"
version = "1.3.2"
@@ -276,7 +296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
dependencies = [
"find-msvc-tools",
"shlex",
"shlex 2.0.1",
]
[[package]]
@@ -291,6 +311,15 @@ dependencies = [
"subtle",
]
[[package]]
name = "cexpr"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
dependencies = [
"nom",
]
[[package]]
name = "cfg-if"
version = "1.0.5"
@@ -324,6 +353,17 @@ dependencies = [
"inout",
]
[[package]]
name = "clang-sys"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
dependencies = [
"glob",
"libc",
"libloading",
]
[[package]]
name = "colorchoice"
version = "1.0.5"
@@ -605,6 +645,31 @@ dependencies = [
"subtle",
]
[[package]]
name = "ffmpeg-next"
version = "6.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e72c72e8dcf638fb0fb03f033a954691662b5dabeaa3f85a6607d101569fccd"
dependencies = [
"bitflags 1.3.2",
"ffmpeg-sys-next",
"libc",
]
[[package]]
name = "ffmpeg-sys-next"
version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2529ad916d08c3562c754c21bc9b17a26c7882c0f5706cc2cd69472175f1620"
dependencies = [
"bindgen",
"cc",
"libc",
"num_cpus",
"pkg-config",
"vcpkg",
]
[[package]]
name = "fiat-crypto"
version = "0.2.9"
@@ -773,6 +838,12 @@ dependencies = [
"polyval",
]
[[package]]
name = "glob"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
[[package]]
name = "group"
version = "0.13.0"
@@ -784,6 +855,12 @@ dependencies = [
"subtle",
]
[[package]]
name = "hermit-abi"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284"
[[package]]
name = "hex"
version = "0.4.3"
@@ -1111,12 +1188,28 @@ version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
[[package]]
name = "lazycell"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "libc"
version = "0.2.190"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
[[package]]
name = "libloading"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
dependencies = [
"cfg-if",
"windows-link",
]
[[package]]
name = "litemap"
version = "0.8.3"
@@ -1265,6 +1358,16 @@ dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "oid-registry"
version = "0.6.1"
@@ -1339,6 +1442,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "pem"
version = "3.0.6"
@@ -1386,6 +1495,12 @@ dependencies = [
"spki",
]
[[package]]
name = "pkg-config"
version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548"
[[package]]
name = "polyval"
version = "0.6.2"
@@ -1472,7 +1587,7 @@ dependencies = [
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustc-hash 2.1.3",
"rustls 0.23.45",
"socket2 0.6.5",
"thiserror 2.0.21",
@@ -1493,7 +1608,7 @@ dependencies = [
"rand 0.10.3",
"rand_pcg",
"ring 0.17.14",
"rustc-hash",
"rustc-hash 2.1.3",
"rustls 0.23.45",
"rustls-pki-types",
"slab",
@@ -1740,6 +1855,12 @@ dependencies = [
"webrtc-util",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
[[package]]
name = "rustc-hash"
version = "2.1.3"
@@ -1958,6 +2079,12 @@ dependencies = [
"digest",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "shlex"
version = "2.0.1"
@@ -2493,6 +2620,7 @@ dependencies = [
"bytes",
"cc",
"env_logger",
"ffmpeg-next",
"flate2",
"futures-util",
"hex",
@@ -2521,6 +2649,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.5"
+1
View File
@@ -37,6 +37,7 @@ nix = { version = "0.29", features = ["term", "process", "fs", "ioctl", "signal"
log = "0.4"
qrcode = { version = "0.14", default-features = false }
env_logger = "0.11"
ffmpeg-next = { version = "6.1", default-features = false, features = ["codec", "format", "device", "software-scaling"] }
[build-dependencies]
cc = "1.0"
+1 -1
View File
@@ -87,7 +87,7 @@ permission; the permission is remembered once granted.
```sh
sudo apt install build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \
libgtk-3-dev ffmpeg
# optional: Wayland backend
sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard
+1 -1
View File
@@ -81,7 +81,7 @@ Wayland 下的限制:只支持单屏、固定分辨率;portal 后端暂不
```sh
sudo apt install build-essential pkg-config \
libx11-dev libxext-dev libxtst-dev libxfixes-dev \
libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \
libgtk-3-dev ffmpeg
# 可选:Wayland 后端
sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard
+155
View File
@@ -0,0 +1,155 @@
//! In-process X11 capture and H.264 encoding through libavdevice/libavcodec.
//! It replaces the `ffmpeg` child process with the same codec options, so the viewer
//! still receives the same Annex-B access units (AUD, SPS/PPS, IDR, P).
use std::ffi::CString;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Once};
use std::time::{Duration, Instant};
use ffmpeg_next as ff;
use ff::format::{format::Format, Pixel};
use ff::util::picture;
use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational};
use tokio::sync::mpsc;
pub struct Config {
pub display: String,
pub input: (u32, u32),
pub output: Option<(u32, u32)>,
pub fps: u32,
pub gop: u32,
pub cq: u32,
pub target_bps: i64,
pub max_bps: i64,
pub buffer_bits: i64,
pub gpu: bool,
}
/// `UU_VIDEO_PIPELINE=process` keeps the original ffmpeg child-process path.
pub fn disabled() -> bool {
std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process")
}
fn text(error: ff::Error) -> String { error.to_string() }
fn ensure_init() -> Result<(), String> {
static INIT: Once = Once::new();
let mut result = Ok(());
INIT.call_once(|| {
if let Err(error) = ff::init() { result = Err(text(error)); }
ff::device::register_all();
});
result
}
/// Sends one access unit per encoder packet. Returns `Ok(false)` when the receiver is gone.
fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender<Vec<u8>>) -> Result<bool, String> {
loop {
match encoder.receive_packet(packet) {
Ok(()) => {
let data = packet.data().unwrap_or_default().to_vec();
if units.blocking_send(data).is_err() { return Ok(false); }
}
Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => return Ok(true),
Err(error) => return Err(text(error)),
}
}
}
/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR.
pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>, idr: Arc<AtomicBool>) -> Result<(), String> {
ensure_init()?;
let name = CString::new("x11grab").map_err(|e| e.to_string())?;
let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) };
if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); }
let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) });
let mut capture_options = Dictionary::new();
capture_options.set("framerate", &config.fps.to_string());
capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1));
// The viewer draws its own pointer; do not bake a second one into the video.
capture_options.set("draw_mouse", "0");
let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input();
let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?;
let index = stream.index();
let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?;
let (width, height) = config.output.unwrap_or(config.input);
let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" })
.ok_or("H.264 encoder is not available in this FFmpeg build")?;
// Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one.
let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?;
context.set_width(width);
context.set_height(height);
context.set_format(Pixel::YUV420P);
context.set_time_base(Rational::new(1, config.fps as i32));
context.set_frame_rate(Some(Rational::new(config.fps as i32, 1)));
context.set_gop(config.gop);
context.set_max_b_frames(0);
context.set_bit_rate(config.target_bps as usize);
context.set_max_bit_rate(config.max_bps as usize);
// The CLI's -maxrate/-bufsize map to these raw fields, which the encoder wrappers read.
unsafe {
let raw = context.as_mut_ptr();
(*raw).rc_max_rate = config.max_bps;
(*raw).rc_buffer_size = config.buffer_bits as i32;
}
let cq = config.cq.to_string();
let gop = config.gop.to_string();
let mut options = Dictionary::new();
if config.gpu {
for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"),
("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] {
options.set(key, value);
}
options.set("cq", &cq);
} else {
let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}");
for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "baseline"), ("threads", "4")] {
options.set(key, value);
}
options.set("crf", &cq);
options.set("x264-params", &x264);
}
let mut encoder = context.open_as_with(codec, options).map_err(text)?;
let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None;
let mut decoded = frame::Video::empty();
let mut yuv = frame::Video::new(Pixel::YUV420P, width, height);
let mut packet = Packet::empty();
let started = Instant::now();
let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps));
let mut last_at: Option<Duration> = None;
let mut last_slot: i64 = -1;
for (stream, input_packet) in input.packets() {
if stop.load(Ordering::Relaxed) { return Ok(()); }
if stream.index() != index { continue; }
decoder.send_packet(&input_packet).map_err(text)?;
match decoder.receive_frame(&mut decoded) {
Ok(()) => {}
Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => continue,
Err(error) => return Err(text(error)),
}
// x11grab catches up after a stall by grabbing back-to-back. A frame arriving within half
// a frame interval of the previous one is part of that burst and is dropped, as the old
// `-fps_mode vfr` did. Normal jitter stays well above that threshold.
let now = started.elapsed();
if last_at.is_some_and(|previous| now - previous < frame_interval / 2) { continue; }
last_at = Some(now);
let slot = ((now.as_micros() * u128::from(config.fps) + 500_000) / 1_000_000) as i64;
let slot = slot.max(last_slot + 1);
last_slot = slot;
let key = (decoded.format(), decoded.width(), decoded.height());
if scaler.as_ref().map(|(_, k)| *k) != Some(key) {
let context = scaling::Context::get(key.0, key.1, key.2, Pixel::YUV420P, width, height, scaling::Flags::BILINEAR).map_err(text)?;
scaler = Some((context, key));
}
if let Some((context, _)) = scaler.as_mut() { context.run(&decoded, &mut yuv).map_err(text)?; }
yuv.set_pts(Some(slot));
yuv.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None });
encoder.send_frame(&yuv).map_err(text)?;
if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); }
}
Ok(())
}
+1
View File
@@ -21,6 +21,7 @@ mod signaling;
mod terminal;
mod webrtc;
mod video;
mod inproc_encoder;
mod capture_setting;
mod audio;
mod system_actions;
+55 -12
View File
@@ -1,4 +1,5 @@
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::{Duration, Instant};
use tokio::io::AsyncReadExt;
use tokio::process::Command;
@@ -99,23 +100,51 @@ impl EncoderTask {
}
}
struct StopOnDrop(Arc<AtomicBool>);
impl Drop for StopOnDrop { fn drop(&mut self) { self.0.store(true, std::sync::atomic::Ordering::Relaxed); } }
async fn read_encoder(
size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
latest: mpsc::Sender<Arc<CompleteFrame>>,
settings: VideoSettings, sequence: Arc<std::sync::atomic::AtomicU64>,
settings: VideoSettings, sequence: Arc<std::sync::atomic::AtomicU64>, idr: Arc<AtomicBool>,
) -> Result<(), String> {
// The sequence is shared so a CPU fallback or settings restart continues it (skip statistics stay correct).
if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings).await; }
match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings).await {
if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await; }
match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings,idr.clone()).await {
Ok(())=>Ok(()),
Err(error) if !latest.is_closed()=>{
log::warn!("NVENC stream unavailable; switching scoped encoder to CPU: {}",error);
read_encoder_with(size,display,portal,latest,false,sequence,settings).await
read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await
},
Err(error)=>Err(error),
}
}
/// Capture and encode inside this process; each packet is one access unit.
async fn read_inprocess(
config: crate::inproc_encoder::Config, latest: mpsc::Sender<Arc<CompleteFrame>>,
sequence: Arc<std::sync::atomic::AtomicU64>, idr: Arc<AtomicBool>,
) -> Result<(), String> {
let (units_tx, mut units) = mpsc::channel::<Vec<u8>>(4);
let stop = Arc::new(AtomicBool::new(false));
// If this future is dropped (reader task aborted), the blocking encoder stops at its next packet.
let _stop_on_drop = StopOnDrop(stop.clone());
let job = tokio::task::spawn_blocking({
let stop = stop.clone();
move || crate::inproc_encoder::run(config, units_tx, stop, idr)
});
let mut gate = DependencyGate::default();
while let Some(data) = units.recv().await {
let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
gate.submit(&latest, Arc::new(CompleteFrame { sequence, completed_at: Instant::now(), data }));
if latest.is_closed() { break; }
}
// Dropping the receiver also unblocks the encoder if it is waiting to send.
stop.store(true, std::sync::atomic::Ordering::Relaxed);
drop(units);
job.await.map_err(|e| format!("Encoder task: {e}"))?
}
/// Rate-control arguments: constant quality capped at 30M (自动), or capped at the viewer's
/// chosen bitrate. The VBV buffer stays at most 500k so a cap never adds queueing latency.
fn rate_args(settings: &VideoSettings) -> (String, String) {
@@ -126,10 +155,22 @@ fn rate_args(settings: &VideoSettings) -> (String, String) {
async fn read_encoder_with(
size: (u32,u32),display:String,portal:Option<Arc<crate::portal::PortalSession>>,latest:mpsc::Sender<Arc<CompleteFrame>>,gpu:bool,sequence:Arc<std::sync::atomic::AtomicU64>,
mut settings: VideoSettings,
mut settings: VideoSettings, idr: Arc<AtomicBool>,
)->Result<(),String>{
let size = format!("{}x{}", size.0, size.1);
let wayland=display.starts_with("wayland:");
if !wayland && portal.is_none() && !crate::inproc_encoder::disabled() {
let max_kbps = settings.max_kbps().unwrap_or(30_000) as i64;
let config = crate::inproc_encoder::Config {
display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps),
gop: (u64::from(settings.fps) * GOP / FPS).max(1) as u32, cq: settings.cq() as u32,
target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000,
max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu,
};
log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process",
display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"});
return read_inprocess(config, latest, sequence, idr).await;
}
let size = format!("{}x{}", size.0, size.1);
// The separate wayland-capture process produces frames at its own fixed rate.
if wayland && portal.is_none() { settings.fps = FPS as u32; }
let fps = settings.fps.to_string();
@@ -256,11 +297,13 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option<Arc<crate::portal
};
let (tx,mut rx)=mpsc::channel(4);
let sequence=Arc::new(std::sync::atomic::AtomicU64::new(0));
let _reader=EncoderTask(tokio::spawn(read_encoder(size,display,portal,tx,VideoSettings::default(),sequence)));
let _reader=EncoderTask(tokio::spawn(read_encoder(size,display,portal,tx,VideoSettings::default(),sequence,Arc::new(AtomicBool::new(false)))));
let mut count=0;
let mut bytes=0;
tokio::time::timeout(Duration::from_secs(15),async {
while count<5 {
// UU_PROBE_FRAMES lengthens the sample for baseline measurements; default is 5 frames.
let wanted:u64=std::env::var("UU_PROBE_FRAMES").ok().and_then(|v|v.parse().ok()).unwrap_or(5);
tokio::time::timeout(Duration::from_secs(15+wanted/30),async {
while count<wanted {
let frame=rx.recv().await.ok_or("Video probe encoder ended")?;
if count==0 && !is_independent_frame(&frame.data) { return Err("Video probe did not begin with IDR".to_owned()); }
if let Some(file)=file.as_mut() { file.write_all(&frame.data).map_err(|e|e.to_string())?; }
@@ -273,7 +316,7 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option<Arc<crate::portal
}
pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalStaticSample>, size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
mut settings: tokio::sync::watch::Receiver<VideoSettings>) -> Result<(), String> {
mut settings: tokio::sync::watch::Receiver<VideoSettings>, idr: Arc<AtomicBool>) -> Result<(), String> {
loop {
if transport_closed(&pc) { return Ok(()); }
if pc.connection_state() == RTCPeerConnectionState::Connected
@@ -284,7 +327,7 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
let (latest, mut receiver) = mpsc::channel(4);
let sequence = Arc::new(std::sync::atomic::AtomicU64::new(0));
let current = *settings.borrow_and_update();
let mut reader = EncoderTask(tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone())));
let mut reader = EncoderTask(tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone())));
let mut previous = None;
let mut timeline = RtpTimeline { fps: u64::from(current.fps), ..Default::default() };
let mut settings_open = true;
@@ -305,7 +348,7 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
// still queued from the old encoder remain a valid chain before it.
let next = *settings.borrow_and_update();
timeline.fps = u64::from(next.fps);
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone())).await;
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await;
},
frame = receiver.recv() => {
// A closed queue means the reader ended; its JoinHandle carries the reason.
+8 -1
View File
@@ -184,8 +184,11 @@ impl WebRtcSession {
"video_0".to_owned(),
));
// Set by PLI/FIR; the desktop encoder answers with an IDR on its next frame.
let idr_request = Arc::new(std::sync::atomic::AtomicBool::new(false));
let sender = pc.add_track(video_track.clone()).await
.map_err(|e| format!("Add video track failed: {}", e))?;
let idr_on_feedback = idr_request.clone();
tokio::spawn(async move {
let mut feedback = [0u8; 1500];
let mut reports = 0usize;
@@ -193,6 +196,10 @@ impl WebRtcSession {
while let Ok((packets, _)) = sender.read(&mut feedback).await {
for packet in packets {
feedback_counts.observe(packet.as_any());
if packet.as_any().is::<webrtc::rtcp::payload_feedbacks::picture_loss_indication::PictureLossIndication>()
|| packet.as_any().is::<webrtc::rtcp::payload_feedbacks::full_intra_request::FullIntraRequest>() {
idr_on_feedback.store(true, std::sync::atomic::Ordering::Release);
}
if let Some(report) = packet.as_any().downcast_ref::<webrtc::rtcp::receiver_report::ReceiverReport>() {
reports += 1;
if reports == 1 || reports % 10 == 0 {
@@ -261,7 +268,7 @@ impl WebRtcSession {
// File/terminal/port-mapping connections carry no desktop video.
if desktop {
tokio::spawn(async move {
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx).await {
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, idr_request).await {
log::error!("Live desktop stream stopped: {}", e);
}
});