Capture only changed frames with XDamage on the in-process path

The in-process encoder can now capture through XShm and XDamage in pure Rust
(x11rb) instead of libav's x11grab. Damage marks the picture dirty; a dirty
picture is captured and encoded on a fixed frame tick so the stream keeps a
steady cadence. An idle screen sends a keepalive every 40-67 ms (15-25 fps,
random) so the viewer does not treat a static desktop as stalled.

- src/damage_capture.rs: XDamage NonEmpty reports and XShm copy-out
- src/inproc_encoder.rs: run_damage loop; x11grab stays as the fallback
- UU_CAPTURE=grab forces the x11grab path
- x11rb is an optional dependency, enabled by the inproc-av feature only

Measured on a static desktop, CPU drops from about 12% to about 1% versus
x11grab. Moving content costs about the same CPU as x11grab.
This commit is contained in:
ferstar authored and VallierDev committed 2026-10-15 08:22:14 +08:00
1 parent b66c0da9e1
commit 367be30a0f
6 files changed
+296 -20

No files matched your search

Generated
+55
View File
@@ -111,6 +111,12 @@ dependencies = [
"rustversion",
]
[[package]]
name = "as-raw-xcb-connection"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b"
[[package]]
name = "asn1-rs"
version = "0.5.2"
@@ -801,6 +807,16 @@ dependencies = [
"zeroize",
]
[[package]]
name = "gethostname"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8"
dependencies = [
"rustix",
"windows-link",
]
[[package]]
name = "getrandom"
version = "0.2.17"
@@ -1210,6 +1226,12 @@ dependencies = [
"windows-link",
]
[[package]]
name = "linux-raw-sys"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
[[package]]
name = "litemap"
version = "0.8.3"
@@ -1885,6 +1907,19 @@ dependencies = [
"nom",
]
[[package]]
name = "rustix"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d"
dependencies = [
"bitflags 2.13.2",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
]
[[package]]
name = "rustls"
version = "0.21.12"
@@ -2636,6 +2671,7 @@ dependencies = [
"tokio",
"tokio-tungstenite",
"webrtc",
"x11rb",
]
[[package]]
@@ -3094,6 +3130,25 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc"
[[package]]
name = "x11rb"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414"
dependencies = [
"as-raw-xcb-connection",
"gethostname",
"libc",
"rustix",
"x11rb-protocol",
]
[[package]]
name = "x11rb-protocol"
version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd"
[[package]]
name = "x25519-dalek"
version = "2.0.1"
+2 -1
View File
@@ -19,7 +19,7 @@ wayland = []
portal = []
# In-process capture and encoding through FFmpeg 6 libraries (libavcodec/libavdevice dev packages).
# Off by default so release binaries do not link the build machine's libav.
inproc-av = ["dep:ffmpeg-next"]
inproc-av = ["dep:ffmpeg-next", "dep:x11rb"]
[dependencies]
tokio = { version = "1", features = ["full"] }
@@ -41,6 +41,7 @@ log = "0.4"
qrcode = { version = "0.14", default-features = false }
env_logger = "0.11"
ffmpeg-next = { version = "6.1", optional = true, default-features = false, features = ["codec", "format", "device", "software-scaling", "software-resampling"] }
x11rb = { version = "0.13", features = ["damage", "shm", "allow-unsafe-code"], optional = true }
[build-dependencies]
cc = "1.0"
+120
View File
@@ -0,0 +1,120 @@
//! Damage-driven X11 capture for the in-process encoder: XDamage reports when the root window changed,
//! XShm copies it out as 32-bit BGRX. Pure Rust over x11rb; only the SysV shared-memory calls use libc.
use std::os::unix::io::AsRawFd;
use std::time::{Duration, Instant};
use x11rb::connection::{Connection, RequestConnection};
use x11rb::protocol::damage::{self, ConnectionExt as _, ReportLevel};
use x11rb::protocol::shm::{self, ConnectionExt as _};
use x11rb::protocol::xproto::Window;
use x11rb::rust_connection::RustConnection;
use x11rb::NONE;
/// X11 ZPixmap image format.
const Z_PIXMAP: u8 = 2;
pub struct DamageCapture {
conn: RustConnection,
root: Window,
damage: damage::Damage,
damage_event: u8,
width: u16,
height: u16,
seg: shm::Seg,
shm_ptr: *mut u8,
len: usize,
}
fn text(error: impl std::fmt::Display) -> String { error.to_string() }
impl DamageCapture {
/// Connects to `display` and sets up XDamage and XShm. Errors name the missing extension or failed step.
pub fn open(display: &str) -> Result<Self, String> {
let (conn, screen_num) = RustConnection::connect(Some(display)).map_err(text)?;
let damage_info = conn.extension_information(damage::X11_EXTENSION_NAME).map_err(text)?
.ok_or("XDamage extension is missing")?;
if conn.extension_information(shm::X11_EXTENSION_NAME).map_err(text)?.is_none() {
return Err("XShm extension is missing".into());
}
conn.damage_query_version(1, 1).map_err(text)?.reply().map_err(text)?;
let setup = conn.setup();
let screen = setup.roots.get(screen_num).ok_or("X screen not found")?;
let (root, width, height, depth) = (screen.root, screen.width_in_pixels, screen.height_in_pixels, screen.root_depth);
// The frame layout assumes packed 32-bit pixels; anything else would shear the picture.
let format = setup.pixmap_formats.iter().find(|f| f.depth == depth).ok_or("no pixmap format for the root depth")?;
if format.bits_per_pixel != 32 { return Err(format!("root depth {depth} is not 32 bits per pixel")); }
let len = width as usize * height as usize * 4;
// The segment is marked for removal once attached; the kernel frees it after the last detach.
let shmid = unsafe { libc::shmget(libc::IPC_PRIVATE, len, libc::IPC_CREAT | 0o600) };
if shmid < 0 { return Err("shmget failed".into()); }
let shm_ptr = unsafe { libc::shmat(shmid, std::ptr::null(), 0) } as *mut u8;
if shm_ptr as isize == -1 {
unsafe { libc::shmctl(shmid, libc::IPC_RMID, std::ptr::null_mut()) };
return Err("shmat failed".into());
}
let seg = conn.generate_id().map_err(text)?;
conn.shm_attach(seg, shmid as u32, false).map_err(text)?.check().map_err(text)?;
unsafe { libc::shmctl(shmid, libc::IPC_RMID, std::ptr::null_mut()) };
// NonEmpty: one event each time the root goes from clean to damaged; re-armed in `wait`.
let damage = conn.generate_id().map_err(text)?;
conn.damage_create(damage, root, ReportLevel::NON_EMPTY).map_err(text)?.check().map_err(text)?;
conn.flush().map_err(text)?;
Ok(Self { conn, root, damage, damage_event: damage_info.first_event, width, height, seg, shm_ptr, len })
}
pub fn size(&self) -> (u32, u32) { (self.width as u32, self.height as u32) }
/// Waits up to `timeout` for the screen to change. Returns `true` on change, `false` on timeout.
pub fn wait(&mut self, timeout: Duration) -> Result<bool, String> {
let deadline = Instant::now() + timeout;
loop {
let mut changed = false;
while let Some(event) = self.conn.poll_for_raw_event().map_err(text)? {
// XDamageNotify is the extension's first event; the wire code keeps the high bit for synthetic events.
if event[0] & 0x7f == self.damage_event { changed = true; }
}
if changed {
self.conn.damage_subtract(self.damage, NONE, NONE).map_err(text)?;
self.conn.flush().map_err(text)?;
return Ok(true);
}
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() { return Ok(false); }
let mut pfd = libc::pollfd { fd: self.conn.stream().as_raw_fd(), events: libc::POLLIN, revents: 0 };
match unsafe { libc::poll(&mut pfd, 1, remaining.as_millis().max(1) as i32) } {
ready if ready < 0 => return Err("poll on the X connection failed".into()),
// Readable but not yet a whole event: poll_for_raw_event would return None and poll would
// fire again at once, spinning a core. Block for the event instead.
ready if ready > 0 => {
let event = self.conn.wait_for_raw_event().map_err(text)?;
if event[0] & 0x7f == self.damage_event { changed = true; }
if changed {
self.conn.damage_subtract(self.damage, NONE, NONE).map_err(text)?;
self.conn.flush().map_err(text)?;
return Ok(true);
}
}
_ => {}
}
}
}
/// Copies the root into the shared segment and returns it as rows of `width * 4` bytes.
/// The slice is valid until the next call.
pub fn capture(&mut self) -> Result<&[u8], String> {
self.conn.shm_get_image(self.root, 0, 0, self.width, self.height, !0, Z_PIXMAP, self.seg, 0)
.map_err(text)?.reply().map_err(text)?;
Ok(unsafe { std::slice::from_raw_parts(self.shm_ptr, self.len) })
}
}
impl Drop for DamageCapture {
fn drop(&mut self) {
let _ = self.conn.damage_destroy(self.damage);
let _ = self.conn.shm_detach(self.seg);
let _ = self.conn.flush();
unsafe { libc::shmdt(self.shm_ptr as *const libc::c_void) };
}
}
+116 -19
View File
@@ -12,6 +12,8 @@ use ff::util::picture;
use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational};
use tokio::sync::mpsc;
use crate::damage_capture::DamageCapture;
pub struct Config {
pub display: String,
pub input: (u32, u32),
@@ -23,6 +25,8 @@ pub struct Config {
pub max_bps: i64,
pub buffer_bits: i64,
pub gpu: bool,
/// Capture on XDamage changes (XShm) instead of x11grab; `UU_CAPTURE=grab` turns it off.
pub damage: bool,
}
/// `UU_VIDEO_PIPELINE=process` keeps the original ffmpeg child-process path.
@@ -30,6 +34,11 @@ pub fn disabled() -> bool {
std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process")
}
/// Damage-driven capture is on unless `UU_CAPTURE=grab` asks for x11grab.
pub fn damage_enabled() -> bool {
std::env::var("UU_CAPTURE").map_or(true, |v| v != "grab")
}
/// In-process capture is used unless the process path is requested, or a bundled FFmpeg prefix
/// (`UU_FFMPEG_ROOT` / config `ffmpeg_root`) is set: that build was chosen on purpose, so keep using it.
pub fn selected() -> bool {
@@ -62,24 +71,8 @@ fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender
}
}
/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR.
pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>, idr: Arc<AtomicBool>) -> Result<(), String> {
ensure_init()?;
let name = CString::new("x11grab").map_err(|e| e.to_string())?;
let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) };
if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); }
let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) });
let mut capture_options = Dictionary::new();
capture_options.set("framerate", &config.fps.to_string());
capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1));
// The viewer draws its own pointer; do not bake a second one into the video.
capture_options.set("draw_mouse", "0");
let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input();
let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?;
let index = stream.index();
let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?;
/// Opens the H.264 encoder for `config`. Returns the encoder, its input pixel format and output size.
fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), String> {
let (width, height) = config.output.unwrap_or(config.input);
let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" })
.ok_or("H.264 encoder is not available in this FFmpeg build")?;
@@ -119,7 +112,35 @@ pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>,
options.set("crf", &cq);
options.set("x264-params", &x264);
}
let mut encoder = context.open_as_with(codec, options).map_err(text)?;
let encoder = context.open_as_with(codec, options).map_err(text)?;
Ok((encoder, pixel, width, height))
}
/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR.
pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>, idr: Arc<AtomicBool>) -> Result<(), String> {
ensure_init()?;
if config.damage {
match DamageCapture::open(&config.display) {
Ok(capture) => return run_damage(config, capture, units, stop, idr),
Err(error) => log::warn!("Damage-driven capture unavailable, using x11grab: {error}"),
}
}
let name = CString::new("x11grab").map_err(|e| e.to_string())?;
let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) };
if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); }
let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) });
let mut capture_options = Dictionary::new();
capture_options.set("framerate", &config.fps.to_string());
capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1));
// The viewer draws its own pointer; do not bake a second one into the video.
capture_options.set("draw_mouse", "0");
let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input();
let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?;
let index = stream.index();
let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?;
let (mut encoder, pixel, width, height) = open_encoder(&config)?;
let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None;
let mut decoded = frame::Video::empty();
@@ -169,6 +190,82 @@ pub fn run(config: Config, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>,
Ok(())
}
/// Damage-driven loop. Damage only marks the picture dirty; a dirty picture is captured and encoded on
/// the next fixed frame tick, so the stream keeps a steady cadence while the screen moves. The viewer's
/// jitter buffer grows with irregular arrivals, and irregular sends were what raised its latency.
/// An idle screen sends a keepalive about once a second so the viewer still sees traffic.
fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender<Vec<u8>>, stop: Arc<AtomicBool>, idr: Arc<AtomicBool>) -> Result<(), String> {
let (mut encoder, pixel, width, height) = open_encoder(&config)?;
let (cap_width, cap_height) = capture.size();
let mut captured = frame::Video::new(Pixel::BGRZ, cap_width, cap_height);
let mut converted = frame::Video::new(pixel, width, height);
let mut scaler = None;
let mut packet = Packet::empty();
let started = Instant::now();
let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps));
// Keepalive gaps are random between 15 and 25 fps: a steady 1 fps reads as a stall to the viewer.
let mut seed = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(1, |d| d.subsec_nanos()) | 1;
let mut keepalive = random_keepalive(&mut seed);
let mut dirty = true;
let mut next_tick = Instant::now();
let mut last_send = Instant::now();
let mut last_slot: i64 = -1;
loop {
if stop.load(Ordering::Relaxed) { return Ok(()); }
let deadline = if dirty { next_tick } else { last_send + keepalive };
let wait = deadline.saturating_duration_since(Instant::now());
if capture.wait(wait.max(Duration::from_millis(1)))? {
dirty = true;
continue;
}
let now = Instant::now();
if dirty && now < next_tick { continue; }
if !dirty && now < last_send + keepalive { continue; }
let source = capture.capture()?;
let bytes_per_line = cap_width as usize * 4;
let stride = captured.stride(0);
let row = bytes_per_line.min(stride);
for (y, src) in source.chunks_exact(bytes_per_line).enumerate() {
captured.data_mut(0)[y * stride..y * stride + row].copy_from_slice(&src[..row]);
}
let slot = ((started.elapsed().as_micros() * u128::from(config.fps) + 500_000) / 1_000_000) as i64;
let slot = slot.max(last_slot + 1);
last_slot = slot;
let frame = if pixel == Pixel::BGRZ && width == cap_width && height == cap_height {
&mut captured
} else {
if scaler.is_none() {
scaler = Some(scaling::Context::get(Pixel::BGRZ, cap_width, cap_height, pixel, width, height, scaling::Flags::BILINEAR).map_err(text)?);
}
if let Some(context) = scaler.as_mut() { context.run(&captured, &mut converted).map_err(text)?; }
&mut converted
};
frame.set_pts(Some(slot));
frame.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None });
encoder.send_frame(frame).map_err(text)?;
if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); }
let now = Instant::now();
if dirty {
// Keep the tick phase; after a long idle, restart the phase from now instead of bursting to catch up.
next_tick = (next_tick + frame_interval).max(now);
dirty = false;
}
last_send = now;
keepalive = random_keepalive(&mut seed);
}
}
/// Picks the next keepalive gap in 40–67 ms (15–25 fps) with a small xorshift generator.
fn random_keepalive(seed: &mut u32) -> Duration {
*seed ^= *seed << 13;
*seed ^= *seed >> 17;
*seed ^= *seed << 5;
Duration::from_micros(40_000 + u64::from(*seed % 26_700))
}
/// Encodes a PulseAudio/PipeWire source to Opus in this process. Each Opus packet is sent as is.
pub fn run_audio(source: &str, units: mpsc::Sender<bytes::Bytes>, stop: Arc<AtomicBool>) -> Result<(), String> {
ensure_init()?;
+2
View File
@@ -22,6 +22,8 @@ mod terminal;
mod webrtc;
mod video;
#[cfg(feature = "inproc-av")]
mod damage_capture;
#[cfg(feature = "inproc-av")]
mod inproc_encoder;
mod capture_setting;
mod audio;
+1
View File
@@ -179,6 +179,7 @@ async fn read_encoder_with(
gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32,
target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000,
max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu,
damage: crate::inproc_encoder::damage_enabled(),
};
log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process",
display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"});