diff --git a/Cargo.lock b/Cargo.lock index 40dbfca..5843abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "as-raw-xcb-connection" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175571dd1d178ced59193a6fc02dde1b972eb0bc56c892cde9beeceac5bf0f6b" + [[package]] name = "asn1-rs" version = "0.5.2" @@ -801,6 +807,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "gethostname" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" +dependencies = [ + "rustix", + "windows-link", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1210,6 +1226,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.3" @@ -1885,6 +1907,19 @@ dependencies = [ "nom", ] +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.21.12" @@ -2636,6 +2671,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "webrtc", + "x11rb", ] [[package]] @@ -3094,6 +3130,25 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x11rb" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9993aa5be5a26815fe2c3eacfc1fde061fc1a1f094bf1ad2a18bf9c495dd7414" +dependencies = [ + "as-raw-xcb-connection", + "gethostname", + "libc", + "rustix", + "x11rb-protocol", +] + +[[package]] +name = "x11rb-protocol" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6fc2961e4ef194dcbfe56bb845534d0dc8098940c7e5c012a258bfec6701bd" + [[package]] name = "x25519-dalek" version = "2.0.1" diff --git a/Cargo.toml b/Cargo.toml index 9709751..d04cc99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ wayland = [] portal = [] # In-process capture and encoding through FFmpeg 6 libraries (libavcodec/libavdevice dev packages). # Off by default so release binaries do not link the build machine's libav. -inproc-av = ["dep:ffmpeg-next"] +inproc-av = ["dep:ffmpeg-next", "dep:x11rb"] [dependencies] tokio = { version = "1", features = ["full"] } @@ -41,6 +41,7 @@ log = "0.4" qrcode = { version = "0.14", default-features = false } env_logger = "0.11" ffmpeg-next = { version = "6.1", optional = true, default-features = false, features = ["codec", "format", "device", "software-scaling", "software-resampling"] } +x11rb = { version = "0.13", features = ["damage", "shm", "allow-unsafe-code"], optional = true } [build-dependencies] cc = "1.0" diff --git a/src/damage_capture.rs b/src/damage_capture.rs new file mode 100644 index 0000000..aad2742 --- /dev/null +++ b/src/damage_capture.rs @@ -0,0 +1,120 @@ +//! Damage-driven X11 capture for the in-process encoder: XDamage reports when the root window changed, +//! XShm copies it out as 32-bit BGRX. Pure Rust over x11rb; only the SysV shared-memory calls use libc. +use std::os::unix::io::AsRawFd; +use std::time::{Duration, Instant}; +use x11rb::connection::{Connection, RequestConnection}; +use x11rb::protocol::damage::{self, ConnectionExt as _, ReportLevel}; +use x11rb::protocol::shm::{self, ConnectionExt as _}; +use x11rb::protocol::xproto::Window; +use x11rb::rust_connection::RustConnection; +use x11rb::NONE; + +/// X11 ZPixmap image format. +const Z_PIXMAP: u8 = 2; + +pub struct DamageCapture { + conn: RustConnection, + root: Window, + damage: damage::Damage, + damage_event: u8, + width: u16, + height: u16, + seg: shm::Seg, + shm_ptr: *mut u8, + len: usize, +} + +fn text(error: impl std::fmt::Display) -> String { error.to_string() } + +impl DamageCapture { + /// Connects to `display` and sets up XDamage and XShm. Errors name the missing extension or failed step. + pub fn open(display: &str) -> Result { + let (conn, screen_num) = RustConnection::connect(Some(display)).map_err(text)?; + let damage_info = conn.extension_information(damage::X11_EXTENSION_NAME).map_err(text)? + .ok_or("XDamage extension is missing")?; + if conn.extension_information(shm::X11_EXTENSION_NAME).map_err(text)?.is_none() { + return Err("XShm extension is missing".into()); + } + conn.damage_query_version(1, 1).map_err(text)?.reply().map_err(text)?; + + let setup = conn.setup(); + let screen = setup.roots.get(screen_num).ok_or("X screen not found")?; + let (root, width, height, depth) = (screen.root, screen.width_in_pixels, screen.height_in_pixels, screen.root_depth); + // The frame layout assumes packed 32-bit pixels; anything else would shear the picture. + let format = setup.pixmap_formats.iter().find(|f| f.depth == depth).ok_or("no pixmap format for the root depth")?; + if format.bits_per_pixel != 32 { return Err(format!("root depth {depth} is not 32 bits per pixel")); } + let len = width as usize * height as usize * 4; + + // The segment is marked for removal once attached; the kernel frees it after the last detach. + let shmid = unsafe { libc::shmget(libc::IPC_PRIVATE, len, libc::IPC_CREAT | 0o600) }; + if shmid < 0 { return Err("shmget failed".into()); } + let shm_ptr = unsafe { libc::shmat(shmid, std::ptr::null(), 0) } as *mut u8; + if shm_ptr as isize == -1 { + unsafe { libc::shmctl(shmid, libc::IPC_RMID, std::ptr::null_mut()) }; + return Err("shmat failed".into()); + } + let seg = conn.generate_id().map_err(text)?; + conn.shm_attach(seg, shmid as u32, false).map_err(text)?.check().map_err(text)?; + unsafe { libc::shmctl(shmid, libc::IPC_RMID, std::ptr::null_mut()) }; + + // NonEmpty: one event each time the root goes from clean to damaged; re-armed in `wait`. + let damage = conn.generate_id().map_err(text)?; + conn.damage_create(damage, root, ReportLevel::NON_EMPTY).map_err(text)?.check().map_err(text)?; + conn.flush().map_err(text)?; + Ok(Self { conn, root, damage, damage_event: damage_info.first_event, width, height, seg, shm_ptr, len }) + } + + pub fn size(&self) -> (u32, u32) { (self.width as u32, self.height as u32) } + + /// Waits up to `timeout` for the screen to change. Returns `true` on change, `false` on timeout. + pub fn wait(&mut self, timeout: Duration) -> Result { + let deadline = Instant::now() + timeout; + loop { + let mut changed = false; + while let Some(event) = self.conn.poll_for_raw_event().map_err(text)? { + // XDamageNotify is the extension's first event; the wire code keeps the high bit for synthetic events. + if event[0] & 0x7f == self.damage_event { changed = true; } + } + if changed { + self.conn.damage_subtract(self.damage, NONE, NONE).map_err(text)?; + self.conn.flush().map_err(text)?; + return Ok(true); + } + let remaining = deadline.saturating_duration_since(Instant::now()); + if remaining.is_zero() { return Ok(false); } + let mut pfd = libc::pollfd { fd: self.conn.stream().as_raw_fd(), events: libc::POLLIN, revents: 0 }; + match unsafe { libc::poll(&mut pfd, 1, remaining.as_millis().max(1) as i32) } { + ready if ready < 0 => return Err("poll on the X connection failed".into()), + // Readable but not yet a whole event: poll_for_raw_event would return None and poll would + // fire again at once, spinning a core. Block for the event instead. + ready if ready > 0 => { + let event = self.conn.wait_for_raw_event().map_err(text)?; + if event[0] & 0x7f == self.damage_event { changed = true; } + if changed { + self.conn.damage_subtract(self.damage, NONE, NONE).map_err(text)?; + self.conn.flush().map_err(text)?; + return Ok(true); + } + } + _ => {} + } + } + } + + /// Copies the root into the shared segment and returns it as rows of `width * 4` bytes. + /// The slice is valid until the next call. + pub fn capture(&mut self) -> Result<&[u8], String> { + self.conn.shm_get_image(self.root, 0, 0, self.width, self.height, !0, Z_PIXMAP, self.seg, 0) + .map_err(text)?.reply().map_err(text)?; + Ok(unsafe { std::slice::from_raw_parts(self.shm_ptr, self.len) }) + } +} + +impl Drop for DamageCapture { + fn drop(&mut self) { + let _ = self.conn.damage_destroy(self.damage); + let _ = self.conn.shm_detach(self.seg); + let _ = self.conn.flush(); + unsafe { libc::shmdt(self.shm_ptr as *const libc::c_void) }; + } +} diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index 6c17052..05f326f 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -12,6 +12,8 @@ use ff::util::picture; use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational}; use tokio::sync::mpsc; +use crate::damage_capture::DamageCapture; + pub struct Config { pub display: String, pub input: (u32, u32), @@ -23,6 +25,8 @@ pub struct Config { pub max_bps: i64, pub buffer_bits: i64, pub gpu: bool, + /// Capture on XDamage changes (XShm) instead of x11grab; `UU_CAPTURE=grab` turns it off. + pub damage: bool, } /// `UU_VIDEO_PIPELINE=process` keeps the original ffmpeg child-process path. @@ -30,6 +34,11 @@ pub fn disabled() -> bool { std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process") } +/// Damage-driven capture is on unless `UU_CAPTURE=grab` asks for x11grab. +pub fn damage_enabled() -> bool { + std::env::var("UU_CAPTURE").map_or(true, |v| v != "grab") +} + /// In-process capture is used unless the process path is requested, or a bundled FFmpeg prefix /// (`UU_FFMPEG_ROOT` / config `ffmpeg_root`) is set: that build was chosen on purpose, so keep using it. pub fn selected() -> bool { @@ -62,24 +71,8 @@ fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender } } -/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR. -pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { - ensure_init()?; - let name = CString::new("x11grab").map_err(|e| e.to_string())?; - let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) }; - if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); } - let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) }); - let mut capture_options = Dictionary::new(); - capture_options.set("framerate", &config.fps.to_string()); - capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1)); - // The viewer draws its own pointer; do not bake a second one into the video. - capture_options.set("draw_mouse", "0"); - let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input(); - - let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?; - let index = stream.index(); - let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?; - +/// Opens the H.264 encoder for `config`. Returns the encoder, its input pixel format and output size. +fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), String> { let (width, height) = config.output.unwrap_or(config.input); let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" }) .ok_or("H.264 encoder is not available in this FFmpeg build")?; @@ -119,7 +112,35 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, options.set("crf", &cq); options.set("x264-params", &x264); } - let mut encoder = context.open_as_with(codec, options).map_err(text)?; + let encoder = context.open_as_with(codec, options).map_err(text)?; + Ok((encoder, pixel, width, height)) +} + +/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR. +pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { + ensure_init()?; + if config.damage { + match DamageCapture::open(&config.display) { + Ok(capture) => return run_damage(config, capture, units, stop, idr), + Err(error) => log::warn!("Damage-driven capture unavailable, using x11grab: {error}"), + } + } + let name = CString::new("x11grab").map_err(|e| e.to_string())?; + let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) }; + if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); } + let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) }); + let mut capture_options = Dictionary::new(); + capture_options.set("framerate", &config.fps.to_string()); + capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1)); + // The viewer draws its own pointer; do not bake a second one into the video. + capture_options.set("draw_mouse", "0"); + let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input(); + + let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?; + let index = stream.index(); + let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?; + + let (mut encoder, pixel, width, height) = open_encoder(&config)?; let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None; let mut decoded = frame::Video::empty(); @@ -169,6 +190,82 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, Ok(()) } +/// Damage-driven loop. Damage only marks the picture dirty; a dirty picture is captured and encoded on +/// the next fixed frame tick, so the stream keeps a steady cadence while the screen moves. The viewer's +/// jitter buffer grows with irregular arrivals, and irregular sends were what raised its latency. +/// An idle screen sends a keepalive about once a second so the viewer still sees traffic. +fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { + let (mut encoder, pixel, width, height) = open_encoder(&config)?; + let (cap_width, cap_height) = capture.size(); + let mut captured = frame::Video::new(Pixel::BGRZ, cap_width, cap_height); + let mut converted = frame::Video::new(pixel, width, height); + let mut scaler = None; + let mut packet = Packet::empty(); + let started = Instant::now(); + let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps)); + // Keepalive gaps are random between 15 and 25 fps: a steady 1 fps reads as a stall to the viewer. + let mut seed = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(1, |d| d.subsec_nanos()) | 1; + let mut keepalive = random_keepalive(&mut seed); + let mut dirty = true; + let mut next_tick = Instant::now(); + let mut last_send = Instant::now(); + let mut last_slot: i64 = -1; + loop { + if stop.load(Ordering::Relaxed) { return Ok(()); } + let deadline = if dirty { next_tick } else { last_send + keepalive }; + let wait = deadline.saturating_duration_since(Instant::now()); + if capture.wait(wait.max(Duration::from_millis(1)))? { + dirty = true; + continue; + } + let now = Instant::now(); + if dirty && now < next_tick { continue; } + if !dirty && now < last_send + keepalive { continue; } + + let source = capture.capture()?; + let bytes_per_line = cap_width as usize * 4; + let stride = captured.stride(0); + let row = bytes_per_line.min(stride); + for (y, src) in source.chunks_exact(bytes_per_line).enumerate() { + captured.data_mut(0)[y * stride..y * stride + row].copy_from_slice(&src[..row]); + } + + let slot = ((started.elapsed().as_micros() * u128::from(config.fps) + 500_000) / 1_000_000) as i64; + let slot = slot.max(last_slot + 1); + last_slot = slot; + + let frame = if pixel == Pixel::BGRZ && width == cap_width && height == cap_height { + &mut captured + } else { + if scaler.is_none() { + scaler = Some(scaling::Context::get(Pixel::BGRZ, cap_width, cap_height, pixel, width, height, scaling::Flags::BILINEAR).map_err(text)?); + } + if let Some(context) = scaler.as_mut() { context.run(&captured, &mut converted).map_err(text)?; } + &mut converted + }; + frame.set_pts(Some(slot)); + frame.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None }); + encoder.send_frame(frame).map_err(text)?; + if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); } + let now = Instant::now(); + if dirty { + // Keep the tick phase; after a long idle, restart the phase from now instead of bursting to catch up. + next_tick = (next_tick + frame_interval).max(now); + dirty = false; + } + last_send = now; + keepalive = random_keepalive(&mut seed); + } +} + +/// Picks the next keepalive gap in 40–67 ms (15–25 fps) with a small xorshift generator. +fn random_keepalive(seed: &mut u32) -> Duration { + *seed ^= *seed << 13; + *seed ^= *seed >> 17; + *seed ^= *seed << 5; + Duration::from_micros(40_000 + u64::from(*seed % 26_700)) +} + /// Encodes a PulseAudio/PipeWire source to Opus in this process. Each Opus packet is sent as is. pub fn run_audio(source: &str, units: mpsc::Sender, stop: Arc) -> Result<(), String> { ensure_init()?; diff --git a/src/main.rs b/src/main.rs index 024e25a..0bdb05c 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,8 @@ mod terminal; mod webrtc; mod video; #[cfg(feature = "inproc-av")] +mod damage_capture; +#[cfg(feature = "inproc-av")] mod inproc_encoder; mod capture_setting; mod audio; diff --git a/src/video.rs b/src/video.rs index 1cb9e1d..292edb3 100644 --- a/src/video.rs +++ b/src/video.rs @@ -179,6 +179,7 @@ async fn read_encoder_with( gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32, target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu, + damage: crate::inproc_encoder::damage_enabled(), }; log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"});