diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c677e38..d8b3b03 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev + libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked @@ -28,7 +28,7 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential pkg-config \ libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \ - libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev + libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked --features wayland,portal diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e8419c..3868597 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev + libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev - uses: dtolnay/rust-toolchain@stable - run: cargo test --release --locked - run: cargo build --release --locked diff --git a/AGENTS.md b/AGENTS.md index b3e8bba..654ff76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ Dependencies (Ubuntu/Debian): ```sh sudo apt install build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev ffmpeg + libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev ffmpeg # --features wayland: libwayland-dev libxkbcommon-dev wl-clipboard # --features wayland,portal: + libglib2.0-dev libpipewire-0.3-dev ``` diff --git a/Cargo.lock b/Cargo.lock index 39b2318..b69f63d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -212,6 +212,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 1.0.109", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -276,7 +296,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838" dependencies = [ "find-msvc-tools", - "shlex", + "shlex 2.0.1", ] [[package]] @@ -291,6 +311,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -324,6 +353,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -605,6 +645,31 @@ dependencies = [ "subtle", ] +[[package]] +name = "ffmpeg-next" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e72c72e8dcf638fb0fb03f033a954691662b5dabeaa3f85a6607d101569fccd" +dependencies = [ + "bitflags 1.3.2", + "ffmpeg-sys-next", + "libc", +] + +[[package]] +name = "ffmpeg-sys-next" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2529ad916d08c3562c754c21bc9b17a26c7882c0f5706cc2cd69472175f1620" +dependencies = [ + "bindgen", + "cc", + "libc", + "num_cpus", + "pkg-config", + "vcpkg", +] + [[package]] name = "fiat-crypto" version = "0.2.9" @@ -773,6 +838,12 @@ dependencies = [ "polyval", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "group" version = "0.13.0" @@ -784,6 +855,12 @@ dependencies = [ "subtle", ] +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + [[package]] name = "hex" version = "0.4.3" @@ -1111,12 +1188,28 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "litemap" version = "0.8.3" @@ -1265,6 +1358,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + [[package]] name = "oid-registry" version = "0.6.1" @@ -1339,6 +1442,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem" version = "3.0.6" @@ -1386,6 +1495,12 @@ dependencies = [ "spki", ] +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + [[package]] name = "polyval" version = "0.6.2" @@ -1472,7 +1587,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.3", "rustls 0.23.45", "socket2 0.6.5", "thiserror 2.0.21", @@ -1493,7 +1608,7 @@ dependencies = [ "rand 0.10.3", "rand_pcg", "ring 0.17.14", - "rustc-hash", + "rustc-hash 2.1.3", "rustls 0.23.45", "rustls-pki-types", "slab", @@ -1740,6 +1855,12 @@ dependencies = [ "webrtc-util", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.3" @@ -1958,6 +2079,12 @@ dependencies = [ "digest", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -2493,6 +2620,7 @@ dependencies = [ "bytes", "cc", "env_logger", + "ffmpeg-next", "flate2", "futures-util", "hex", @@ -2521,6 +2649,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" diff --git a/Cargo.toml b/Cargo.toml index 81322eb..44d10ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ nix = { version = "0.29", features = ["term", "process", "fs", "ioctl", "signal" log = "0.4" qrcode = { version = "0.14", default-features = false } env_logger = "0.11" +ffmpeg-next = { version = "6.1", default-features = false, features = ["codec", "format", "device", "software-scaling"] } [build-dependencies] cc = "1.0" diff --git a/README.en.md b/README.en.md index fe7e280..f62de0a 100644 --- a/README.en.md +++ b/README.en.md @@ -87,7 +87,7 @@ permission; the permission is remembered once granted. ```sh sudo apt install build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev \ + libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \ libgtk-3-dev ffmpeg # optional: Wayland backend sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard diff --git a/README.md b/README.md index 7ec682e..e1fd140 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Wayland 下的限制:只支持单屏、固定分辨率;portal 后端暂不 ```sh sudo apt install build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev \ + libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \ libgtk-3-dev ffmpeg # 可选:Wayland 后端 sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs new file mode 100644 index 0000000..3734c9d --- /dev/null +++ b/src/inproc_encoder.rs @@ -0,0 +1,155 @@ +//! In-process X11 capture and H.264 encoding through libavdevice/libavcodec. +//! It replaces the `ffmpeg` child process with the same codec options, so the viewer +//! still receives the same Annex-B access units (AUD, SPS/PPS, IDR, P). +use std::ffi::CString; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Once}; +use std::time::{Duration, Instant}; +use ffmpeg_next as ff; +use ff::format::{format::Format, Pixel}; +use ff::util::picture; +use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational}; +use tokio::sync::mpsc; + +pub struct Config { + pub display: String, + pub input: (u32, u32), + pub output: Option<(u32, u32)>, + pub fps: u32, + pub gop: u32, + pub cq: u32, + pub target_bps: i64, + pub max_bps: i64, + pub buffer_bits: i64, + pub gpu: bool, +} + +/// `UU_VIDEO_PIPELINE=process` keeps the original ffmpeg child-process path. +pub fn disabled() -> bool { + std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process") +} + +fn text(error: ff::Error) -> String { error.to_string() } + +fn ensure_init() -> Result<(), String> { + static INIT: Once = Once::new(); + let mut result = Ok(()); + INIT.call_once(|| { + if let Err(error) = ff::init() { result = Err(text(error)); } + ff::device::register_all(); + }); + result +} + +/// Sends one access unit per encoder packet. Returns `Ok(false)` when the receiver is gone. +fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender>) -> Result { + loop { + match encoder.receive_packet(packet) { + Ok(()) => { + let data = packet.data().unwrap_or_default().to_vec(); + if units.blocking_send(data).is_err() { return Ok(false); } + } + Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => return Ok(true), + Err(error) => return Err(text(error)), + } + } +} + +/// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR. +pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { + ensure_init()?; + let name = CString::new("x11grab").map_err(|e| e.to_string())?; + let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) }; + if raw.is_null() { return Err("x11grab input is not available in this FFmpeg build".into()); } + let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) }); + let mut capture_options = Dictionary::new(); + capture_options.set("framerate", &config.fps.to_string()); + capture_options.set("video_size", &format!("{}x{}", config.input.0, config.input.1)); + // The viewer draws its own pointer; do not bake a second one into the video. + capture_options.set("draw_mouse", "0"); + let mut input = ff::format::open_with(&config.display, &input_format, capture_options).map_err(text)?.input(); + + let stream = input.streams().best(ff::media::Type::Video).ok_or("X11 capture has no video stream")?; + let index = stream.index(); + let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().video().map_err(text)?; + + let (width, height) = config.output.unwrap_or(config.input); + let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" }) + .ok_or("H.264 encoder is not available in this FFmpeg build")?; + // Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one. + let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?; + context.set_width(width); + context.set_height(height); + context.set_format(Pixel::YUV420P); + context.set_time_base(Rational::new(1, config.fps as i32)); + context.set_frame_rate(Some(Rational::new(config.fps as i32, 1))); + context.set_gop(config.gop); + context.set_max_b_frames(0); + context.set_bit_rate(config.target_bps as usize); + context.set_max_bit_rate(config.max_bps as usize); + // The CLI's -maxrate/-bufsize map to these raw fields, which the encoder wrappers read. + unsafe { + let raw = context.as_mut_ptr(); + (*raw).rc_max_rate = config.max_bps; + (*raw).rc_buffer_size = config.buffer_bits as i32; + } + let cq = config.cq.to_string(); + let gop = config.gop.to_string(); + let mut options = Dictionary::new(); + if config.gpu { + for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"), + ("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] { + options.set(key, value); + } + options.set("cq", &cq); + } else { + let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}"); + for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "baseline"), ("threads", "4")] { + options.set(key, value); + } + options.set("crf", &cq); + options.set("x264-params", &x264); + } + let mut encoder = context.open_as_with(codec, options).map_err(text)?; + + let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None; + let mut decoded = frame::Video::empty(); + let mut yuv = frame::Video::new(Pixel::YUV420P, width, height); + let mut packet = Packet::empty(); + let started = Instant::now(); + let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps)); + let mut last_at: Option = None; + let mut last_slot: i64 = -1; + for (stream, input_packet) in input.packets() { + if stop.load(Ordering::Relaxed) { return Ok(()); } + if stream.index() != index { continue; } + decoder.send_packet(&input_packet).map_err(text)?; + match decoder.receive_frame(&mut decoded) { + Ok(()) => {} + Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => continue, + Err(error) => return Err(text(error)), + } + // x11grab catches up after a stall by grabbing back-to-back. A frame arriving within half + // a frame interval of the previous one is part of that burst and is dropped, as the old + // `-fps_mode vfr` did. Normal jitter stays well above that threshold. + let now = started.elapsed(); + if last_at.is_some_and(|previous| now - previous < frame_interval / 2) { continue; } + last_at = Some(now); + let slot = ((now.as_micros() * u128::from(config.fps) + 500_000) / 1_000_000) as i64; + let slot = slot.max(last_slot + 1); + last_slot = slot; + + let key = (decoded.format(), decoded.width(), decoded.height()); + if scaler.as_ref().map(|(_, k)| *k) != Some(key) { + let context = scaling::Context::get(key.0, key.1, key.2, Pixel::YUV420P, width, height, scaling::Flags::BILINEAR).map_err(text)?; + scaler = Some((context, key)); + } + if let Some((context, _)) = scaler.as_mut() { context.run(&decoded, &mut yuv).map_err(text)?; } + + yuv.set_pts(Some(slot)); + yuv.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None }); + encoder.send_frame(&yuv).map_err(text)?; + if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); } + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 4657a06..0d83dd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ mod signaling; mod terminal; mod webrtc; mod video; +mod inproc_encoder; mod capture_setting; mod audio; mod system_actions; diff --git a/src/video.rs b/src/video.rs index 0ad1c3c..5c9e70a 100644 --- a/src/video.rs +++ b/src/video.rs @@ -1,4 +1,5 @@ use std::sync::Arc; +use std::sync::atomic::AtomicBool; use std::time::{Duration, Instant}; use tokio::io::AsyncReadExt; use tokio::process::Command; @@ -99,23 +100,51 @@ impl EncoderTask { } } +struct StopOnDrop(Arc); +impl Drop for StopOnDrop { fn drop(&mut self) { self.0.store(true, std::sync::atomic::Ordering::Relaxed); } } + async fn read_encoder( size: (u32, u32), display: String, portal:Option>, latest: mpsc::Sender>, - settings: VideoSettings, sequence: Arc, + settings: VideoSettings, sequence: Arc, idr: Arc, ) -> Result<(), String> { // The sequence is shared so a CPU fallback or settings restart continues it (skip statistics stay correct). - if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings).await; } - match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings).await { + if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await; } + match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings,idr.clone()).await { Ok(())=>Ok(()), Err(error) if !latest.is_closed()=>{ log::warn!("NVENC stream unavailable; switching scoped encoder to CPU: {}",error); - read_encoder_with(size,display,portal,latest,false,sequence,settings).await + read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await }, Err(error)=>Err(error), } } +/// Capture and encode inside this process; each packet is one access unit. +async fn read_inprocess( + config: crate::inproc_encoder::Config, latest: mpsc::Sender>, + sequence: Arc, idr: Arc, +) -> Result<(), String> { + let (units_tx, mut units) = mpsc::channel::>(4); + let stop = Arc::new(AtomicBool::new(false)); + // If this future is dropped (reader task aborted), the blocking encoder stops at its next packet. + let _stop_on_drop = StopOnDrop(stop.clone()); + let job = tokio::task::spawn_blocking({ + let stop = stop.clone(); + move || crate::inproc_encoder::run(config, units_tx, stop, idr) + }); + let mut gate = DependencyGate::default(); + while let Some(data) = units.recv().await { + let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + gate.submit(&latest, Arc::new(CompleteFrame { sequence, completed_at: Instant::now(), data })); + if latest.is_closed() { break; } + } + // Dropping the receiver also unblocks the encoder if it is waiting to send. + stop.store(true, std::sync::atomic::Ordering::Relaxed); + drop(units); + job.await.map_err(|e| format!("Encoder task: {e}"))? +} + /// Rate-control arguments: constant quality capped at 30M (自动), or capped at the viewer's /// chosen bitrate. The VBV buffer stays at most 500k so a cap never adds queueing latency. fn rate_args(settings: &VideoSettings) -> (String, String) { @@ -126,10 +155,22 @@ fn rate_args(settings: &VideoSettings) -> (String, String) { async fn read_encoder_with( size: (u32,u32),display:String,portal:Option>,latest:mpsc::Sender>,gpu:bool,sequence:Arc, - mut settings: VideoSettings, + mut settings: VideoSettings, idr: Arc, )->Result<(),String>{ - let size = format!("{}x{}", size.0, size.1); let wayland=display.starts_with("wayland:"); + if !wayland && portal.is_none() && !crate::inproc_encoder::disabled() { + let max_kbps = settings.max_kbps().unwrap_or(30_000) as i64; + let config = crate::inproc_encoder::Config { + display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps), + gop: (u64::from(settings.fps) * GOP / FPS).max(1) as u32, cq: settings.cq() as u32, + target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, + max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu, + }; + log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", + display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"}); + return read_inprocess(config, latest, sequence, idr).await; + } + let size = format!("{}x{}", size.0, size.1); // The separate wayland-capture process produces frames at its own fixed rate. if wayland && portal.is_none() { settings.fps = FPS as u32; } let fps = settings.fps.to_string(); @@ -256,11 +297,13 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option, track: Arc, size: (u32, u32), display: String, portal:Option>, - mut settings: tokio::sync::watch::Receiver) -> Result<(), String> { + mut settings: tokio::sync::watch::Receiver, idr: Arc) -> Result<(), String> { loop { if transport_closed(&pc) { return Ok(()); } if pc.connection_state() == RTCPeerConnectionState::Connected @@ -284,7 +327,7 @@ pub async fn stream_desktop(pc: Arc, track: Arc, track: Arc { // A closed queue means the reader ended; its JoinHandle carries the reason. diff --git a/src/webrtc.rs b/src/webrtc.rs index f3842cb..b9810f2 100644 --- a/src/webrtc.rs +++ b/src/webrtc.rs @@ -184,8 +184,11 @@ impl WebRtcSession { "video_0".to_owned(), )); + // Set by PLI/FIR; the desktop encoder answers with an IDR on its next frame. + let idr_request = Arc::new(std::sync::atomic::AtomicBool::new(false)); let sender = pc.add_track(video_track.clone()).await .map_err(|e| format!("Add video track failed: {}", e))?; + let idr_on_feedback = idr_request.clone(); tokio::spawn(async move { let mut feedback = [0u8; 1500]; let mut reports = 0usize; @@ -193,6 +196,10 @@ impl WebRtcSession { while let Ok((packets, _)) = sender.read(&mut feedback).await { for packet in packets { feedback_counts.observe(packet.as_any()); + if packet.as_any().is::() + || packet.as_any().is::() { + idr_on_feedback.store(true, std::sync::atomic::Ordering::Release); + } if let Some(report) = packet.as_any().downcast_ref::() { reports += 1; if reports == 1 || reports % 10 == 0 { @@ -261,7 +268,7 @@ impl WebRtcSession { // File/terminal/port-mapping connections carry no desktop video. if desktop { tokio::spawn(async move { - if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx).await { + if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, idr_request).await { log::error!("Live desktop stream stopped: {}", e); } });