From 2355790e178e41dd82aecbbc4e7344c25c73c402 Mon Sep 17 00:00:00 2001 From: ferstar Date: Thu, 15 Oct 2026 03:22:35 +0800 Subject: [PATCH] Make in-process capture and encoding an opt-in feature Address PR review: - The FFmpeg 6 bindings (AVChannelLayout, ffmpeg-next 6.1) do not build on the 22.04 runners, which ship FFmpeg 4.4. The default build no longer depends on libav at all, so `cargo test --locked` and the release job on 22.04 no longer compile that code. - The release tarball no longer links the build machine's libav SONAMEs. Default builds use the FFmpeg child process for video and audio again. - `UU_FFMPEG_ROOT` / config `ffmpeg_root` now selects the child-process path even in an inproc-av build, so a bundled FFmpeg with working NVENC is not silently bypassed. Build with `--features inproc-av` (FFmpeg 6 dev packages, Ubuntu 24.04) to get in-process capture, encoding and Opus. CI has a test-inproc job for it. `UU_VIDEO_PIPELINE=process` still forces the child path for video. Probes on ros2go-gpu (440 Hz tone, 5 s): default (ffmpeg child): 44.0 packets/s, 132 kbps, max gap 43 ms inproc-av: 49.7 packets/s, 133.5 kbps, max gap 22 ms desktop-video-probe under inproc-av with UU_FFMPEG_ROOT=/usr uses the child path (no `pipeline=in-process` in the log); without it, in-process. --- .github/workflows/ci.yml | 18 +++++++- .github/workflows/release.yml | 2 +- AGENTS.md | 2 + Cargo.toml | 5 ++- README.en.md | 2 +- README.md | 2 +- src/audio.rs | 85 ++++++++++++++++++++++++++++------- src/inproc_encoder.rs | 6 +++ src/main.rs | 1 + src/video.rs | 8 +++- 10 files changed, 106 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8b3b03..b802939 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev + libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked @@ -28,7 +28,21 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential pkg-config \ libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \ - libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev + libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked --features wayland,portal + + test-inproc: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + - name: Install build dependencies (X11 + FFmpeg 6 for the inproc-av feature) + run: | + sudo apt-get update + sudo apt-get install -y build-essential pkg-config \ + libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \ + libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + - run: cargo test --locked --features inproc-av diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3868597..6e8419c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev + libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev - uses: dtolnay/rust-toolchain@stable - run: cargo test --release --locked - run: cargo build --release --locked diff --git a/AGENTS.md b/AGENTS.md index 654ff76..e58b361 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,8 @@ Dependencies (Ubuntu/Debian): ```sh sudo apt install build-essential pkg-config \ libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev ffmpeg +# --features inproc-av: libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev +# (FFmpeg 6.x; in-process capture/encoding, Ubuntu 24.04 ships it) # --features wayland: libwayland-dev libxkbcommon-dev wl-clipboard # --features wayland,portal: + libglib2.0-dev libpipewire-0.3-dev ``` diff --git a/Cargo.toml b/Cargo.toml index 4400774..c0c2061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,9 @@ path = "src/main.rs" default = [] wayland = [] portal = [] +# In-process capture and encoding through FFmpeg 6 libraries (libavcodec/libavdevice dev packages). +# Off by default so release binaries do not link the build machine's libav. +inproc-av = ["dep:ffmpeg-next"] [dependencies] tokio = { version = "1", features = ["full"] } @@ -37,7 +40,7 @@ nix = { version = "0.29", features = ["term", "process", "fs", "ioctl", "signal" log = "0.4" qrcode = { version = "0.14", default-features = false } env_logger = "0.11" -ffmpeg-next = { version = "6.1", default-features = false, features = ["codec", "format", "device", "software-scaling", "software-resampling"] } +ffmpeg-next = { version = "6.1", optional = true, default-features = false, features = ["codec", "format", "device", "software-scaling", "software-resampling"] } [build-dependencies] cc = "1.0" diff --git a/README.en.md b/README.en.md index f62de0a..fe7e280 100644 --- a/README.en.md +++ b/README.en.md @@ -87,7 +87,7 @@ permission; the permission is remembered once granted. ```sh sudo apt install build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \ + libx11-dev libxext-dev libxtst-dev libxfixes-dev \ libgtk-3-dev ffmpeg # optional: Wayland backend sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard diff --git a/README.md b/README.md index e1fd140..7ec682e 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ Wayland 下的限制:只支持单屏、固定分辨率;portal 后端暂不 ```sh sudo apt install build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev \ + libx11-dev libxext-dev libxtst-dev libxfixes-dev \ libgtk-3-dev ffmpeg # 可选:Wayland 后端 sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard diff --git a/src/audio.rs b/src/audio.rs index 1c654ea..a848917 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -1,11 +1,16 @@ //! Desktop sound for the viewer's 声音 menu: the default output's monitor (PulseAudio or -//! PipeWire-pulse) is encoded to Opus in this process and sent on the negotiated audio m-line. +//! PipeWire-pulse) is encoded to Opus and sent on the negotiated audio m-line. +//! Encoding runs in this process with the `inproc-av` feature (unless `UU_FFMPEG_ROOT` is set), +//! otherwise an FFmpeg child process does it. //! PipeWire monitors ignore the sink's mute by default, so 被控端静音运行 keeps the stream audible. //! `UU_AUDIO=0` disables it; `UU_AUDIO_SOURCE` picks another Pulse source. +use std::io::Read; +use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; +use webrtc::media::io::ogg_reader::OggReader; use webrtc::media::Sample; use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; use webrtc::peer_connection::RTCPeerConnection; @@ -16,6 +21,56 @@ const FRAME: Duration = Duration::from_millis(20); pub fn enabled() -> bool { std::env::var("UU_AUDIO").map_or(true, |v| v != "0") } +/// Stops one running encoder when dropped: sets the in-process stop flag, or kills the FFmpeg child. +struct Stopper { stop: Arc, child: Option } +impl Drop for Stopper { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(child) = self.child.as_mut() { let _ = child.kill(); let _ = child.wait(); } + } +} + +fn spawn_encoder(source: &str) -> Result { + let mut command = match crate::config::ffmpeg_root() { + Some(root) => { let mut c = Command::new(root.join("bin/ffmpeg")); c.env("LD_LIBRARY_PATH", root.join("lib/x86_64-linux-gnu")); c } + None => Command::new("ffmpeg"), + }; + command.args(["-hide_banner", "-loglevel", "error", "-nostdin", + "-f", "pulse", "-fragment_size", "1920", "-i", source, "-vn", + "-c:a", "libopus", "-ar", "48000", "-ac", "2", "-b:a", "128k", "-application", "lowdelay", + "-frame_duration", "20", "-page_duration", "20000", "-flush_packets", "1", "-f", "ogg", "pipe:1"]) + .stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); + command.spawn().map_err(|e| format!("Start audio encoder: {e}")) +} + +/// Reads Opus packets from the FFmpeg child's Ogg output until it ends or `tx` closes. Blocking. +fn read_packets(stdout: impl Read, tx: &tokio::sync::mpsc::Sender) -> Result<(), String> { + let (mut reader, _) = OggReader::new(stdout, false).map_err(|e| format!("Audio Ogg header: {e}"))?; + loop { + let (payload, _) = reader.parse_next_page().map_err(|e| format!("Audio Ogg page: {e}"))?; + if payload.starts_with(b"OpusTags") { continue; } + if tx.blocking_send(payload.freeze()).is_err() { return Ok(()); } + } +} + +/// Starts one encoder feeding `tx`. The job ends with its result once the stopper is dropped or `tx` closes. +fn start(source: &str, tx: tokio::sync::mpsc::Sender) + -> Result<(Stopper, tokio::task::JoinHandle>), String> { + let stop = Arc::new(AtomicBool::new(false)); + #[cfg(feature = "inproc-av")] + if crate::inproc_encoder::selected() { + let job = tokio::task::spawn_blocking({ + let (stop, source) = (stop.clone(), source.to_owned()); + move || crate::inproc_encoder::run_audio(&source, tx, stop) + }); + return Ok((Stopper { stop, child: None }, job)); + } + let mut child = spawn_encoder(source)?; + let stdout = child.stdout.take().ok_or("Encoder stdout missing")?; + let job = tokio::task::spawn_blocking(move || read_packets(stdout, &tx)); + Ok((Stopper { stop, child: Some(child) }, job)) +} + fn closed(pc: &RTCPeerConnection) -> bool { matches!(pc.connection_state(), RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) } @@ -30,11 +85,9 @@ pub async fn stream_audio(pc: Arc, track: Arc(25); - let stop = Arc::new(AtomicBool::new(false)); - let reader = { - let stop = stop.clone(); - let source = source.clone(); - tokio::task::spawn_blocking(move || crate::inproc_encoder::run_audio(&source, tx, stop)) + let (stopper, job) = match start(&source, tx) { + Ok(started) => started, + Err(e) => { log::warn!("{e}"); return; } }; log::info!("Desktop audio started: opus 48k stereo"); let mut packets = 0u64; @@ -51,10 +104,10 @@ pub async fn stream_audio(pc: Arc, track: Arc if closed(&pc) { break }, } } - // Stopping before dropping the receiver lets the encoder thread exit at its next packet. - stop.store(true, Ordering::Relaxed); + // Dropping the receiver and the stopper ends the encoder at its next packet. drop(rx); - let result = reader.await.unwrap_or_else(|e| Err(e.to_string())); + drop(stopper); + let result = job.await.unwrap_or_else(|e| Err(e.to_string())); if closed(&pc) { break; } log::warn!("Desktop audio stopped after {packets} packets: {:?}", result.err()); failures = if packets > 250 { 0 } else { failures + 1 }; @@ -68,11 +121,7 @@ pub async fn stream_audio(pc: Arc, track: Arc Result<(), String> { let source = std::env::var("UU_AUDIO_SOURCE").unwrap_or_else(|_| "@DEFAULT_MONITOR@".into()); let (tx, mut rx) = tokio::sync::mpsc::channel::(256); - let stop = Arc::new(AtomicBool::new(false)); - let worker = { - let stop = stop.clone(); - std::thread::spawn(move || crate::inproc_encoder::run_audio(&source, tx, stop)) - }; + let (stopper, job) = start(&source, tx)?; let started = std::time::Instant::now(); let mut packets = 0u64; let mut bytes = 0u64; @@ -90,10 +139,12 @@ pub async fn probe(seconds: u64) -> Result<(), String> { _ => break, } } - stop.store(true, Ordering::Relaxed); - drop(rx); let elapsed = started.elapsed().as_secs_f64(); + drop(rx); + drop(stopper); println!("Desktop audio probe: packets={packets} bytes={bytes} kbps={:.1} packets_per_s={:.1} max_gap_ms={}", bytes as f64 * 8.0 / 1000.0 / elapsed, packets as f64 / elapsed, max_gap.as_millis()); - worker.join().map_err(|_| "audio worker panicked".to_string())? + // The encoder is stopped on purpose, so its end-of-stream error only matters if nothing arrived. + let result = job.await.map_err(|e| e.to_string())?; + if packets == 0 { result } else { Ok(()) } } diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index f2813fa..23427a6 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -30,6 +30,12 @@ pub fn disabled() -> bool { std::env::var("UU_VIDEO_PIPELINE").is_ok_and(|v| v == "process") } +/// In-process capture is used unless the process path is requested, or a bundled FFmpeg prefix +/// (`UU_FFMPEG_ROOT` / config `ffmpeg_root`) is set: that build was chosen on purpose, so keep using it. +pub fn selected() -> bool { + !disabled() && crate::config::ffmpeg_root().is_none() +} + fn text(error: ff::Error) -> String { error.to_string() } fn ensure_init() -> Result<(), String> { diff --git a/src/main.rs b/src/main.rs index 5e5394e..2816d5a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -21,6 +21,7 @@ mod signaling; mod terminal; mod webrtc; mod video; +#[cfg(feature = "inproc-av")] mod inproc_encoder; mod capture_setting; mod audio; diff --git a/src/video.rs b/src/video.rs index 1a0184a..604c1b5 100644 --- a/src/video.rs +++ b/src/video.rs @@ -110,7 +110,9 @@ impl EncoderTask { } } +#[cfg(feature = "inproc-av")] struct StopOnDrop(Arc); +#[cfg(feature = "inproc-av")] impl Drop for StopOnDrop { fn drop(&mut self) { self.0.store(true, std::sync::atomic::Ordering::Relaxed); } } async fn read_encoder( @@ -131,6 +133,7 @@ async fn read_encoder( } /// Capture and encode inside this process; each packet is one access unit. +#[cfg(feature = "inproc-av")] async fn read_inprocess( config: crate::inproc_encoder::Config, latest: mpsc::Sender>, sequence: Arc, idr: Arc, @@ -165,10 +168,11 @@ fn rate_args(settings: &VideoSettings) -> (String, String) { async fn read_encoder_with( size: (u32,u32),display:String,portal:Option>,latest:mpsc::Sender>,gpu:bool,sequence:Arc, - mut settings: VideoSettings, idr: Arc, + mut settings: VideoSettings, #[cfg_attr(not(feature = "inproc-av"), allow(unused_variables))] idr: Arc, )->Result<(),String>{ let wayland=display.starts_with("wayland:"); - if !wayland && portal.is_none() && !crate::inproc_encoder::disabled() { + #[cfg(feature = "inproc-av")] + if !wayland && portal.is_none() && crate::inproc_encoder::selected() { let max_kbps = settings.max_kbps().unwrap_or(30_000) as i64; let config = crate::inproc_encoder::Config { display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps),