diff --git a/Cargo.toml b/Cargo.toml index 44d10ab..4400774 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ nix = { version = "0.29", features = ["term", "process", "fs", "ioctl", "signal" log = "0.4" qrcode = { version = "0.14", default-features = false } env_logger = "0.11" -ffmpeg-next = { version = "6.1", default-features = false, features = ["codec", "format", "device", "software-scaling"] } +ffmpeg-next = { version = "6.1", default-features = false, features = ["codec", "format", "device", "software-scaling", "software-resampling"] } [build-dependencies] cc = "1.0" diff --git a/src/audio.rs b/src/audio.rs index e9ffebc..1c654ea 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -1,50 +1,21 @@ //! Desktop sound for the viewer's 声音 menu: the default output's monitor (PulseAudio or -//! PipeWire-pulse) is encoded to Opus by FFmpeg and sent on the negotiated audio m-line. +//! PipeWire-pulse) is encoded to Opus in this process and sent on the negotiated audio m-line. //! PipeWire monitors ignore the sink's mute by default, so 被控端静音运行 keeps the stream audible. //! `UU_AUDIO=0` disables it; `UU_AUDIO_SOURCE` picks another Pulse source. -use std::io::Read; -use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; -use webrtc::media::io::ogg_reader::OggReader; use webrtc::media::Sample; use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; use webrtc::peer_connection::RTCPeerConnection; use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; -/// One Opus packet per Ogg page (FFmpeg -page_duration), 20 ms each. +/// One Opus packet per 20 ms frame. const FRAME: Duration = Duration::from_millis(20); pub fn enabled() -> bool { std::env::var("UU_AUDIO").map_or(true, |v| v != "0") } -struct Encoder(Child); -impl Drop for Encoder { fn drop(&mut self) { let _ = self.0.kill(); let _ = self.0.wait(); } } - -fn spawn_encoder() -> Result { - let source = std::env::var("UU_AUDIO_SOURCE").unwrap_or_else(|_| "@DEFAULT_MONITOR@".into()); - let mut command = match crate::config::ffmpeg_root() { - Some(root) => { let mut c = Command::new(root.join("bin/ffmpeg")); c.env("LD_LIBRARY_PATH", root.join("lib/x86_64-linux-gnu")); c } - None => Command::new("ffmpeg"), - }; - command.args(["-hide_banner", "-loglevel", "error", "-nostdin", - "-f", "pulse", "-fragment_size", "1920", "-i", &source, "-vn", - "-c:a", "libopus", "-ar", "48000", "-ac", "2", "-b:a", "128k", "-application", "lowdelay", - "-frame_duration", "20", "-page_duration", "20000", "-flush_packets", "1", "-f", "ogg", "pipe:1"]) - .stdin(Stdio::null()).stdout(Stdio::piped()).stderr(Stdio::inherit()); - command.spawn().map(Encoder).map_err(|e| format!("Start audio encoder: {e}")) -} - -/// Reads Opus packets until the encoder ends or `tx` closes. Blocking. -fn read_packets(stdout: impl Read, tx: &tokio::sync::mpsc::Sender) -> Result<(), String> { - let (mut reader, _) = OggReader::new(stdout, false).map_err(|e| format!("Audio Ogg header: {e}"))?; - loop { - let (payload, _) = reader.parse_next_page().map_err(|e| format!("Audio Ogg page: {e}"))?; - if payload.starts_with(b"OpusTags") { continue; } - if tx.blocking_send(payload.freeze()).is_err() { return Ok(()); } - } -} - fn closed(pc: &RTCPeerConnection) -> bool { matches!(pc.connection_state(), RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) } @@ -55,15 +26,16 @@ pub async fn stream_audio(pc: Arc, track: Arc encoder, - Err(e) => { log::warn!("{e}"); return; } - }; - let Some(stdout) = encoder.0.stdout.take() else { return }; let (tx, mut rx) = tokio::sync::mpsc::channel::(25); - let reader = tokio::task::spawn_blocking(move || read_packets(stdout, &tx)); + let stop = Arc::new(AtomicBool::new(false)); + let reader = { + let stop = stop.clone(); + let source = source.clone(); + tokio::task::spawn_blocking(move || crate::inproc_encoder::run_audio(&source, tx, stop)) + }; log::info!("Desktop audio started: opus 48k stereo"); let mut packets = 0u64; let mut health = tokio::time::interval(Duration::from_millis(500)); @@ -79,8 +51,9 @@ pub async fn stream_audio(pc: Arc, track: Arc if closed(&pc) { break }, } } + // Stopping before dropping the receiver lets the encoder thread exit at its next packet. + stop.store(true, Ordering::Relaxed); drop(rx); - drop(encoder); // kills FFmpeg, which ends the blocking reader let result = reader.await.unwrap_or_else(|e| Err(e.to_string())); if closed(&pc) { break; } log::warn!("Desktop audio stopped after {packets} packets: {:?}", result.err()); @@ -90,3 +63,37 @@ pub async fn stream_audio(pc: Arc, track: Arc Result<(), String> { + let source = std::env::var("UU_AUDIO_SOURCE").unwrap_or_else(|_| "@DEFAULT_MONITOR@".into()); + let (tx, mut rx) = tokio::sync::mpsc::channel::(256); + let stop = Arc::new(AtomicBool::new(false)); + let worker = { + let stop = stop.clone(); + std::thread::spawn(move || crate::inproc_encoder::run_audio(&source, tx, stop)) + }; + let started = std::time::Instant::now(); + let mut packets = 0u64; + let mut bytes = 0u64; + let mut max_gap = Duration::ZERO; + let mut last = None::; + while started.elapsed() < Duration::from_secs(seconds) { + match tokio::time::timeout(Duration::from_secs(2), rx.recv()).await { + Ok(Some(data)) => { + let now = std::time::Instant::now(); + if let Some(prev) = last { max_gap = max_gap.max(now - prev); } + last = Some(now); + packets += 1; + bytes += data.len() as u64; + } + _ => break, + } + } + stop.store(true, Ordering::Relaxed); + drop(rx); + let elapsed = started.elapsed().as_secs_f64(); + println!("Desktop audio probe: packets={packets} bytes={bytes} kbps={:.1} packets_per_s={:.1} max_gap_ms={}", + bytes as f64 * 8.0 / 1000.0 / elapsed, packets as f64 / elapsed, max_gap.as_millis()); + worker.join().map_err(|_| "audio worker panicked".to_string())? +} diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index 3734c9d..f2813fa 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -6,7 +6,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Once}; use std::time::{Duration, Instant}; use ffmpeg_next as ff; -use ff::format::{format::Format, Pixel}; +use ff::format::{format::Format, sample, Pixel, Sample}; +use ff::ChannelLayout; use ff::util::picture; use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational}; use tokio::sync::mpsc; @@ -153,3 +154,90 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, } Ok(()) } + +/// Encodes a PulseAudio/PipeWire source to Opus in this process. Each Opus packet is sent as is. +pub fn run_audio(source: &str, units: mpsc::Sender, stop: Arc) -> Result<(), String> { + ensure_init()?; + let name = CString::new("pulse").map_err(|e| e.to_string())?; + let raw = unsafe { ff::sys::av_find_input_format(name.as_ptr()) }; + if raw.is_null() { return Err("pulse input is not available in this FFmpeg build".into()); } + let input_format = Format::Input(unsafe { ff::format::format::Input::wrap(raw as *mut _) }); + let mut capture_options = Dictionary::new(); + capture_options.set("fragment_size", "1920"); + let mut input = ff::format::open_with(&source, &input_format, capture_options).map_err(text)?.input(); + + let stream = input.streams().best(ff::media::Type::Audio).ok_or("Pulse source has no audio stream")?; + let index = stream.index(); + let mut decoder = codec::context::Context::from_parameters(stream.parameters()).map_err(text)?.decoder().audio().map_err(text)?; + + let codec = encoder::find_by_name("libopus").ok_or("Opus encoder is not available in this FFmpeg build")?; + let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().audio().map_err(text)?; + context.set_rate(48_000); + context.set_format(Sample::I16(sample::Type::Packed)); + // FFmpeg 6 reads the AVChannelLayout field; the deprecated channel_layout alone makes libopus fail with EINVAL. + unsafe { ff::sys::av_channel_layout_default(&mut (*context.as_mut_ptr()).ch_layout, 2) }; + context.set_time_base(Rational::new(1, 48_000)); + context.set_bit_rate(128_000); + let mut options = Dictionary::new(); + options.set("application", "lowdelay"); + options.set("frame_duration", "20"); + let mut encoder = context.open_as_with(codec, options).map_err(text)?; + // libopus takes fixed-size frames: 960 samples per channel for 20 ms. + let frame_samples = match encoder.frame_size() as usize { 0 => 960, n => n }; + + let mut resampler = None; + let mut decoded = frame::Audio::empty(); + let mut converted = frame::Audio::empty(); + let mut fifo: Vec = Vec::new(); + let mut pts: i64 = 0; + let mut packet = Packet::empty(); + for (stream, input_packet) in input.packets() { + if stop.load(Ordering::Relaxed) { return Ok(()); } + if stream.index() != index { continue; } + decoder.send_packet(&input_packet).map_err(text)?; + loop { + match decoder.receive_frame(&mut decoded) { + Ok(()) => {} + Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => break, + Err(error) => return Err(text(error)), + } + // The decoder may leave the deprecated channel_layout unset; derive it from the channel count. + let source_layout = ChannelLayout::default(i32::from(decoded.channels())); + if resampler.is_none() { + resampler = Some(ff::software::resampling::Context::get( + decoded.format(), source_layout, decoded.rate(), + Sample::I16(sample::Type::Packed), ChannelLayout::STEREO, 48_000, + ).map_err(text)?); + } + // Keep both layout fields of the decoded frame in step with the resampler's input layout. + decoded.set_channel_layout(source_layout); + unsafe { ff::sys::av_channel_layout_default(&mut (*decoded.as_mut_ptr()).ch_layout, i32::from(decoded.channels())) }; + if let Some(resampler) = resampler.as_mut() { resampler.run(&decoded, &mut converted).map_err(text)?; } + let samples = converted.samples() * 2; + fifo.extend(converted.data(0)[..samples * 2].chunks_exact(2).map(|b| i16::from_ne_bytes([b[0], b[1]]))); + + while fifo.len() >= frame_samples * 2 { + let mut out = frame::Audio::new(Sample::I16(sample::Type::Packed), frame_samples, ChannelLayout::STEREO); + unsafe { ff::sys::av_channel_layout_default(&mut (*out.as_mut_ptr()).ch_layout, 2) }; + out.set_rate(48_000); + out.set_pts(Some(pts)); + for (dst, src) in out.data_mut(0)[..frame_samples * 4].chunks_exact_mut(2).zip(fifo.drain(..frame_samples * 2)) { + dst.copy_from_slice(&src.to_ne_bytes()); + } + pts += frame_samples as i64; + encoder.send_frame(&out).map_err(text)?; + loop { + match encoder.receive_packet(&mut packet) { + Ok(()) => { + let data = bytes::Bytes::copy_from_slice(packet.data().unwrap_or_default()); + if units.blocking_send(data).is_err() { return Ok(()); } + } + Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => break, + Err(error) => return Err(text(error)), + } + } + } + } + } + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 0d83dd6..5e5394e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -51,7 +51,7 @@ use webrtc::WebRtcSession; async fn main() -> Result<(), Box> { env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); // An unknown argument must never fall through to starting a second full host server. - const SUBCOMMANDS: [&str; 8] = ["login", "unlock-check", "desktop-probe", "desktop-video-probe", "wayland-capture", + const SUBCOMMANDS: [&str; 9] = ["login", "unlock-check", "desktop-probe", "desktop-video-probe", "desktop-audio-probe", "wayland-capture", "wayland-input-probe", "clipboard-acceptance", "portal-acceptance"]; if let Some(arg) = std::env::args().nth(1).filter(|a| !SUBCOMMANDS.contains(&a.as_str())) { let usage = format!("usage: uuremote-server [{}]\n (no argument: run the host)", SUBCOMMANDS.join(" | ")); @@ -186,6 +186,9 @@ async fn main() -> Result<(), Box> { capture::keep_display_awake()?; return video::probe((screen.width,screen.height),screen.display.clone(),screen.portal.clone(),std::env::args().nth(2)).await.map_err(|e|e.into()); } + if std::env::args().nth(1).as_deref() == Some("desktop-audio-probe") { + return audio::probe(std::env::args().nth(2).and_then(|v| v.parse().ok()).unwrap_or(5)).await.map_err(|e| e.into()); + } if std::env::args().nth(1).as_deref() == Some("unlock-check") { return session_lock::check_from_stdin().map_err(|e| e.into()); }