From e603761aa84a2dd5e11996e0b5e354172bbb057d Mon Sep 17 00:00:00 2001 From: wenmou <99769916+mengdehong@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:31:00 +0800 Subject: [PATCH] Video: reuse capture buffers, cache unchanged frames and propagate send backpressure (#20) * Wayland: reuse DMA-BUF capture buffers and release protocol objects * Video: cache unchanged frames and propagate send backpressure --- c_engine/portal_engine.c | 13 +++- c_engine/portal_engine.h | 1 + c_engine/wayland_engine.c | 63 ++++++++++++---- src/capture.rs | 10 +-- src/inproc_encoder.rs | 83 ++++++++++++++++----- src/portal.rs | 33 +++++++-- src/video.rs | 150 ++++++++++++++++++++++++++------------ src/video_track.rs | 5 +- src/wayland.rs | 10 ++- 9 files changed, 267 insertions(+), 101 deletions(-) diff --git a/c_engine/portal_engine.c b/c_engine/portal_engine.c index d253506..2e05e3a 100644 --- a/c_engine/portal_engine.c +++ b/c_engine/portal_engine.c @@ -51,6 +51,7 @@ struct uu_portal { uint8_t *frame; size_t frame_length; uint64_t sequence; + uint64_t revision; }; #include "portal_clipboard.inc" @@ -182,6 +183,7 @@ static void copy_video_buffer(struct uu_portal *p, struct spa_buffer *spa) { memset(p->frame, 0, p->frame_length); for (size_t pixel = 0; pixel < p->frame_length / 4; pixel++) p->frame[pixel * 4 + 3] = 255; p->sequence++; + p->revision++; return; } struct spa_meta_region *crop = spa_buffer_find_meta_data(spa, SPA_META_VideoCrop, sizeof(*crop)); @@ -203,6 +205,7 @@ static void copy_video_buffer(struct uu_portal *p, struct spa_buffer *spa) { } } p->sequence++; + p->revision++; } } } @@ -418,14 +421,22 @@ void uu_portal_reset_frame(struct uu_portal *p) { if (!p) return; pthread_mutex_lock(&p->lock); p->sequence = 0; + p->revision++; if (p->frame) memset(p->frame, 0, p->frame_length); pthread_mutex_unlock(&p->lock); } int uu_portal_capture(struct uu_portal *p, uint8_t *frame, size_t length) { + return uu_portal_capture_since(p, frame, length, NULL); +} +int uu_portal_capture_since(struct uu_portal *p, uint8_t *frame, size_t length, uint64_t *revision) { if (!p || !frame) return 11; pthread_mutex_lock(&p->lock); int result = atomic_load(&p->closed) ? 9 : p->error ? p->error : !p->sequence ? 10 : length != p->frame_length ? 11 : 0; - if (!result) memcpy(frame, p->frame, length); + if (!result) { + if (revision && *revision == p->revision) result = 12; + else memcpy(frame, p->frame, length); + if (revision) *revision = p->revision; + } pthread_mutex_unlock(&p->lock); return result; } int uu_portal_alive(struct uu_portal *p) { diff --git a/c_engine/portal_engine.h b/c_engine/portal_engine.h index 7fe1167..7d1efc3 100644 --- a/c_engine/portal_engine.h +++ b/c_engine/portal_engine.h @@ -16,6 +16,7 @@ int uu_portal_clipboard_poll(struct uu_portal *portal, uint8_t *text, size_t cap int uu_portal_alive(struct uu_portal *portal); void uu_portal_reset_frame(struct uu_portal *portal); int uu_portal_capture(struct uu_portal *portal, uint8_t *bgra, size_t length); +int uu_portal_capture_since(struct uu_portal *portal, uint8_t *bgra, size_t length, uint64_t *revision); int uu_portal_move(struct uu_portal *portal, double x, double y, int relative); int uu_portal_button(struct uu_portal *portal, int button, int pressed); int uu_portal_key(struct uu_portal *portal, uint32_t keysym, int pressed); diff --git a/c_engine/wayland_engine.c b/c_engine/wayland_engine.c index eae6428..0bbb34e 100644 --- a/c_engine/wayland_engine.c +++ b/c_engine/wayland_engine.c @@ -21,6 +21,7 @@ static pthread_mutex_t lock = PTHREAD_MUTEX_INITIALIZER; static struct wl_display *display; +static struct wl_registry *registry; static struct wl_shm *shm; static struct wl_seat *seat; static struct zwlr_screencopy_manager_v1 *copy_manager; @@ -72,14 +73,15 @@ struct frame { int shm_fd,shm_busy; uint32_t shm_width,shm_height,shm_stride,shm_format; uint32_t width,height,stride,format,flags; - int done,failed,submitted; + int done,failed,submitted,released; struct zwp_linux_buffer_params_v1 *params; struct gbm_bo *bo; int bo_fd,dmabuf_used; void *map_data; uint32_t dma_format,dma_width,dma_height; + uint32_t bo_format,bo_width,bo_height; }; -static struct frame frames[2]={{.shm_fd=-1},{.shm_fd=-1}}; +static struct frame frames[2]={{.shm_fd=-1,.bo_fd=-1},{.shm_fd=-1,.bo_fd=-1}}; static unsigned pending; static uint32_t timestamp(void) { @@ -189,20 +191,28 @@ static void shm_clear(struct frame *frame) { frame->shm_length=0; frame->shm_fd=-1; frame->shm_busy=0; frame->shm_width=frame->shm_height=frame->shm_stride=frame->shm_format=0; } +static void dma_clear(struct frame *frame) { + if(frame->buffer) wl_buffer_destroy(frame->buffer); + if(frame->dmabuf_used && frame->pixels) { + gbm.bo_unmap(frame->bo,frame->map_data); frame->pixels=NULL; frame->map_data=NULL; + } + if(frame->bo) { if(frame->bo_fd>=0)close(frame->bo_fd); gbm.bo_destroy(frame->bo); } + frame->buffer=NULL; frame->bo=NULL; frame->bo_fd=-1; frame->dmabuf_used=frame->released=0; + frame->bo_format=frame->bo_width=frame->bo_height=0; +} static void frame_clear(struct frame *frame) { if(frame->handle) zwlr_screencopy_frame_v1_destroy(frame->handle); if(frame->params) zwp_linux_buffer_params_v1_destroy(frame->params); - if(frame->buffer) wl_buffer_destroy(frame->buffer); if(frame->dmabuf_used && frame->pixels) gbm.bo_unmap(frame->bo,frame->map_data); - if(frame->bo) { if(frame->bo_fd>=0)close(frame->bo_fd); gbm.bo_destroy(frame->bo); } if(frame->shm_busy) shm_clear(frame); - frame->handle=NULL; frame->params=NULL; frame->buffer=NULL; - frame->pixels=NULL; frame->map_data=NULL; frame->bo=NULL; - frame->bo_fd=-1; frame->dmabuf_used=0; + frame->handle=NULL; frame->params=NULL; + frame->pixels=NULL; frame->map_data=NULL; frame->dmabuf_used=0; frame->width=frame->height=frame->stride=frame->format=frame->flags=0; frame->done=frame->failed=frame->submitted=0; frame->dma_format=frame->dma_width=frame->dma_height=0; } +static void buffer_release(void*d,struct wl_buffer*b) { ((struct frame*)d)->released=1; } +static const struct wl_buffer_listener buffer_listener={buffer_release}; static void frame_fail(struct frame *frame) { frame->failed=frame->done=frame->submitted=1; } static void copy_buffer(void*d,struct zwlr_screencopy_frame_v1*f,uint32_t fmt,uint32_t w,uint32_t h,uint32_t s) { struct frame *frame=d; @@ -266,22 +276,33 @@ static void submit_shm(struct frame *frame) { static void dma_created(void*d,struct zwp_linux_buffer_params_v1*params,struct wl_buffer*buffer) { struct frame *frame=d; zwp_linux_buffer_params_v1_destroy(params); frame->params=NULL; - shm_clear(frame); frame->pixels=NULL; + frame->pixels=NULL; frame->buffer=buffer; frame->dmabuf_used=1; frame->width=frame->dma_width; frame->height=frame->dma_height; frame->format=frame->dma_format==0x34325258 ? WL_SHM_FORMAT_XRGB8888: frame->dma_format==0x34325241 ? WL_SHM_FORMAT_ARGB8888:frame->dma_format; + wl_buffer_add_listener(buffer,&buffer_listener,frame); + frame->released=0; zwlr_screencopy_frame_v1_copy(frame->handle,buffer); frame->submitted=1; } static void dma_failed(void*d,struct zwp_linux_buffer_params_v1*params) { struct frame *frame=d; zwp_linux_buffer_params_v1_destroy(params); frame->params=NULL; - dma_disable("import"); submit_shm(frame); + dma_disable("import"); dma_clear(frame); submit_shm(frame); } static const struct zwp_linux_buffer_params_v1_listener dma_params_listener={dma_created,dma_failed}; static int dma_start(struct frame *frame) { if(!dmabuf_requested || dmabuf_disabled || !gbm.device || frame->failed || !frame->shm_buffer || !(dmabuf_linear_formats&dma_format_bit(frame->dma_format))) return 0; + if(frame->buffer && frame->bo && frame->released && frame->bo_width==frame->dma_width && + frame->bo_height==frame->dma_height && frame->bo_format==frame->dma_format) { + frame->pixels=NULL; frame->dmabuf_used=1; frame->released=0; + frame->width=frame->dma_width; frame->height=frame->dma_height; + frame->format=frame->dma_format==0x34325258 ? WL_SHM_FORMAT_XRGB8888: + frame->dma_format==0x34325241 ? WL_SHM_FORMAT_ARGB8888:frame->dma_format; + zwlr_screencopy_frame_v1_copy(frame->handle,frame->buffer); frame->submitted=1; return 1; + } + dma_clear(frame); /* Single-plane linear RGB only; unsupported imports are asynchronous failures, never create_immed protocol errors. No privilege or device probing changes. */ frame->bo=gbm.bo_create(gbm.device,frame->dma_width,frame->dma_height,frame->dma_format,(1u<<2)|(1u<<4)); @@ -290,7 +311,8 @@ static int dma_start(struct frame *frame) { uint32_t stride=gbm.bo_get_stride(frame->bo); if(gbm.bo_get_plane_count(frame->bo)!=1 || gbm.bo_get_modifier(frame->bo)!=0 || stridedma_width*4 || stride>131072 || (uint64_t)stride*frame->dma_height>256*1024*1024 || - (frame->bo_fd=gbm.bo_get_fd(frame->bo))<0) {dma_disable("layout-or-export"); return 0;} + (frame->bo_fd=gbm.bo_get_fd(frame->bo))<0) {dma_disable("layout-or-export"); dma_clear(frame); return 0;} + frame->bo_format=frame->dma_format; frame->bo_width=frame->dma_width; frame->bo_height=frame->dma_height; frame->params=zwp_linux_dmabuf_v1_create_params(dmabuf); zwp_linux_buffer_params_v1_add_listener(frame->params,&dma_params_listener,frame); zwp_linux_buffer_params_v1_add(frame->params,frame->bo_fd,0,gbm.bo_get_offset(frame->bo,0),stride,0,0); @@ -311,14 +333,25 @@ static void reset(void) { if (key_state) xkb_state_unref(key_state); if (keymap) xkb_keymap_unref(keymap); if (xkb) xkb_context_unref(xkb); - for(unsigned i=0;i<2;i++) { frame_clear(&frames[i]); shm_clear(&frames[i]); } + for(unsigned i=0;i<2;i++) { frame_clear(&frames[i]); dma_clear(&frames[i]); shm_clear(&frames[i]); } if(gbm.device) gbm.device_destroy(gbm.device); if(gbm.fd>=0) close(gbm.fd); if(gbm.library) dlclose(gbm.library); memset(&gbm,0,sizeof(gbm)); gbm.fd=-1; - /* Disconnect drops every proxy; failure never leaves an initialized backend. */ + if(keyboard) zwp_virtual_keyboard_v1_destroy(keyboard); + if(pointer) zwlr_virtual_pointer_v1_destroy(pointer); + if(power) zwlr_output_power_v1_destroy(power); + if(keyboard_manager) zwp_virtual_keyboard_manager_v1_destroy(keyboard_manager); + if(pointer_manager) zwlr_virtual_pointer_manager_v1_destroy(pointer_manager); + if(power_manager) zwlr_output_power_manager_v1_destroy(power_manager); + if(copy_manager) zwlr_screencopy_manager_v1_destroy(copy_manager); + if(dmabuf) zwp_linux_dmabuf_v1_destroy(dmabuf); + for(unsigned i=0;i Result, String> { if let Some(error)=&self.init_error { return Err(error.clone()); } let mut buf = self.buffer.lock().unwrap(); + buf.resize(self.width as usize * self.height as usize * 4, 0); if let Some(kms)=&self.kms { kms.capture(&mut buf)?; return Ok(buf.clone()); } if let Some(session)=&self.portal { session.capture(&mut buf)?; return Ok(buf.clone()); } if self.backend == crate::desktop_backend::DesktopBackend::Wayland { diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index d906245..e417e43 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -11,6 +11,8 @@ use ff::ChannelLayout; use ff::util::picture; use ff::{codec, encoder, frame, software::scaling, Dictionary, Packet, Rational}; use tokio::sync::mpsc; +use tokio::sync::{Semaphore, OwnedSemaphorePermit}; +use std::collections::VecDeque; use crate::damage_capture::DamageCapture; @@ -58,13 +60,38 @@ fn ensure_init() -> Result<(), String> { result } -/// Sends one access unit per encoder packet. Returns `Ok(false)` when the receiver is gone. -fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender>) -> Result { +pub struct EncodedUnit { pub data: Vec, pub permit: OwnedSemaphorePermit, pub completed_at: Instant } + +fn acquire(budget: &Arc, units: &mpsc::Sender) -> Option { + tokio::runtime::Handle::current().block_on(async { + tokio::select! { + permit = budget.clone().acquire_owned() => permit.ok(), + _ = units.closed() => None, + } + }) +} + +#[cfg(test)] +mod budget_tests { + use super::*; + #[tokio::test] + async fn closing_encoder_output_wakes_a_blocked_capture() { + let budget=Arc::new(Semaphore::new(0)); + let (tx,rx)=mpsc::channel(1); + let worker=tokio::task::spawn_blocking(move || acquire(&budget,&tx)); + tokio::task::yield_now().await; + drop(rx); + assert!(tokio::time::timeout(Duration::from_secs(1),worker).await.unwrap().unwrap().is_none()); + } +} + +fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender, permits: &mut VecDeque) -> Result { loop { match encoder.receive_packet(packet) { Ok(()) => { let data = packet.data().unwrap_or_default().to_vec(); - if units.blocking_send(data).is_err() { return Ok(false); } + let permit = permits.pop_front().ok_or("Encoder returned an unbudgeted access unit")?; + if units.blocking_send(EncodedUnit {data, permit, completed_at: Instant::now()}).is_err() { return Ok(false); } } Err(ff::Error::Other { errno }) if errno == libc::EAGAIN => return Ok(true), Err(error) => return Err(text(error)), @@ -168,11 +195,11 @@ fn send_video_frame(encoder: &mut encoder::Video, frame: &frame::Video, kind: cr } /// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR. -pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { +pub fn run(config: Config, units: mpsc::Sender, stop: Arc, idr: Arc, budget: Arc) -> Result<(), String> { ensure_init()?; if config.damage { match DamageCapture::open(&config.display) { - Ok(capture) => return run_damage(config, capture, units, stop, idr), + Ok(capture) => return run_damage(config, capture, units, stop, idr, budget), Err(error) => log::warn!("Damage-driven capture unavailable, using x11grab: {error}"), } } @@ -195,14 +222,18 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, let mut scaler: Option<(scaling::Context, (Pixel, u32, u32))> = None; let mut decoded = frame::Video::empty(); - let mut converted = frame::Video::new(pixel, width, height); + let mut converted = frame::Video::empty(); let mut packet = Packet::empty(); let started = Instant::now(); let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps)); let mut last_at: Option = None; let mut last_slot: i64 = -1; - for (stream, input_packet) in input.packets() { + let mut permits = VecDeque::with_capacity(4); + let mut packets = input.packets(); + loop { if stop.load(Ordering::Relaxed) { return Ok(()); } + let Some(permit) = acquire(&budget, &units) else { return Ok(()); }; + let Some((stream, input_packet)) = packets.next() else { return Ok(()); }; if stream.index() != index { continue; } decoder.send_packet(&input_packet).map_err(text)?; match decoder.receive_frame(&mut decoded) { @@ -236,20 +267,19 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, frame.set_pts(Some(slot)); frame.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None }); send_video_frame(&mut encoder, frame, config.encoder)?; - if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); } + permits.push_back(permit); + if !drain(&mut encoder, &mut packet, &units, &mut permits)? { return Ok(()); } } - Ok(()) } /// Damage-driven loop. Damage only marks the picture dirty; a dirty picture is captured and encoded on /// the next fixed frame tick, so the stream keeps a steady cadence while the screen moves. The viewer's /// jitter buffer grows with irregular arrivals, and irregular sends were what raised its latency. -/// An idle screen sends a keepalive about once a second so the viewer still sees traffic. -fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { +fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender, stop: Arc, idr: Arc, budget: Arc) -> Result<(), String> { let (mut encoder, pixel, width, height) = open_encoder(&config)?; let (cap_width, cap_height) = capture.size(); let mut captured = frame::Video::new(Pixel::BGRZ, cap_width, cap_height); - let mut converted = frame::Video::new(pixel, width, height); + let mut converted = frame::Video::empty(); let mut scaler = None; let mut packet = Packet::empty(); let started = Instant::now(); @@ -261,6 +291,9 @@ fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender i32; fn uu_portal_reset_frame(portal: *mut c_void); - fn uu_portal_capture(portal: *mut c_void, bgra: *mut u8, length: usize) -> i32; + fn uu_portal_capture_since(portal: *mut c_void, bgra: *mut u8, length: usize, revision: *mut u64) -> i32; fn uu_portal_move(portal: *mut c_void, x: f64, y: f64, relative: i32) -> i32; fn uu_portal_button(portal: *mut c_void, button: i32, pressed: i32) -> i32; fn uu_portal_key(portal: *mut c_void, keysym: u32, pressed: i32) -> i32; @@ -26,6 +28,10 @@ mod native { // The engine owns callback threads; public calls are serialized by Mutex. unsafe impl Send for Handle {} impl Drop for Handle { fn drop(&mut self) { unsafe { uu_portal_close(self.0.as_ptr()) } } } + fn blank_changed(frame: &mut [u8], version: &mut Option) -> bool { + if *version == Some(super::FrameVersion(0, 0)) { return false; } + super::blank(frame); *version = Some(super::FrameVersion(0, 0)); true + } pub struct Session { handle: Mutex, locked: tokio::sync::watch::Receiver>, was_locked: std::sync::atomic::AtomicBool, retry_at: Mutex>, pub width: u32, pub height: u32 } impl Session { pub fn is_alive(&self) -> bool { @@ -56,6 +62,10 @@ mod native { Ok(Self { handle: Mutex::new(Handle(handle)), locked: crate::session_lock::watch_locked(), was_locked: std::sync::atomic::AtomicBool::new(false), retry_at: Mutex::new(None), width, height }) } pub fn capture(&self, frame: &mut [u8]) -> Result<(), String> { + self.capture_changed(frame, &mut None).map(|_| ()) + } + pub fn capture_changed(&self, frame: &mut [u8], version: &mut Option) -> Result { + if frame.len() != self.width as usize * self.height as usize * 4 { return Err(super::error(11)); } let mut handle = self.handle.lock().map_err(|_| "Portal session lock poisoned")?; let state = *self.locked.borrow(); if state != Some(false) { @@ -64,30 +74,30 @@ mod native { self.was_locked.store(true, std::sync::atomic::Ordering::Release); unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; } - super::blank(frame); return Ok(()); + return Ok(blank_changed(frame, version)); } if self.was_locked.swap(false, std::sync::atomic::Ordering::AcqRel) { // First unlocked capture after a real lock: drop frames captured while locked. unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; - super::blank(frame); return Ok(()); + return Ok(blank_changed(frame, version)); } if unsafe { uu_portal_alive(handle.0.as_ptr()) } == 0 { // GNOME can close the stream before the periodic lock watcher updates. if crate::session_lock::locked_now() != Some(false) { self.was_locked.store(true, std::sync::atomic::Ordering::Release); - super::blank(frame); return Ok(()); + return Ok(blank_changed(frame, version)); } // logind's LockedHint can lag compositor stream closure. Give it a // full watcher interval before attempting another permission session. let mut retry = self.retry_at.lock().map_err(|_| "Portal recovery lock poisoned")?; let deadline = retry.get_or_insert_with(|| std::time::Instant::now() + std::time::Duration::from_secs(1)); - if std::time::Instant::now() < *deadline { super::blank(frame); return Ok(()); } + if std::time::Instant::now() < *deadline { return Ok(blank_changed(frame, version)); } let replacement = match Self::open() { Ok(value) => value, Err(error) => { log::warn!("Portal capture recovery deferred: {error}"); *retry = Some(std::time::Instant::now() + std::time::Duration::from_secs(5)); - super::blank(frame); return Ok(()); + return Ok(blank_changed(frame, version)); } }; *retry = None; @@ -97,8 +107,14 @@ mod native { *handle = replacement.handle.into_inner().map_err(|_| "Portal session lock poisoned")?; log::info!("Portal capture session restored after closure"); } - let code = unsafe { uu_portal_capture(handle.0.as_ptr(), frame.as_mut_ptr(), frame.len()) }; - if code == 10 || code == 9 { super::blank(frame); Ok(()) } else { super::check(code) } + let identity = handle.0.as_ptr() as usize; + let mut revision = version.filter(|v| v.0 == identity).map_or(0, |v| v.1); + let code = unsafe { uu_portal_capture_since(handle.0.as_ptr(), frame.as_mut_ptr(), frame.len(), &mut revision) }; + match code { + 0 | 12 => { *version = Some(super::FrameVersion(identity, revision)); Ok(code == 0) } + 9 | 10 => Ok(blank_changed(frame, version)), + _ => Err(super::error(code)), + } } pub fn clipboard_supported(&self) -> bool { @@ -166,6 +182,7 @@ impl PortalSession { pub fn is_alive(&self) -> bool { false } pub fn open() -> Result { Err("Portal backend requires Linux and --features portal".into()) } pub fn capture(&self, _: &mut [u8]) -> Result<(), String> { Err(error(2)) } + pub fn capture_changed(&self, _: &mut [u8], _: &mut Option) -> Result { Err(error(2)) } pub fn move_pointer(&self, _: f64, _: f64, _: bool) -> Result<(), String> { Err(error(2)) } pub fn button(&self, _: i32, _: bool) -> Result<(), String> { Err(error(2)) } pub fn key(&self, _: u64, _: bool) -> Result<(), String> { Err(error(2)) } diff --git a/src/video.rs b/src/video.rs index 81ee52d..e103be2 100644 --- a/src/video.rs +++ b/src/video.rs @@ -3,7 +3,7 @@ use std::sync::atomic::AtomicBool; use std::time::{Duration, Instant}; use tokio::io::AsyncReadExt; use tokio::process::Command; -use tokio::sync::mpsc; +use tokio::sync::{mpsc, Semaphore, OwnedSemaphorePermit}; use webrtc::media::Sample; use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; use webrtc::peer_connection::RTCPeerConnection; @@ -66,6 +66,7 @@ struct CompleteFrame { sequence: u64, completed_at: Instant, data: Vec, + _permit: Option, } const RTP_CLOCK: u64 = 90_000; @@ -269,16 +270,16 @@ impl Drop for StopOnDrop { fn drop(&mut self) { self.0.store(true, std::sync::at async fn read_encoder( size: (u32, u32), display: String, portal:Option>, latest: mpsc::Sender>, - settings: VideoSettings, sequence: Arc, idr: Arc, + settings: VideoSettings, sequence: Arc, idr: Arc, budget: Arc, ) -> Result<(), String> { // The sequence is shared so a CPU fallback or settings restart continues it (skip statistics stay correct). let encoder = Encoder::requested(); - if encoder == Encoder::Cpu { return read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await; } - match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,sequence.clone(),settings,idr.clone()).await { + if encoder == Encoder::Cpu { return read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr,budget).await; } + match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,sequence.clone(),settings,idr.clone(),budget.clone()).await { Ok(())=>Ok(()), Err(error) if !latest.is_closed()=>{ log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name_for(settings.codec),error); - read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await + read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr,budget).await }, Err(error)=>Err(error), } @@ -288,21 +289,21 @@ async fn read_encoder( #[cfg(feature = "inproc-av")] async fn read_inprocess( config: crate::inproc_encoder::Config, latest: mpsc::Sender>, - sequence: Arc, idr: Arc, + sequence: Arc, idr: Arc, budget: Arc, ) -> Result<(), String> { let codec = config.codec; - let (units_tx, mut units) = mpsc::channel::>(4); + let (units_tx, mut units) = mpsc::channel::(4); let stop = Arc::new(AtomicBool::new(false)); // If this future is dropped (reader task aborted), the blocking encoder stops at its next packet. let _stop_on_drop = StopOnDrop(stop.clone()); let job = tokio::task::spawn_blocking({ let stop = stop.clone(); - move || crate::inproc_encoder::run(config, units_tx, stop, idr) + move || crate::inproc_encoder::run(config, units_tx, stop, idr, budget) }); let mut gate = DependencyGate::default(); - while let Some(data) = units.recv().await { + while let Some(unit) = units.recv().await { let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; - gate.submit(&latest, Arc::new(CompleteFrame { codec, sequence, completed_at: Instant::now(), data })); + gate.submit(&latest, Arc::new(CompleteFrame { codec, sequence, completed_at: unit.completed_at, data: unit.data, _permit: Some(unit.permit) })); if latest.is_closed() { break; } } // Dropping the receiver also unblocks the encoder if it is waiting to send. @@ -333,10 +334,11 @@ fn rate_args(settings: &VideoSettings, encoder: Encoder) -> (String, String) { async fn read_encoder_with( size: (u32,u32),display:String,portal:Option>,latest:mpsc::Sender>,encoder:Encoder,sequence:Arc, - settings: VideoSettings, #[cfg_attr(not(feature = "inproc-av"), allow(unused_variables))] idr: Arc, + settings: VideoSettings, #[cfg_attr(not(feature = "inproc-av"), allow(unused_variables))] idr: Arc, budget: Arc, )->Result<(),String>{ let kms=display.starts_with("kms:"); let wayland=kms || display.starts_with("wayland:"); + let (permits_tx,mut permits_rx)=mpsc::channel::(4); let kms_capture=if kms { let capture=tokio::task::spawn_blocking(crate::kms::Capture::open).await.map_err(|_|"KMS worker failed")??; if (capture.width,capture.height)!=size { return Err("KMS mode changed; restart host to refresh desktop geometry".into()); } @@ -354,7 +356,7 @@ async fn read_encoder_with( }; log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name_for(settings.codec)); - return read_inprocess(config, latest, sequence, idr).await; + return read_inprocess(config, latest, sequence, idr, budget).await; } let size = format!("{}x{}", size.0, size.1); let fps = settings.fps.to_string(); @@ -397,7 +399,8 @@ async fn read_encoder_with( #[cfg(not(target_os="linux"))] let executable=std::env::current_exe().map_err(|e|format!("Capture executable: {e}"))?; let mut capture=wayland_capture_command(&executable,settings.fps) - .kill_on_drop(true).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::inherit()) + .env("UU_WAYLAND_DEMAND","1") + .kill_on_drop(true).stdin(std::process::Stdio::piped()).stdout(std::process::Stdio::piped()).stderr(std::process::Stdio::inherit()) .spawn().map_err(|e|format!("Start Wayland capture: {e}"))?; let source:std::process::Stdio=capture.stdout.take().ok_or("Capture stdout missing")? .try_into().map_err(|e|format!("Capture pipe: {e}"))?; @@ -416,11 +419,13 @@ async fn read_encoder_with( use tokio::io::AsyncWriteExt; let mut tick=tokio::time::interval(Duration::from_nanos(1_000_000_000/u64::from(settings.fps))); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut bytes=vec![0;capture.width as usize*capture.height as usize*4]; loop { tick.tick().await; + let permit=budget.clone().acquire_owned().await.map_err(|_|"Encoder stopped")?; + permits_tx.send(permit).await.map_err(|_|"Encoder stopped")?; let capture=capture.clone(); - let bytes=tokio::task::spawn_blocking(move || { - let mut bytes=vec![0;capture.width as usize*capture.height as usize*4]; + bytes=tokio::task::spawn_blocking(move || { capture.capture(&mut bytes)?; Ok::<_,String>(bytes) }).await.map_err(|_|"KMS worker failed")??; sink.write_all(&bytes).await.map_err(|_|"KMS encoder pipe failed")?; @@ -433,23 +438,42 @@ async fn read_encoder_with( let interval=Duration::from_nanos(1_000_000_000/u64::from(settings.fps)); let mut tick=tokio::time::interval(interval); tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut frame=vec![0;(session.width as usize)*(session.height as usize)*4]; + let mut version=None; loop { tick.tick().await; + let permit=budget.clone().acquire_owned().await.map_err(|_|"Encoder stopped")?; + permits_tx.send(permit).await.map_err(|_|"Encoder stopped")?; let session=session.clone(); - let frame=tokio::task::spawn_blocking(move || { - let mut frame=vec![0;(session.width as usize)*(session.height as usize)*4]; - session.capture(&mut frame)?; - Ok::<_,String>(frame) + (frame,version)=tokio::task::spawn_blocking(move || { + session.capture_changed(&mut frame,&mut version)?; + Ok::<_,String>((frame,version)) }).await.map_err(|e|format!("Portal capture worker: {e}"))??; sink.write_all(&frame).await.map_err(|e|format!("Portal encoder pipe: {e}"))?; } }))) } else if let Some(mut capture)=native_capture { + let mut demand=capture.stdin.take().ok_or("Capture stdin missing")?; Some(EncoderTask(tokio::spawn(async move { - let status=capture.wait().await.map_err(|e|format!("Wayland capture: {e}"))?; - // Nothing awaits this task; log so the cause survives the generic encoder EOF. - log::warn!("Wayland capture exited: {status}"); - Err(format!("Wayland capture exited: {status}")) + use tokio::io::AsyncWriteExt; + tokio::select! { + result=async { + let mut tick=tokio::time::interval(Duration::from_nanos(1_000_000_000/u64::from(settings.fps))); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + let permit=budget.clone().acquire_owned().await.map_err(|_|"Encoder stopped")?; + tick.tick().await; + permits_tx.send(permit).await.map_err(|_|"Encoder stopped")?; + demand.write_all(&[1]).await.map_err(|_|"Capture demand pipe closed")?; + } + } => result, + status=capture.wait() => { + let status=status.map_err(|e|format!("Wayland capture: {e}"))?; + // Nothing awaits this task; log so the cause survives the generic encoder EOF. + log::warn!("Wayland capture exited: {status}"); + Err(format!("Wayland capture exited: {status}")) + }, + } }))) } else { None }; let mut stdout = child.stdout.take().ok_or("Encoder stdout missing")?; @@ -468,8 +492,9 @@ async fn read_encoder_with( if next > MAX_ACCESS_UNIT { return Err("Encoder access unit exceeds limit".into()); } let frame: Vec = pending.drain(..next).collect(); let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + let permit=if wayland {Some(permits_rx.recv().await.ok_or("Capture producer stopped")?)} else {None}; gate.submit(&latest, Arc::new(CompleteFrame { - codec: settings.codec, sequence, completed_at: Instant::now(), data: frame, + codec: settings.codec, sequence, completed_at: Instant::now(), data: frame, _permit: permit, })); if latest.is_closed() { return Ok(()); } } @@ -499,7 +524,7 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option, track: Arc, track: Arc, track: Arc { let result = result.map_err(|e| format!("Encoder reader task: {e}"))?; if let (Err(error), VideoCodec::H265, true) = (&result, current.codec, track.supports(VideoCodec::H264)) { log::warn!("HEVC stream failed ({error}); falling back to negotiated H264"); - current.codec = VideoCodec::H264; applied.send_modify(|value| value.codec = VideoCodec::H264); current = *settings.borrow_and_update(); while receiver.try_recv().is_ok() {} - reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone())); + reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone(), budget.clone())); continue; } return result; }, _ = health.tick() => { if transport_closed(&pc) { return Ok(()); } }, changed = settings.changed(), if settings_open => { - if changed.is_err() { settings_open = false; continue; } - // Restart the encoder with the viewer's new 帧率/画质; it begins with an IDR, and frames - // still queued from the old encoder remain a valid chain before it. - let next = *settings.borrow_and_update(); - let codec_changed = next.codec != current.codec; - current = next; - timeline.fps = u64::from(next.fps); - if codec_changed { - reader.0.abort(); - let _ = (&mut reader.0).await; - // Drain only after the old producer is gone, before the new IDR can enter. - while receiver.try_recv().is_ok() {} - reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())); - } else { - reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await; - } + if changed.is_err() { settings_open = false; } + continue; }, frame = receiver.recv() => { // A closed queue means the reader ended; its JoinHandle carries the reason. @@ -600,6 +627,10 @@ pub async fn stream_desktop(pc: Arc, track: Arc { result.map_err(|e| format!("Video send: {e}"))?; break; }, _ = health.tick() => { if transport_closed(&pc) { return Ok(()); } }, + changed = settings.changed(), if settings_open => { + if changed.is_err() { settings_open=false; } + else if *settings.borrow()!=current { send_interrupted=true; continue 'stream; } + }, } } previous = Some(frame.sequence); @@ -630,6 +661,29 @@ mod tests { } } use super::*; + #[tokio::test] + async fn frame_budget_spans_queued_sending_and_restarted_encoders() { + let budget=Arc::new(Semaphore::new(2)); + let (tx,mut rx)=mpsc::channel(4); + let mut gate=DependencyGate::default(); + for sequence in 1..=2 { + let permit=budget.clone().acquire_owned().await.unwrap(); + gate.submit(&tx,Arc::new(CompleteFrame {codec:VideoCodec::H264,sequence,completed_at:Instant::now(), + data:vec![0,0,1,5,42],_permit:Some(permit)})); + } + let sending=rx.recv().await.unwrap(); + assert_eq!(budget.available_permits(),0); + let restarted=tokio::spawn(budget.clone().acquire_owned()); + tokio::task::yield_now().await; + assert!(!restarted.is_finished()); + drop(sending); + let permit=tokio::time::timeout(Duration::from_secs(1),restarted).await.unwrap().unwrap().unwrap(); + assert_eq!(budget.available_permits(),0); + drop(rx); + assert_eq!(budget.available_permits(),1); + drop(permit); + assert_eq!(budget.available_permits(),2); + } #[test] fn x264_buffer_fits_keyframes() { assert_eq!(buffer_kbits(2000, Encoder::Nvenc), 250); @@ -667,7 +721,7 @@ mod tests { let (tx, mut rx) = mpsc::channel(4); let old_tx = tx.clone(); let (started_tx, started_rx) = tokio::sync::oneshot::channel(); - let frame = |sequence, idr| Arc::new(CompleteFrame { + let frame = |sequence, idr| Arc::new(CompleteFrame { _permit: None, sequence, codec: VideoCodec::H264, completed_at: Instant::now(), data: vec![0, 0, 1, 9, 240, 0, 0, 1, if idr { 5 } else { 1 }, 42], }); @@ -762,7 +816,7 @@ mod tests { async fn bounded_queue_never_sends_broken_prediction_chain_after_overload(){ let(tx,mut rx)=mpsc::channel(1); let mut gate=DependencyGate::default(); - let frame=|sequence,idr|Arc::new(CompleteFrame{sequence,codec:VideoCodec::H264,completed_at:Instant::now(),data:vec![0,0,1,9,240,0,0,1,if idr{5}else{1},42]}); + let frame=|sequence,idr|Arc::new(CompleteFrame{_permit:None,sequence,codec:VideoCodec::H264,completed_at:Instant::now(),data:vec![0,0,1,9,240,0,0,1,if idr{5}else{1},42]}); gate.submit(&tx,frame(1,false));assert!(rx.try_recv().is_err()); // no IDR yet gate.submit(&tx,frame(2,true)); gate.submit(&tx,frame(3,false)); // queue full: P3 lost @@ -795,7 +849,7 @@ mod tests { async fn hevc_dependency_gate_recovers_only_at_irap_after_overflow() { let (tx, mut rx) = mpsc::channel(1); let mut gate = DependencyGate::default(); - let frame = |sequence, irap| Arc::new(CompleteFrame { codec: VideoCodec::H265, + let frame = |sequence, irap| Arc::new(CompleteFrame { _permit: None, codec: VideoCodec::H265, sequence, completed_at: Instant::now(), data: vec![0,0,1,if irap { 38 } else { 2 },1,42] }); gate.submit(&tx, frame(1, false)); assert!(rx.try_recv().is_err()); diff --git a/src/video_track.rs b/src/video_track.rs index 4851154..85919b9 100644 --- a/src/video_track.rs +++ b/src/video_track.rs @@ -61,14 +61,15 @@ impl VideoTrack { }; let count = payloads.len(); let mut complete = count > 0; + let timestamp = state.timestamp; + state.timestamp = state.timestamp.wrapping_add((sample.duration.as_secs_f64() * 90_000.0) as u32); for (index, payload) in payloads.into_iter().enumerate() { - let packet = Packet { header: Header { version: 2, sequence_number: state.sequence, timestamp: state.timestamp, + let packet = Packet { header: Header { version: 2, sequence_number: state.sequence, timestamp, marker: index + 1 == count, ..Default::default() }, payload }; state.sequence = state.sequence.wrapping_add(1); complete &= track.write_rtp_with_extensions(&packet, extensions).await? > 0; } if complete { self.progress.send_modify(|value| { value.0 += 1; value.1 = codec; }); } - state.timestamp = state.timestamp.wrapping_add((sample.duration.as_secs_f64() * 90_000.0) as u32); Ok(()) } } diff --git a/src/wayland.rs b/src/wayland.rs index 27c83be..4c77152 100644 --- a/src/wayland.rs +++ b/src/wayland.rs @@ -63,7 +63,7 @@ pub fn key(symbol:u64,pressed:bool)->i32 { /// Capture and the pipe write run on separate threads with two frame buffers, so a /// slow encoder read no longer delays the next screencopy request. pub fn capture_stdout() -> Result<(),String> { - use std::io::Write; + use std::io::{Read,Write}; let (w,h)=init()?; let size=(w as usize)*(h as usize)*4; let fps=capture_fps(std::env::var("UU_WAYLAND_FPS").ok().as_deref())?; @@ -78,17 +78,23 @@ pub fn capture_stdout() -> Result<(),String> { let mut stdout=std::io::stdout().lock(); for frame in full_rx { stdout.write_all(&frame).map_err(|e|format!("Capture pipe: {e}"))?; + stdout.flush().map_err(|e|format!("Capture pipe: {e}"))?; if free_tx.send(frame).is_err() { break; } } Ok(()) }).map_err(|e|format!("Capture writer: {e}"))?; + let demand=std::env::var("UU_WAYLAND_DEMAND").is_ok_and(|v|v=="1"); + let mut stdin=std::io::stdin().lock(); loop { + if demand { stdin.read_exact(&mut [0]).map_err(|_|"Capture demand pipe closed")?; } // Time from buffer availability: a slow encoder read must not shorten the next interval. let Ok(mut frame)=free_rx.recv() else { break }; let start=std::time::Instant::now(); capture(&mut frame)?; if full_tx.send(frame).is_err() { break; } - if let Some(left)=interval.checked_sub(start.elapsed()) { std::thread::sleep(left); } + if !demand { + if let Some(left)=interval.checked_sub(start.elapsed()) { std::thread::sleep(left); } + } } drop(full_tx); writer.join().map_err(|_|"Capture writer panicked")?