From 8a8a4c165f9eef12a57c9d37313f6f5a06407614 Mon Sep 17 00:00:00 2001 From: VallierDev <24893073+VallierDev@users.noreply.github.com> Date: Thu, 15 Oct 2026 10:28:24 +0800 Subject: [PATCH] Add explicit VAAPI encoding with CPU fallback in both pipelines --- AGENTS.md | 9 ++++++ README.en.md | 9 ++++++ README.md | 9 ++++++ src/inproc_encoder.rs | 57 +++++++++++++++++++++++++++++++----- src/video.rs | 67 ++++++++++++++++++++++++++++++++++--------- 5 files changed, 131 insertions(+), 20 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 3c81dc9..ba97282 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -224,3 +224,12 @@ started a second full host server instead, so check that a subcommand exists in - New protocol behavior: cite the evidence (observed field numbers, official log strings) in a comment. - Update README.md / README.en.md when user-visible features or limitations change, and add `docs/release-notes/.md` before tagging a release. + +### VAAPI (opt-in / 显式启用) + +`UU_VIDEO_ENCODER=vaapi` selects Intel/AMD H.264 VAAPI; `UU_VAAPI_DEVICE` defaults to +`/dev/dri/renderD128` (render-node permissions required). No automatic VAAPI selection. +Both FFmpeg process and `--features inproc-av` upload NV12 frames to a VAAPI frame pool. +VBR uses the viewer bitrate preset and CQ-derived minimum QP, no B-frames, async depth 1, +AUD and the existing GOP/VFR timing. Unsupported device/driver/initialization falls back to libx264 +with a warning. Intel/AMD hardware acceptance is still required (未实机验证). diff --git a/README.en.md b/README.en.md index a92e4e7..1904087 100644 --- a/README.en.md +++ b/README.en.md @@ -262,3 +262,12 @@ Released under the [GPL-3.0](LICENSE): you may use, modify and redistribute it, - The NetEase UU Remote team, for a remote tool that is a pleasure to use. - [webrtc-rs](https://github.com/webrtc-rs/webrtc), FFmpeg, GTK and the other open-source projects this builds on. - Most of the code was written with the help of Claude and Codex. + +### VAAPI (opt-in / 显式启用) + +`UU_VIDEO_ENCODER=vaapi` selects Intel/AMD H.264 VAAPI; `UU_VAAPI_DEVICE` defaults to +`/dev/dri/renderD128` (render-node permissions required). No automatic VAAPI selection. +Both FFmpeg process and `--features inproc-av` upload NV12 frames to a VAAPI frame pool. +VBR uses the viewer bitrate preset and CQ-derived minimum QP, no B-frames, async depth 1, +AUD and the existing GOP/VFR timing. Unsupported device/driver/initialization falls back to libx264 +with a warning. Intel/AMD hardware acceptance is still required (未实机验证). diff --git a/README.md b/README.md index 1dfef5a..776027e 100644 --- a/README.md +++ b/README.md @@ -244,3 +244,12 @@ flowchart LR - 网易 UU 远程团队:做了一个好用的远程工具。 - [webrtc-rs](https://github.com/webrtc-rs/webrtc)、FFmpeg、GTK 等开源项目。 - 本项目的大部分代码在 Claude 和 Codex 的协助下完成。 + +### VAAPI (opt-in / 显式启用) + +`UU_VIDEO_ENCODER=vaapi` selects Intel/AMD H.264 VAAPI; `UU_VAAPI_DEVICE` defaults to +`/dev/dri/renderD128` (render-node permissions required). No automatic VAAPI selection. +Both FFmpeg process and `--features inproc-av` upload NV12 frames to a VAAPI frame pool. +VBR uses the viewer bitrate preset and CQ-derived minimum QP, no B-frames, async depth 1, +AUD and the existing GOP/VFR timing. Unsupported device/driver/initialization falls back to libx264 +with a warning. Intel/AMD hardware acceptance is still required (未实机验证). diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index 9f9a089..dfbd7cd 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -24,7 +24,7 @@ pub struct Config { pub target_bps: i64, pub max_bps: i64, pub buffer_bits: i64, - pub gpu: bool, + pub encoder: crate::video::Encoder, /// Capture on XDamage changes (XShm) instead of x11grab; `UU_CAPTURE=grab` turns it off. pub damage: bool, } @@ -74,15 +74,16 @@ fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender /// Opens the H.264 encoder for `config`. Returns the encoder, its input pixel format and output size. fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), String> { let (width, height) = config.output.unwrap_or(config.input); - let codec = encoder::find_by_name(if config.gpu { "h264_nvenc" } else { "libx264" }) + let codec = encoder::find_by_name(config.encoder.name()) .ok_or("H.264 encoder is not available in this FFmpeg build")?; // Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one. let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?; context.set_width(width); context.set_height(height); // NVENC takes packed BGR0 directly, so the GPU path skips the CPU colour conversion to YUV420P. - let pixel = if config.gpu { Pixel::BGRZ } else { Pixel::YUV420P }; - context.set_format(pixel); + let pixel = match config.encoder { crate::video::Encoder::Nvenc => Pixel::BGRZ, crate::video::Encoder::Vaapi => Pixel::NV12, _ => Pixel::YUV420P }; + context.set_format(if config.encoder == crate::video::Encoder::Vaapi { Pixel::VAAPI } else { pixel }); + if config.encoder == crate::video::Encoder::Vaapi { attach_vaapi(&mut context, width, height)?; } context.set_time_base(Rational::new(1, config.fps as i32)); context.set_frame_rate(Some(Rational::new(config.fps as i32, 1))); context.set_gop(config.gop); @@ -98,7 +99,10 @@ fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), St let cq = config.cq.to_string(); let gop = config.gop.to_string(); let mut options = Dictionary::new(); - if config.gpu { + if config.encoder == crate::video::Encoder::Vaapi { + for (key,value) in [("profile", "constrained_baseline"), ("rc_mode", "VBR"), ("async_depth", "1"), ("aud", "1")] { options.set(key,value); } + options.set("qmin", &cq); + } else if config.encoder == crate::video::Encoder::Nvenc { for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"), ("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] { options.set(key, value); @@ -116,6 +120,45 @@ fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), St Ok((encoder, pixel, width, height)) } +// Each codec owns a reference to its hardware frame pool. Every error path releases +// the temporary device/pool refs; AVCodecContext releases its own ref on drop. +fn attach_vaapi(context: &mut encoder::video::Video, width: u32, height: u32) -> Result<(), String> { + use ff::sys::*; + let device_name = CString::new(crate::video::vaapi_device()).map_err(|_| "Invalid VAAPI device")?; + unsafe { + let mut device = std::ptr::null_mut(); + let result = av_hwdevice_ctx_create(&mut device, AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI, device_name.as_ptr(), std::ptr::null_mut(), 0); + if result < 0 { return Err(format!("VAAPI device initialization failed: {}", text(ff::Error::from(result)))); } + let mut pool = av_hwframe_ctx_alloc(device); + av_buffer_unref(&mut device); + if pool.is_null() { return Err("VAAPI frame pool allocation failed".into()); } + let frames = (*pool).data as *mut AVHWFramesContext; + (*frames).format = AVPixelFormat::AV_PIX_FMT_VAAPI; + (*frames).sw_format = AVPixelFormat::AV_PIX_FMT_NV12; + (*frames).width = width as i32; + (*frames).height = height as i32; + (*frames).initial_pool_size = 8; + let result = av_hwframe_ctx_init(pool); + if result < 0 { av_buffer_unref(&mut pool); return Err(format!("VAAPI frame pool initialization failed: {}", text(ff::Error::from(result)))); } + (*context.as_mut_ptr()).hw_frames_ctx = pool; + } + Ok(()) +} + +fn send_video_frame(encoder: &mut encoder::Video, frame: &frame::Video, kind: crate::video::Encoder) -> Result<(), String> { + if kind != crate::video::Encoder::Vaapi { return encoder.send_frame(frame).map_err(text); } + let mut uploaded = frame::Video::empty(); + unsafe { + let result = ff::sys::av_hwframe_get_buffer((*encoder.as_ptr()).hw_frames_ctx, uploaded.as_mut_ptr(), 0); + if result < 0 { return Err(text(ff::Error::from(result))); } + let result = ff::sys::av_hwframe_transfer_data(uploaded.as_mut_ptr(), frame.as_ptr(), 0); + if result < 0 { return Err(text(ff::Error::from(result))); } + let result = ff::sys::av_frame_copy_props(uploaded.as_mut_ptr(), frame.as_ptr()); + if result < 0 { return Err(text(ff::Error::from(result))); } + } + encoder.send_frame(&uploaded).map_err(text) +} + /// Runs until `stop` is set or the receiver is dropped. A set `idr` flag forces the next frame to be an IDR. pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, idr: Arc) -> Result<(), String> { ensure_init()?; @@ -184,7 +227,7 @@ pub fn run(config: Config, units: mpsc::Sender>, stop: Arc, frame.set_pts(Some(slot)); frame.set_kind(if idr.swap(false, Ordering::AcqRel) { picture::Type::I } else { picture::Type::None }); - encoder.send_frame(frame).map_err(text)?; + send_video_frame(&mut encoder, frame, config.encoder)?; if !drain(&mut encoder, &mut packet, &units)? { return Ok(()); } } Ok(()) @@ -244,7 +287,7 @@ fn run_damage(config: Config, mut capture: DamageCapture, units: mpsc::Sender bool { - std::env::var("UU_VIDEO_ENCODER").map_or(true, |v| v != "cpu") +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Encoder { Cpu, Nvenc, Vaapi } +impl Encoder { + pub(crate) fn requested() -> Self { Self::from_setting(std::env::var("UU_VIDEO_ENCODER").ok().as_deref()) } + fn from_setting(value: Option<&str>) -> Self { + match value { Some("cpu") => Self::Cpu, Some("vaapi") => Self::Vaapi, _ => Self::Nvenc } + } + pub(crate) fn name(self) -> &'static str { match self { Self::Cpu => "libx264", Self::Nvenc => "h264_nvenc", Self::Vaapi => "h264_vaapi" } } +} +pub(crate) fn vaapi_device() -> String { std::env::var("UU_VAAPI_DEVICE").unwrap_or_else(|_| "/dev/dri/renderD128".into()) } + +/// NV12 conversion must precede upload; scaling runs on software frames, before hwupload. +fn vaapi_filter(output: Option<(u32, u32)>) -> String { + let scale = output.map_or(String::new(), |(w,h)| format!("scale={w}:{h}:flags=bilinear,")); + format!("{scale}format=nv12,hwupload") +} +fn vaapi_options(settings: &VideoSettings, gop: &str) -> Vec { + let (maxrate, buffer) = rate_args(settings); + // QVBR is not uniformly available on Intel/AMD drivers. VBR + qmin sets a quality + // floor while retaining the viewer's bitrate cap; unsupported drivers fall back to CPU. + ["-c:v", "h264_vaapi", "-profile:v", "constrained_baseline", "-g", gop, "-bf", "0", + "-rc_mode", "VBR", "-b:v", &settings.max_kbps().map_or("16M".into(), |k| format!("{}k",k*3/4)), + "-maxrate", &maxrate, "-bufsize", &buffer, "-qmin", &settings.cq().to_string(), "-async_depth", "1", "-aud", "1"] + .into_iter().map(str::to_owned).collect() } /// Round UP (+1ns float guard) so webrtc's floor(duration*90000) yields exactly `ticks`. @@ -121,12 +142,13 @@ async fn read_encoder( settings: VideoSettings, sequence: Arc, idr: Arc, ) -> Result<(), String> { // The sequence is shared so a CPU fallback or settings restart continues it (skip statistics stay correct). - if !gpu_encoder_requested() { return read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await; } - match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),true,sequence.clone(),settings,idr.clone()).await { + let encoder = Encoder::requested(); + if encoder == Encoder::Cpu { return read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await; } + match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,sequence.clone(),settings,idr.clone()).await { Ok(())=>Ok(()), Err(error) if !latest.is_closed()=>{ - log::warn!("NVENC stream unavailable; switching scoped encoder to CPU: {}",error); - read_encoder_with(size,display,portal,latest,false,sequence,settings,idr).await + log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name(),error); + read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await }, Err(error)=>Err(error), } @@ -167,7 +189,7 @@ fn rate_args(settings: &VideoSettings) -> (String, String) { } async fn read_encoder_with( - size: (u32,u32),display:String,portal:Option>,latest:mpsc::Sender>,gpu:bool,sequence:Arc, + size: (u32,u32),display:String,portal:Option>,latest:mpsc::Sender>,encoder:Encoder,sequence:Arc, mut settings: VideoSettings, #[cfg_attr(not(feature = "inproc-av"), allow(unused_variables))] idr: Arc, )->Result<(),String>{ let wayland=display.starts_with("wayland:"); @@ -178,11 +200,11 @@ async fn read_encoder_with( display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps), gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32, target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, - max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, gpu, + max_bps: max_kbps * 1000, buffer_bits: (max_kbps / 8).clamp(100, 500) * 1000, encoder, damage: crate::inproc_encoder::damage_enabled(), }; log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", - display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,if gpu{"h264_nvenc"}else{"libx264"}); + display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name()); return read_inprocess(config, latest, sequence, idr).await; } let size = format!("{}x{}", size.0, size.1); @@ -194,13 +216,14 @@ async fn read_encoder_with( let (maxrate, bufsize) = rate_args(&settings); let cq = settings.cq().to_string(); let target = settings.max_kbps().map_or("16M".to_string(), |k| format!("{}k", k * 3 / 4)); - log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,if gpu{"h264_nvenc"}else{"libx264"}); + log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,encoder.name()); // A bundled FFmpeg prefix (config ffmpeg_root / UU_FFMPEG_ROOT) or `ffmpeg` on PATH. let mut command=match crate::config::ffmpeg_root() { Some(root) => { let mut c=Command::new(root.join("bin/ffmpeg")); c.env("LD_LIBRARY_PATH",root.join("lib/x86_64-linux-gnu")); c } None => Command::new("ffmpeg"), }; command.args(["-hide_banner","-loglevel","error","-nostdin"]); + if encoder == Encoder::Vaapi { command.args(["-vaapi_device", &vaapi_device()]); } if wayland { command.args(["-f","rawvideo","-pixel_format","bgra","-video_size",&size, "-framerate",&fps,"-i","pipe:0","-an"]); @@ -210,10 +233,14 @@ async fn read_encoder_with( "-video_size",&size,"-i",&display,"-an"]); } // 其他分辨率: scale the captured desktop (the host display mode is left untouched). - if let Some((width, height)) = settings.output { + if encoder == Encoder::Vaapi { + command.args(["-vf", &vaapi_filter(settings.output)]); + } else if let Some((width, height)) = settings.output { command.args(["-vf", &format!("scale={width}:{height}:flags=bilinear")]); } - if gpu{ + if encoder == Encoder::Vaapi { + command.args(vaapi_options(&settings, &gop)); + } else if encoder == Encoder::Nvenc { command.args(["-c:v","h264_nvenc","-preset","p4","-tune","ull", "-profile:v","baseline","-pix_fmt","yuv420p","-g",&gop,"-bf","0", "-rc","vbr","-cq",&cq,"-b:v",&target,"-maxrate",&maxrate,"-bufsize",&bufsize, @@ -403,6 +430,20 @@ pub async fn stream_desktop(pc: Arc, track: Arc