From 708c770b19939d282e74223ff659539c55c4d575 Mon Sep 17 00:00:00 2001 From: VallierDev <24893073+VallierDev@users.noreply.github.com> Date: Fri, 16 Oct 2026 00:13:54 +0800 Subject: [PATCH] Support H.265 (HEVC), selected by the host when both sides support it One video track carries H.264 or H.265 (RFC 7798 payloader); the host announces its encoders with CONTROL Message 18 CodecNegotiation and picks H.265 when the viewer negotiated it (the official Mac viewer offers H265 first; CaptureSettingRequest field 13 only reports the current codec). hevc_nvenc / libx265 / hevc_vaapi with fallback to software HEVC, then H.264; UU_VIDEO_CODEC=h264 forces H.264. True color 4:4:4 applies to H.265. Verified with the official Mac viewer on NVENC. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 +- Cargo.lock | 4 +- Cargo.toml | 4 +- README.en.md | 10 ++ README.md | 8 + docs/release-notes/v0.1.16.md | 22 +++ src/assist.rs | 2 +- src/capability.rs | 24 ++- src/capture_setting.rs | 88 ++++++++++- src/hevc.rs | 72 +++++++++ src/inproc_encoder.rs | 20 ++- src/main.rs | 2 + src/video.rs | 277 +++++++++++++++++++++++++++------- src/video_track.rs | 173 +++++++++++++++++++++ src/webrtc.rs | 48 ++++-- 15 files changed, 672 insertions(+), 89 deletions(-) create mode 100644 docs/release-notes/v0.1.16.md create mode 100644 src/hevc.rs create mode 100644 src/video_track.rs diff --git a/AGENTS.md b/AGENTS.md index 2104c54..58024d2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -110,6 +110,7 @@ viewer does not reconnect them automatically. | `UU_TERMINAL_SHORTCUTS=0` | disable Ctrl+V → Ctrl+Shift+V in focused X11 terminals (Mac Cmd and Ctrl both arrive as VK_LCONTROL) | | `contrib/virtual-desktop/` | Xvfb + session (`UU_VIRTUAL_SESSION`, `UU_VIRTUAL_BUS=user|private`) for monitorless hosts; serve it with `DISPLAY=:99` | | `UU_VIDEO_ENCODER` | `cpu`: libx264; `vaapi`: explicit Intel/AMD; `nvenc`: force; unset: NVENC→CPU only if `/proc/driver/nvidia/version` or `/dev/nvidia0` exists, else CPU (#10) | +| `UU_VIDEO_CODEC=h264` | force H.264 even when the viewer negotiated H.265 (default: H.265 when both sides support it) | | `UU_FFMPEG_ROOT` | FFmpeg prefix (`bin/ffmpeg`, libs in `lib/x86_64-linux-gnu`) | | `UU_AUDIO=0`, `UU_AUDIO_SOURCE` | disable sound / choose the Pulse source (default `@DEFAULT_MONITOR@`) | | `UU_LOGIN_WINDOW=0` | don't auto-open the login window | @@ -126,7 +127,7 @@ every 60 s. | `src/signaling.rs` | Engine.IO v4 / Socket.IO client; `be-controlled` binary attachment = ConnectOptions (field 1 capture_type) | | `src/auth.rs`, `src/config.rs`, `src/login.rs`, `src/login_ui.rs` | signed REST client, config, device init + QR login, GTK window | | `src/webrtc.rs` | peer connection, data channels, CONTROL dispatch, initialization messages | -| `src/video.rs`, `src/capture_setting.rs` | FFmpeg H.264 pipeline; viewer quality/fps/resolution settings | +| `src/video.rs`, `src/capture_setting.rs`, `src/video_track.rs`, `src/hevc.rs` | FFmpeg H.264/H.265 pipeline, one video track for both codecs (RFC 7798 HEVC payloader); viewer quality/fps/resolution/chroma settings | | `src/audio.rs` | Pulse/PipeWire monitor → Opus audio track | | `src/input.rs`, `src/input_mapping.rs`, `src/uinput.rs` | input injection (X11 XTest / Wayland / portal / uinput), Windows VK → keysym → evdev | | `src/cursor.rs` | pointer shape mirroring (X11) | @@ -171,6 +172,10 @@ started a second full host server instead, so check that a subcommand exists in (field 3 = the chroma actually encoded). - ConnectOptions field 4 `decoder_cap_list` = DecoderCap {1 fps, 2 codec_type (1 H264, 2 H265), 3 width, 4 height, 5 chroma_format}. The Mac viewer lists H.264 and H.265, each 4:2:0 and 4:4:4, up to 3840x2160@120. +- Codec choice is the host's (official negotiateCodec): the Mac viewer offers H265 before H264 in SDP and + CaptureSettingRequest field 13 only reports its current codec. The host sends CONTROL Message 18 + CodecNegotiation {1 EncoderCap {1 capture_mode=1, 2 fps, 3 codec_type, 4 width, 5 height}} at CONTROL open and + selects H.265 when both m-line codecs are bound (`UU_VIDEO_CODEC=h264` forces H.264). - SystemMetrics (Message 13): GeneralMetric `2 system_sound_support` is an enum, **2 = supported** (1 = not supported); SettingMetric 1 keep_mute, 2 mute_permanent, 3 auto_lock, 5 file_transfer_save_location, 8/9/10 permission states (2 = allow). diff --git a/Cargo.lock b/Cargo.lock index 9bad1b9..31d2f6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2649,8 +2649,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uu-remote-linux" -version = "0.1.15" +version = "0.1.16" dependencies = [ + "async-trait", "base64 0.22.1", "bytes", "cc", @@ -2664,6 +2665,7 @@ dependencies = [ "log", "nix 0.29.0", "qrcode", + "rand 0.8.8", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 5e11382..1d3129c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "uu-remote-linux" -version = "0.1.15" +version = "0.1.16" edition = "2021" authors = ["uu-remote-linux contributors"] description = "Unofficial Linux host for NetEase UU Remote: control a Linux desktop from the official clients" @@ -36,6 +36,8 @@ hex = "0.4" base64 = "0.22" flate2 = "1.0" bytes = "1" +async-trait = "0.1" +rand = "0.8" webrtc = "0.10" libc = "0.2" nix = { version = "0.29", features = ["term", "process", "fs", "ioctl", "signal"] } diff --git a/README.en.md b/README.en.md index a9bf9f9..bf7aace 100644 --- a/README.en.md +++ b/README.en.md @@ -311,6 +311,16 @@ Released under the [GPL-3.0](LICENSE): you may use, modify and redistribute it, - [webrtc-rs](https://github.com/webrtc-rs/webrtc), FFmpeg, GTK and the other open-source projects this builds on. - Most of the code was written with the help of Claude and Codex. +### H.265 / HEVC + +When both sides support H.265 the host uses it (as the official host does: the host picks the codec from both +capability lists; the official Mac viewer lists H265 first in its SDP). The first desktop connection probes a +synthetic frame and announces encoder support with CodecNegotiation. `hevc_nvenc`, `libx265` and explicit +`UU_VIDEO_ENCODER=vaapi` (`hevc_vaapi`); hardware failure falls back to software HEVC, then to H.264. +`UU_VIDEO_CODEC=h264` forces H.264. 8-bit; true color 4:4:4 uses hevc_nvenc `rext` / libx265 `main444-8`. +Verified with the official Mac viewer on an RTX 3090 (hevc_nvenc, including 4:4:4); libx265, hevc_vaapi and +other viewers are not hardware-verified. + ### VAAPI (opt-in / 显式启用) `UU_VIDEO_ENCODER=vaapi` selects Intel/AMD H.264 VAAPI; `UU_VAAPI_DEVICE` defaults to diff --git a/README.md b/README.md index 452afef..53cfdb7 100644 --- a/README.md +++ b/README.md @@ -275,6 +275,14 @@ flowchart LR - [webrtc-rs](https://github.com/webrtc-rs/webrtc)、FFmpeg、GTK 等开源项目。 - 本项目的大部分代码在 Claude 和 Codex 的协助下完成。 +### H.265 / HEVC + +控制端和主机都支持 H.265 时,主机自动使用 H.265(与官方主机一致,由主机按双方能力选择;官方 Mac 客户端在 SDP +中优先列出 H.265)。首次桌面连接会用合成帧探测实际编码能力,并通过 CodecNegotiation 告知客户端。支持 `hevc_nvenc`、 +`libx265`、显式 `UU_VIDEO_ENCODER=vaapi` 的 `hevc_vaapi`;硬件失败先回落软件 HEVC,再回落 H.264。 +`UU_VIDEO_CODEC=h264` 强制使用 H.264。8-bit;真彩 4:4:4 使用 hevc_nvenc `rext` / libx265 `main444-8`。 +已用官方 Mac 客户端在 RTX 3090(hevc_nvenc,含 4:4:4)实机验收;libx265、hevc_vaapi 及其他客户端尚未实机验证。 + ### VAAPI 硬件编码(显式启用) 设置 `UU_VIDEO_ENCODER=vaapi` 使用 Intel/AMD H.264 VAAPI;`UU_VAAPI_DEVICE` 默认 diff --git a/docs/release-notes/v0.1.16.md b/docs/release-notes/v0.1.16.md new file mode 100644 index 0000000..a79c42d --- /dev/null +++ b/docs/release-notes/v0.1.16.md @@ -0,0 +1,22 @@ +支持 H.265(HEVC)。/ H.265 (HEVC) support. + +### 新功能 / Added +- **H.265**:控制端和主机都支持时自动使用 H.265,同等画质下码率更低。与官方主机一致,由主机按双方能力选择编码 + (官方 Mac 客户端没有手动切换编码的菜单)。连接时主机会告知客户端自己能编哪些格式(CodecNegotiation)。 + 支持 `hevc_nvenc`、`libx265` 和显式 VAAPI 的 `hevc_vaapi`;硬件失败先回落软件 HEVC,再回落 H.264。 + 需要时用 `UU_VIDEO_CODEC=h264` 强制使用 H.264。 +- 真彩(4:4:4)同样适用于 H.265(hevc_nvenc `rext` / libx265 `main444-8`)。 + ✅ 已用官方 Mac 客户端在 RTX 3090(hevc_nvenc,含 4:4:4)实机验收:画面正常、重连正常、无丢帧。 + ⚠️ libx265、hevc_vaapi、Windows/手机客户端尚未实机验证;如有问题请设置 `UU_VIDEO_CODEC=h264` 并提 issue。 + The host now uses H.265 when both sides support it (the host picks the codec, like the official host), announces + its encoders with CodecNegotiation, and falls back to software HEVC, then H.264. `UU_VIDEO_CODEC=h264` forces H.264. + Verified with the official Mac viewer on NVENC including 4:4:4; other encoders and viewers are unverified. + +### 修复 / Fixed +- 日志新增客户端 offer 中的编码列表与最终协商结果(仅编码名),便于排查画面问题。 + The log shows offered and negotiated video codecs (names only). + +### 下载 / Download +`uu-remote-linux-v0.1.16-x86_64-linux.tar.gz`:X11 版可执行文件(x86_64,glibc ≥ 2.35),附带 `contrib/`。 + +**非官方项目,与网易无关。** / **Unofficial, not affiliated with NetEase.** diff --git a/src/assist.rs b/src/assist.rs index 1ba1ce3..e1e3644 100644 --- a/src/assist.rs +++ b/src/assist.rs @@ -1,4 +1,4 @@ -//! Opt-in guest assistance. Protocol sources: docs/validation/remote-assist-evidence.md. +//! Opt-in guest assistance. Protocol sources: AGENTS.md "Remote assistance". //! Secret values never implement Display/Debug or enter diagnostics. use crate::auth::{Credentials, RoomData, UUAuthClient}; use serde_json::{json, Value}; diff --git a/src/capability.rs b/src/capability.rs index 345926f..c30887c 100644 --- a/src/capability.rs +++ b/src/capability.rs @@ -27,17 +27,20 @@ pub fn h264_device_capability( fps: i32, codec_impl: i32, ) -> Value { + device_capability(client_id, ice_id, display_id, fps, codec_impl, false) +} + +pub fn device_capability(client_id: &str, ice_id: &str, display_id: i32, fps: i32, codec_impl: i32, hevc: bool) -> Value { + let mut codecs = vec![json!({"video_codec": VIDEO_CODEC_H264, "chroma_sampling": 1, + "codec_impl": codec_impl, "bit_depth": 8})]; + // Official VideoCodecCapability::SerializeJson uses the same fields for CodecType H265=2. + if hevc { codecs.push(json!({"video_codec": 2, "chroma_sampling": 1, "codec_impl": codec_impl, "bit_depth": 8})); } json!({ "client_id": client_id, "type": "device_capability", "data": { "display_info": [{"id": display_id, "fps": fps, "type": 0, "hdr": 0}], - "video_codec_capability": [{ - "video_codec": VIDEO_CODEC_H264, - "chroma_sampling": 1, - "codec_impl": codec_impl, - "bit_depth": 8 - }], + "video_codec_capability": codecs, "ice_id": ice_id } }) @@ -47,6 +50,15 @@ pub fn h264_device_capability( mod tests { use super::*; + #[test] + fn hevc_capability_uses_official_codec_enum_and_420_only() { + let payload = device_capability("client-example", "ice-example", 0, 60, 5, true); + assert_eq!(payload["data"]["video_codec_capability"], json!([ + {"video_codec":1,"chroma_sampling":1,"codec_impl":5,"bit_depth":8}, + {"video_codec":2,"chroma_sampling":1,"codec_impl":5,"bit_depth":8}])); + assert_eq!(device_capability("c", "i", 0, 60, 5, false)["data"]["video_codec_capability"].as_array().unwrap().len(), 1); + } + #[test] fn ice_finished_keeps_string_identifiers_and_exact_fields() { let payload = ice_finished("ice-example", "control-example"); diff --git a/src/capture_setting.rs b/src/capture_setting.rs index 582040f..9f2ed6b 100644 --- a/src/capture_setting.rs +++ b/src/capture_setting.rs @@ -30,9 +30,18 @@ pub const CHROMA_444: u64 = 3; const MIN_FPS: u32 = 5; const MAX_FPS: u32 = 144; +/// Official GvPb_CodecType, CaptureSettingRequest.codec_type (field 13). +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)] +pub enum VideoCodec { #[default] H264, H265 } +impl VideoCodec { + pub fn wire(self) -> u64 { match self { Self::H264 => 1, Self::H265 => 2 } } + pub fn mime(self) -> &'static str { match self { Self::H264 => "video/H264", Self::H265 => "video/H265" } } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct VideoSettings { pub fps: u32, + pub codec: VideoCodec, pub quality: u64, /// Viewer max_custom_bitrate (only used for QUALITY_CUSTOM): Mbit/s, or kbit/s above 1000. pub custom_mbps: u32, @@ -43,7 +52,7 @@ pub struct VideoSettings { } impl Default for VideoSettings { - fn default() -> Self { Self { fps: 60, quality: QUALITY_AUTO, custom_mbps: 0, output: None, chroma: CHROMA_420 } } + fn default() -> Self { Self { codec: VideoCodec::H264, fps: 60, quality: QUALITY_AUTO, custom_mbps: 0, output: None, chroma: CHROMA_420 } } } /// Selectable smaller stream sizes with the desktop's aspect ratio (even dimensions). @@ -102,10 +111,21 @@ fn fps_from_option(value: u64) -> Option { /// Applies a capture-setting RPC to `settings` and returns the RpcResponse payload (Message 22), /// or None when the RPC is not a capture/fps/quality setting. pub fn handle_rpc(rpc: &[WireField], settings: &watch::Sender, native: (u32, u32)) -> Option> { - apply_rpc(rpc, settings, native, crate::video::full_chroma_available()) + handle_rpc_with_codecs(rpc, settings, native, true, false) } -fn apply_rpc(rpc: &[WireField], settings: &watch::Sender, native: (u32, u32), full_chroma: bool) -> Option> { +/// Initial codec once the m-line is negotiated: H.265 when both sides support it (the Mac viewer +/// lists H265 first in its offer), unless UU_VIDEO_CODEC=h264. +pub fn preferred_codec(h264: bool, h265: bool) -> VideoCodec { + let forced_h264 = std::env::var("UU_VIDEO_CODEC").is_ok_and(|v| v.eq_ignore_ascii_case("h264")); + if h265 && !(forced_h264 && h264) { VideoCodec::H265 } else { VideoCodec::H264 } +} + +pub fn handle_rpc_with_codecs(rpc: &[WireField], settings: &watch::Sender, native: (u32, u32), h264: bool, h265: bool) -> Option> { + apply_rpc(rpc, settings, native, h264, h265, crate::video::full_chroma_available()) +} + +fn apply_rpc(rpc: &[WireField], settings: &watch::Sender, native: (u32, u32), h264: bool, h265: bool, full_chroma: bool) -> Option> { let header = message(rpc, 1).unwrap_or(&[]); let mut next = *settings.borrow(); if let Some(request) = message(rpc, 2) { @@ -113,6 +133,10 @@ fn apply_rpc(rpc: &[WireField], settings: &watch::Sender, native: // Numbers only: fps, quality, bitrate, resolution and flags carry no user content. log::info!("Capture setting request: {:?}", fields.iter().filter_map(|f| match f.value { WireValue::Varint(v) => Some((f.number, v)), _ => None }).collect::>()); + // codec_type 13 (1 H264, 2 H265): like the official host (negotiateCodec), the host picks the + // codec from both capability lists; the Mac viewer sends its current codec (1 while receiving + // H.264), so only an explicit H.265 request switches, never one absent from the m-line. + if number(&fields, 13) == Some(2) && h265 { next.codec = VideoCodec::H265; } // Field 1 is the viewer's fps option index (1 = 30, 2 = 60, ...); 18 fpsCount is the frame rate. if let Some(fps) = number(&fields, 18).filter(|v| *v > 0).or_else(|| number(&fields, 1).and_then(fps_from_option)) { next.fps = fps as u32; } if let Some(quality) = number(&fields, 2).filter(|v| *v > 0) { next.quality = quality; } @@ -183,6 +207,23 @@ pub fn sync_payload(settings: &VideoSettings, native: (u32, u32)) -> Vec { out } +/// Message 18 CodecNegotiation {1 encoder_cap_list: EncoderCap {1 capture_mode (1 DESKTOP), 2 fps, +/// 3 codec_type, 4 width, 5 height}} (official Windows host descriptor). The viewer's own list is +/// ConnectOptions field 4 decoder_cap_list; it only selects a codec the host announces here. +pub fn codec_negotiation(size: (u32, u32), fps: u32, hevc: bool) -> Vec { + let mut out = Vec::new(); + for codec in [Some(VideoCodec::H264), hevc.then_some(VideoCodec::H265)].into_iter().flatten() { + let mut cap = Vec::new(); + put_varint(1, 1, &mut cap); + put_varint(2, u64::from(fps), &mut cap); + put_varint(3, codec.wire(), &mut cap); + put_varint(4, u64::from(size.0), &mut cap); + put_varint(5, u64::from(size.1), &mut cap); + put_bytes(1, &cap, &mut out); + } + out +} + #[cfg(test)] mod tests { use super::*; @@ -205,7 +246,7 @@ mod tests { put_varint(18, 30, &mut body); let request = rpc(77, 2, &body); let response = handle_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080)).unwrap(); - assert_eq!(*rx.borrow(), VideoSettings { fps: 30, quality: QUALITY_GENERAL, custom_mbps: 0, output: None, chroma: CHROMA_420 }); + assert_eq!(*rx.borrow(), VideoSettings { codec: VideoCodec::H264, fps: 30, quality: QUALITY_GENERAL, custom_mbps: 0, output: None, chroma: CHROMA_420 }); assert_eq!(rx.borrow().max_kbps(), Some(2_000)); let fields = decode_wire(&response).unwrap(); assert_eq!(message(&fields, 1), Some(&[8u8, 77][..])); @@ -231,15 +272,24 @@ mod tests { let (tx, rx) = watch::channel(VideoSettings::default()); // Observed Mac viewer request with 真彩 on: field 7 = 3. let request = rpc(5, 2, &[8, 2, 16, 4, 56, 3, 144, 1, 60]); - apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), false).unwrap(); + apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true, false, false).unwrap(); assert_eq!(rx.borrow().chroma, CHROMA_420); - apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true).unwrap(); + apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true, false, true).unwrap(); assert_eq!(rx.borrow().chroma, CHROMA_444); assert_eq!(number(&decode_wire(&sync_payload(&rx.borrow(), (1920, 1080))).unwrap(), 3), Some(CHROMA_444)); - apply_rpc(&decode_wire(&rpc(6, 2, &[56, 1])).unwrap(), &tx, (1920, 1080), true).unwrap(); + apply_rpc(&decode_wire(&rpc(6, 2, &[56, 1])).unwrap(), &tx, (1920, 1080), true, false, true).unwrap(); assert_eq!(rx.borrow().chroma, CHROMA_420); } + #[test] + fn codec_negotiation_lists_h264_then_optional_h265() { + assert_eq!(codec_negotiation((1920, 1080), 60, false), [10, 12, 8, 1, 16, 60, 24, 1, 32, 128, 15, 40, 184, 8]); + let both = codec_negotiation((1920, 1080), 60, true); + let caps: Vec<_> = decode_wire(&both).unwrap().iter().filter_map(|f| match f.value { WireValue::Bytes(b) => Some(b), _ => None }) + .map(|b| number(&decode_wire(b).unwrap(), 3)).collect(); + assert_eq!(caps, [Some(1), Some(2)]); + } + #[test] fn resolution_requests_scale_below_native_only() { let (tx, rx) = watch::channel(VideoSettings::default()); @@ -267,3 +317,27 @@ mod tests { assert_eq!(number(&fields, 10), Some(1080)); } } + +#[cfg(test)] +mod codec_tests { + use super::*; + #[test] + fn codec_is_opt_in_and_requires_negotiated_support() { + let (tx, rx) = watch::channel(VideoSettings::default()); + let rpc = |value| decode_wire(if value == 2 { &[18, 2, 104, 2] } else { &[18, 2, 104, 1] }).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H264); + handle_rpc_with_codecs(&rpc(2), &tx, (1920,1080), true, false).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H264); + handle_rpc_with_codecs(&rpc(2), &tx, (1920,1080), true, true).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H265); + handle_rpc_with_codecs(&rpc(1), &tx, (1920,1080), false, true).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H265); // unnegotiated H264 is never sent + // An unrelated FPS RPC must preserve the explicit codec selection. + handle_rpc_with_codecs(&decode_wire(&[26,2,8,1]).unwrap(), &tx, (1920,1080), true, true).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H265); + // The viewer reports its current codec (1) in every request; that must not undo H.265. + handle_rpc_with_codecs(&rpc(1), &tx, (1920,1080), true, true).unwrap(); + assert_eq!(rx.borrow().codec, VideoCodec::H265); + assert_eq!(preferred_codec(true, false), VideoCodec::H264); + } +} diff --git a/src/hevc.rs b/src/hevc.rs new file mode 100644 index 0000000..fad3a33 --- /dev/null +++ b/src/hevc.rs @@ -0,0 +1,72 @@ +//! RFC 7798 §4.4.1 / §4.4.3, single-layer non-interleaved HEVC (no DONL). +//! Parameter sets travel in-band before each IDR; no aggregation is required. +use bytes::Bytes; + +pub fn payload(mtu: usize, annex_b: &[u8]) -> Result, String> { + if mtu < 4 { return Err("HEVC payload MTU too small".into()); } + let mut starts = Vec::new(); + let mut i = 0; + while i + 3 <= annex_b.len() { + let prefix = if annex_b[i..].starts_with(&[0, 0, 0, 1]) { 4 } + else if annex_b[i..].starts_with(&[0, 0, 1]) { 3 } else { i += 1; continue; }; + starts.push((i, i + prefix)); i += prefix; + } + if starts.is_empty() { return Err("HEVC access unit has no Annex-B start code".into()); } + let mut out = Vec::new(); + for (index, &(_, start)) in starts.iter().enumerate() { + let mut end = starts.get(index + 1).map_or(annex_b.len(), |s| s.0); + while end > start && annex_b[end - 1] == 0 { end -= 1; } // trailing_zero_8bits + let nal = &annex_b[start..end]; + if nal.len() < 2 || nal[0] & 0x80 != 0 || nal[1] & 7 == 0 { + return Err("Invalid HEVC NAL header".into()); + } + let kind = (nal[0] >> 1) & 63; + if kind >= 48 { return Err("HEVC Annex-B contains RTP packet type".into()); } + if nal.len() <= mtu { out.push(Bytes::copy_from_slice(nal)); continue; } + let chunks = nal[2..].chunks(mtu - 3); + let last = chunks.len() - 1; + for (n, chunk) in chunks.enumerate() { + let mut fu = Vec::with_capacity(3 + chunk.len()); + // Keep F, LayerId and TID, replacing only the NAL type with FU=49. + fu.extend_from_slice(&[(nal[0] & 0x81) | (49 << 1), nal[1], + kind | if n == 0 { 0x80 } else { 0 } | if n == last { 0x40 } else { 0 }]); + fu.extend_from_slice(chunk); out.push(fu.into()); + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn parameter_sets_and_small_nals_remain_in_order_without_start_codes() { + assert_eq!(payload(1200, &[0,0,0,1,64,1,9,0,0,1,66,1,8,0,0,1,38,1,7]).unwrap(), + vec![Bytes::from_static(&[64,1,9]), Bytes::from_static(&[66,1,8]), Bytes::from_static(&[38,1,7])]); + } + #[test] + fn fu_round_trip_preserves_header_payload_and_mtu_boundaries() { + for mtu in [4, 5, 7, 1190] { + let header = [0x27, 0x2b]; // IDR type19, nonzero layer and temporal id + let data: Vec = (0..2501).map(|i| (i % 251 + 1) as u8).collect(); + let mut unit = vec![0,0,0,1]; unit.extend(header); unit.extend(&data); + let packets = payload(mtu, &unit).unwrap(); + assert!(packets.len() > 1); + let mut rebuilt = vec![(packets[0][0] & 0x81) | ((packets[0][2] & 63) << 1), packets[0][1]]; + for (i, p) in packets.iter().enumerate() { + assert!(p.len() <= mtu); assert_eq!((p[0] >> 1) & 63, 49); + assert_eq!(p[2] & 0x80 != 0, i == 0); + assert_eq!(p[2] & 0x40 != 0, i + 1 == packets.len()); + rebuilt.extend_from_slice(&p[3..]); + } + assert_eq!(&rebuilt[..2], &header); assert_eq!(&rebuilt[2..], &data); + } + } + #[test] + fn rejects_invalid_headers_and_too_small_mtu() { + for bytes in [&[0,0,1,64][..], &[0,0,1,64,0][..], &[0,0,1,128,1][..], &[64,1][..]] { + assert!(payload(1200, bytes).is_err()); + } + assert!(payload(3, &[0,0,1,64,1]).is_err()); + } +} diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index dfbd7cd..d906245 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -1,4 +1,4 @@ -//! In-process X11 capture and H.264 encoding through libavdevice/libavcodec. +//! In-process X11 capture and H.264/HEVC encoding through libavdevice/libavcodec. //! It replaces the `ffmpeg` child process with the same codec options, so the viewer //! still receives the same Annex-B access units (AUD, SPS/PPS, IDR, P). use std::ffi::CString; @@ -25,6 +25,7 @@ pub struct Config { pub max_bps: i64, pub buffer_bits: i64, pub encoder: crate::video::Encoder, + pub codec: crate::capture_setting::VideoCodec, /// Capture on XDamage changes (XShm) instead of x11grab; `UU_CAPTURE=grab` turns it off. pub damage: bool, } @@ -71,11 +72,11 @@ fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender } } -/// Opens the H.264 encoder for `config`. Returns the encoder, its input pixel format and output size. +/// Opens the selected video encoder for `config`. Returns the encoder, its input pixel format and output size. fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), String> { let (width, height) = config.output.unwrap_or(config.input); - let codec = encoder::find_by_name(config.encoder.name()) - .ok_or("H.264 encoder is not available in this FFmpeg build")?; + let codec = encoder::find_by_name(config.encoder.name_for(config.codec)) + .ok_or_else(|| format!("{} is not available in this FFmpeg build", config.encoder.name_for(config.codec)))?; // Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one. let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?; context.set_width(width); @@ -99,15 +100,22 @@ fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), St let cq = config.cq.to_string(); let gop = config.gop.to_string(); let mut options = Dictionary::new(); + let hevc = config.codec == crate::capture_setting::VideoCodec::H265; if config.encoder == crate::video::Encoder::Vaapi { - for (key,value) in [("profile", "constrained_baseline"), ("rc_mode", "VBR"), ("async_depth", "1"), ("aud", "1")] { options.set(key,value); } + for (key,value) in [("profile", if hevc { "main" } else { "constrained_baseline" }), ("rc_mode", "VBR"), ("async_depth", "1"), ("aud", "1")] { options.set(key,value); } options.set("qmin", &cq); } else if config.encoder == crate::video::Encoder::Nvenc { - for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"), + for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", if hevc { "main" } else { "baseline" }), ("rc", "vbr"), ("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] { options.set(key, value); } options.set("cq", &cq); + } else if hevc { + for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "main"), ("threads", "4"), ("forced-idr", "1")] { + options.set(key, value); + } + options.set("crf", &cq); + options.set("x265-params", &crate::video::x265_params(&gop)); } else { let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}"); for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "baseline"), ("threads", "4")] { diff --git a/src/main.rs b/src/main.rs index 6e36ede..00b6290 100644 --- a/src/main.rs +++ b/src/main.rs @@ -24,6 +24,8 @@ mod signaling; mod terminal; mod webrtc; mod video; +mod video_track; +mod hevc; #[cfg(feature = "inproc-av")] mod damage_capture; #[cfg(feature = "inproc-av")] diff --git a/src/video.rs b/src/video.rs index e8d9578..7f7f7ca 100644 --- a/src/video.rs +++ b/src/video.rs @@ -7,8 +7,7 @@ use tokio::sync::mpsc; use webrtc::media::Sample; use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState; use webrtc::peer_connection::RTCPeerConnection; -use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample; -use crate::capture_setting::VideoSettings; +use crate::capture_setting::{VideoCodec, VideoSettings}; /// Default frame rate; the viewer's 帧率 setting overrides it per connection. pub const FPS: u64 = 60; @@ -28,24 +27,42 @@ fn gop_frames(fps: u32, in_process: bool) -> u64 { } const MAX_ACCESS_UNIT: usize = 8 * 1024 * 1024; -fn nal_offsets(bytes: &[u8], kind: u8) -> Vec { +fn nal_offsets_for(bytes: &[u8], kind: u8, codec: VideoCodec) -> Vec { let mut result = Vec::new(); let mut i = 0; while i + 3 < bytes.len() { let prefix = if bytes[i..].starts_with(&[0, 0, 0, 1]) { 4 } else if bytes[i..].starts_with(&[0, 0, 1]) { 3 } else { i += 1; continue }; - if i + prefix < bytes.len() && bytes[i + prefix] & 31 == kind { result.push(i); } + let header = i + prefix; + let matches = match codec { + VideoCodec::H264 => header < bytes.len() && bytes[header] & 31 == kind, + // HEVC has a two-byte header: forbidden_zero_bit=0, temporal_id_plus1 != 0. + VideoCodec::H265 => header + 1 < bytes.len() && bytes[header] & 128 == 0 + && bytes[header + 1] & 7 != 0 && (bytes[header] >> 1) & 63 == kind, + }; + if matches { result.push(i); } i += prefix; } result } -fn aud_offsets(bytes: &[u8]) -> Vec { nal_offsets(bytes, 9) } -fn is_independent_frame(bytes: &[u8]) -> bool { - !nal_offsets(bytes, 5).is_empty() && nal_offsets(bytes, 1).is_empty() +fn aud_offsets_for(bytes: &[u8], codec: VideoCodec) -> Vec { + nal_offsets_for(bytes, if codec == VideoCodec::H265 { 35 } else { 9 }, codec) } +fn is_independent_frame_for(bytes: &[u8], codec: VideoCodec) -> bool { + match codec { + VideoCodec::H264 => !nal_offsets_for(bytes, 5, codec).is_empty() && nal_offsets_for(bytes, 1, codec).is_empty(), + // BLA/IDR/CRA are IRAP (RFC 7798 §4.1); no leading pictures with our no-B-frame encoder. + VideoCodec::H265 => (16..=21).any(|kind| !nal_offsets_for(bytes, kind, codec).is_empty()) + && !(0..16).any(|kind| !nal_offsets_for(bytes, kind, codec).is_empty()), + } +} +#[cfg(test)] +fn aud_offsets(bytes: &[u8]) -> Vec { aud_offsets_for(bytes, VideoCodec::H264) } +fn is_independent_frame(bytes: &[u8]) -> bool { is_independent_frame_for(bytes, VideoCodec::H264) } #[derive(Debug)] struct CompleteFrame { + codec: VideoCodec, sequence: u64, completed_at: Instant, data: Vec, @@ -63,7 +80,13 @@ impl Encoder { match value { Some("cpu") => Self::Cpu, Some("vaapi") => Self::Vaapi, Some("nvenc") => Self::Nvenc, _ => if nvidia { Self::Nvenc } else { Self::Cpu } } } - pub(crate) fn name(self) -> &'static str { match self { Self::Cpu => "libx264", Self::Nvenc => "h264_nvenc", Self::Vaapi => "h264_vaapi" } } + pub(crate) fn name_for(self, codec: VideoCodec) -> &'static str { + match (self, codec) { + (Self::Cpu, VideoCodec::H264) => "libx264", (Self::Cpu, VideoCodec::H265) => "libx265", + (Self::Nvenc, VideoCodec::H264) => "h264_nvenc", (Self::Nvenc, VideoCodec::H265) => "hevc_nvenc", + (Self::Vaapi, VideoCodec::H264) => "h264_vaapi", (Self::Vaapi, VideoCodec::H265) => "hevc_vaapi", + } + } } fn nvidia_present() -> bool { std::path::Path::new("/proc/driver/nvidia/version").exists() || std::path::Path::new("/dev/nvidia0").exists() @@ -76,12 +99,14 @@ pub fn full_chroma_available() -> bool { Encoder::requested() != Encoder::Vaapi } -/// (-profile:v, -pix_fmt) for NVENC / libx264: 4:4:4 needs a High 4:4:4 profile. +/// (-profile:v, -pix_fmt) for NVENC / libx264 / libx265: 4:4:4 (真彩) needs a 4:4:4 profile. fn chroma_args(settings: &VideoSettings, encoder: Encoder) -> (&'static str, &'static str) { - match (settings.chroma == crate::capture_setting::CHROMA_444, encoder) { - (true, Encoder::Nvenc) => ("high444p", "yuv444p"), - (true, _) => ("high444", "yuv444p"), - (false, _) => ("baseline", "yuv420p"), + if settings.chroma != crate::capture_setting::CHROMA_444 { return (profile_for(settings.codec, false), "yuv420p"); } + match (settings.codec, encoder) { + (VideoCodec::H264, Encoder::Nvenc) => ("high444p", "yuv444p"), + (VideoCodec::H264, _) => ("high444", "yuv444p"), + (VideoCodec::H265, Encoder::Nvenc) => ("rext", "yuv444p"), + (VideoCodec::H265, _) => ("main444-8", "yuv444p"), } } pub(crate) fn vaapi_device() -> String { std::env::var("UU_VAAPI_DEVICE").unwrap_or_else(|_| "/dev/dri/renderD128".into()) } @@ -95,12 +120,86 @@ fn vaapi_options(settings: &VideoSettings, gop: &str) -> Vec { let (maxrate, buffer) = rate_args(settings, Encoder::Vaapi); // QVBR is not uniformly available on Intel/AMD drivers. VBR + qmin sets a quality // floor while retaining the viewer's bitrate cap; unsupported drivers fall back to CPU. - ["-c:v", "h264_vaapi", "-profile:v", "constrained_baseline", "-g", gop, "-bf", "0", + ["-c:v", Encoder::Vaapi.name_for(settings.codec), "-profile:v", profile_for(settings.codec, true), "-g", gop, "-bf", "0", "-rc_mode", "VBR", "-b:v", &settings.max_kbps().map_or("16M".into(), |k| format!("{}k",k*3/4)), "-maxrate", &maxrate, "-bufsize", &buffer, "-qmin", &settings.cq().to_string(), "-async_depth", "1", "-aud", "1"] .into_iter().map(str::to_owned).collect() } +fn profile_for(codec: VideoCodec, vaapi: bool) -> &'static str { + match codec { VideoCodec::H265 => "main", VideoCodec::H264 => if vaapi { "constrained_baseline" } else { "baseline" } } +} +pub(crate) fn x265_params(gop: &str) -> String { + // Explicit no-B/no-lookahead avoids delayed frames; repeat parameter sets at each IRAP. + format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}:open-gop=0:bframes=0:rc-lookahead=0:frame-threads=1:pools=4") +} +fn cpu_hevc_options(settings: &VideoSettings, gop: &str) -> Vec { + let (maxrate, buffer) = rate_args(settings, Encoder::Cpu); + let (profile, pix_fmt) = chroma_args(settings, Encoder::Cpu); + ["-c:v", "libx265", "-preset", "superfast", "-tune", "zerolatency", "-profile:v", profile, + "-pix_fmt", pix_fmt, "-g", gop, "-bf", "0", "-threads", "4", "-crf", &settings.cq().to_string(), + "-maxrate", &maxrate, "-bufsize", &buffer, "-x265-params", &x265_params(gop)] + .into_iter().map(str::to_owned).collect() +} + +fn encoder_options(settings: &VideoSettings, encoder: Encoder, gop: &str) -> Vec { + let (maxrate, bufsize) = rate_args(settings, encoder); + let cq = settings.cq().to_string(); + let target = settings.max_kbps().map_or("16M".to_string(), |k| format!("{}k", k * 3 / 4)); + if encoder == Encoder::Vaapi { return vaapi_options(settings, gop); } + let (profile, pix_fmt) = chroma_args(settings, encoder); + if encoder == Encoder::Nvenc { + return ["-c:v", encoder.name_for(settings.codec), "-preset", "p4", "-tune", "ull", + "-profile:v", profile, "-pix_fmt", pix_fmt, "-g", gop, "-bf", "0", + "-rc", "vbr", "-cq", &cq, "-b:v", &target, "-maxrate", &maxrate, "-bufsize", &bufsize, + "-rc-lookahead", "0", "-zerolatency", "1", "-delay", "0", "-aud", "1"] + .into_iter().map(str::to_owned).collect(); + } + if settings.codec == VideoCodec::H265 { return cpu_hevc_options(settings, gop); } + let params = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}"); + ["-c:v", "libx264", "-preset", "superfast", "-tune", "zerolatency", + "-profile:v", profile, "-pix_fmt", pix_fmt, "-g", gop, "-threads", "4", + "-crf", &cq, "-maxrate", &maxrate, "-bufsize", &bufsize, "-x264-params", ¶ms] + .into_iter().map(str::to_owned).collect() +} + +fn ffmpeg_command() -> Command { + match crate::config::ffmpeg_root() { + Some(root) => { let mut command = Command::new(root.join("bin/ffmpeg")); + command.env("LD_LIBRARY_PATH", root.join("lib/x86_64-linux-gnu")); command } + None => Command::new("ffmpeg"), + } +} + +/// A compiled-in encoder name does not prove that its driver can initialize. +/// Probe one synthetic 4:2:0 frame once per process before advertising HEVC. +pub async fn hevc_available() -> bool { + static AVAILABLE: tokio::sync::OnceCell = tokio::sync::OnceCell::const_new(); + *AVAILABLE.get_or_init(|| async { + let requested = Encoder::requested(); + let mut candidates = vec![requested]; + if requested != Encoder::Cpu { candidates.push(Encoder::Cpu); } + for encoder in candidates { + let settings = VideoSettings { codec: VideoCodec::H265, ..Default::default() }; + let mut command = ffmpeg_command(); + command.args(["-hide_banner", "-loglevel", "error", "-nostdin"]); + if encoder == Encoder::Vaapi { command.args(["-vaapi_device", &vaapi_device()]); } + command.args(["-f", "lavfi", "-i", "color=black:size=256x256:rate=1", "-frames:v", "1", "-an"]); + if encoder == Encoder::Vaapi { command.args(["-vf", &vaapi_filter(None)]); } + command.args(encoder_options(&settings, encoder, "30")); + command.args(["-f", "hevc", "pipe:1"]).kill_on_drop(true).stderr(std::process::Stdio::null()); + if let Ok(Ok(output)) = tokio::time::timeout(Duration::from_secs(5), command.output()).await { + if output.status.success() && is_independent_frame_for(&output.stdout, VideoCodec::H265) { + log::info!("HEVC encoder probe: {} ok", encoder.name_for(VideoCodec::H265)); + return true; + } + } + } + log::info!("HEVC encoder probe: unavailable"); + false + }).await +} + /// Round UP (+1ns float guard) so webrtc's floor(duration*90000) yields exactly `ticks`. fn ticks_to_duration(ticks: u64) -> Duration { Duration::from_nanos((ticks * 1_000_000_000 + RTP_CLOCK - 1) / RTP_CLOCK + 1) @@ -131,7 +230,7 @@ struct DependencyGate {needs_idr:bool} impl Default for DependencyGate {fn default()->Self{Self{needs_idr:true}}} impl DependencyGate{ fn submit(&mut self,tx:&mpsc::Sender>,frame:Arc){ - if self.needs_idr&&!is_independent_frame(&frame.data){return;} + if self.needs_idr&&!is_independent_frame_for(&frame.data, frame.codec){return;} match tx.try_send(frame){ Ok(())=>self.needs_idr=false, Err(mpsc::error::TrySendError::Full(_))=>self.needs_idr=true, @@ -169,7 +268,7 @@ async fn read_encoder( match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,sequence.clone(),settings,idr.clone()).await { Ok(())=>Ok(()), Err(error) if !latest.is_closed()=>{ - log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name(),error); + log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name_for(settings.codec),error); read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await }, Err(error)=>Err(error), @@ -182,6 +281,7 @@ async fn read_inprocess( config: crate::inproc_encoder::Config, latest: mpsc::Sender>, sequence: Arc, idr: Arc, ) -> Result<(), String> { + let codec = config.codec; let (units_tx, mut units) = mpsc::channel::>(4); let stop = Arc::new(AtomicBool::new(false)); // If this future is dropped (reader task aborted), the blocking encoder stops at its next packet. @@ -193,7 +293,7 @@ async fn read_inprocess( let mut gate = DependencyGate::default(); while let Some(data) = units.recv().await { let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; - gate.submit(&latest, Arc::new(CompleteFrame { sequence, completed_at: Instant::now(), data })); + gate.submit(&latest, Arc::new(CompleteFrame { codec, sequence, completed_at: Instant::now(), data })); if latest.is_closed() { break; } } // Dropping the receiver also unblocks the encoder if it is waiting to send. @@ -211,9 +311,15 @@ fn buffer_kbits(max_kbps: u32, encoder: Encoder) -> u32 { if encoder == Encoder::Cpu { max_kbps.clamp(500, 8000) } else { (max_kbps / 8).clamp(100, 500) } } +fn buffer_for(settings: &VideoSettings, encoder: Encoder) -> u32 { + if settings.codec == VideoCodec::H265 && encoder == Encoder::Cpu { + settings.max_kbps().unwrap_or(30_000) // one second of HEVC bitrate, including high presets + } else { buffer_kbits(settings.max_kbps().unwrap_or(30_000), encoder) } +} + fn rate_args(settings: &VideoSettings, encoder: Encoder) -> (String, String) { let max_kbps = settings.max_kbps().unwrap_or(30_000); - (format!("{max_kbps}k"), format!("{}k", buffer_kbits(max_kbps, encoder))) + (format!("{max_kbps}k"), format!("{}k", buffer_for(settings, encoder))) } async fn read_encoder_with( @@ -234,27 +340,22 @@ async fn read_encoder_with( display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps), gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32, target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, - max_bps: max_kbps * 1000, buffer_bits: i64::from(buffer_kbits(max_kbps as u32, encoder)) * 1000, encoder, + max_bps: max_kbps * 1000, buffer_bits: i64::from(buffer_for(&settings, encoder)) * 1000, encoder, codec: settings.codec, damage: crate::inproc_encoder::damage_enabled(), }; log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", - display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name()); + display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name_for(settings.codec)); return read_inprocess(config, latest, sequence, idr).await; } let size = format!("{}x{}", size.0, size.1); let fps = settings.fps.to_string(); // Keep the IDR interval at 2 s whatever the frame rate. let gop = gop_frames(settings.fps, false).to_string(); - let (maxrate, bufsize) = rate_args(&settings, encoder); + let (maxrate, _) = rate_args(&settings, encoder); let cq = settings.cq().to_string(); - let target = settings.max_kbps().map_or("16M".to_string(), |k| format!("{}k", k * 3 / 4)); - let (profile, pix_fmt) = chroma_args(&settings, encoder); - log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} pix_fmt={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,encoder.name(),if encoder == Encoder::Vaapi { "nv12" } else { pix_fmt }); + log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} pix_fmt={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,encoder.name_for(settings.codec),if encoder == Encoder::Vaapi { "nv12" } else { chroma_args(&settings, encoder).1 }); // A bundled FFmpeg prefix (config ffmpeg_root / UU_FFMPEG_ROOT) or `ffmpeg` on PATH. - let mut command=match crate::config::ffmpeg_root() { - Some(root) => { let mut c=Command::new(root.join("bin/ffmpeg")); c.env("LD_LIBRARY_PATH",root.join("lib/x86_64-linux-gnu")); c } - None => Command::new("ffmpeg"), - }; + let mut command = ffmpeg_command(); command.args(["-hide_banner","-loglevel","error","-nostdin"]); if encoder == Encoder::Vaapi { command.args(["-vaapi_device", &vaapi_device()]); } if wayland { @@ -271,27 +372,13 @@ async fn read_encoder_with( } else if let Some((width, height)) = settings.output { command.args(["-vf", &format!("scale={width}:{height}:flags=bilinear")]); } - if encoder == Encoder::Vaapi { - command.args(vaapi_options(&settings, &gop)); - } else if encoder == Encoder::Nvenc { - command.args(["-c:v","h264_nvenc","-preset","p4","-tune","ull", - "-profile:v",profile,"-pix_fmt",pix_fmt,"-g",&gop,"-bf","0", - "-rc","vbr","-cq",&cq,"-b:v",&target,"-maxrate",&maxrate,"-bufsize",&bufsize, - "-rc-lookahead","0","-zerolatency","1","-delay","0","-aud","1"]); - }else{ - let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}"); - command.args([ - "-c:v", "libx264", "-preset", "superfast", "-tune", "zerolatency", - "-profile:v", profile, "-pix_fmt", pix_fmt, "-g", &gop, "-threads", "4", - "-crf", &cq, "-maxrate", &maxrate, "-bufsize", &bufsize, - "-x264-params", &x264]); - } + command.args(encoder_options(&settings, encoder, &gop)); // x11grab paces against an absolute schedule; after a capture stall it grabs // back-to-back to "catch up" (measured 69 frames in 200ms). vfr on a 1/FPS // timebase drops those same-slot frames instead of encoding a burst. let time_base = format!("1/{fps}"); if wayland { command.stdin(std::process::Stdio::piped()); } - let mut child=command.args(["-fps_mode","vfr","-enc_time_base",&time_base,"-f","h264","pipe:1"]) + let mut child=command.args(["-fps_mode","vfr","-enc_time_base",&time_base,"-f",if settings.codec == VideoCodec::H265 { "hevc" } else { "h264" },"pipe:1"]) .kill_on_drop(true) .stdout(std::process::Stdio::piped()) .stderr(std::process::Stdio::inherit()) @@ -359,14 +446,14 @@ async fn read_encoder_with( pending.extend_from_slice(&chunk[..count]); // A chunk can include the next AUD, so split before enforcing the bound. loop { - let aud = aud_offsets(&pending); + let aud = aud_offsets_for(&pending, settings.codec); if aud.len() < 2 { break; } let next = aud[1]; if next > MAX_ACCESS_UNIT { return Err("Encoder access unit exceeds limit".into()); } let frame: Vec = pending.drain(..next).collect(); let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; gate.submit(&latest, Arc::new(CompleteFrame { - sequence, completed_at: Instant::now(), data: frame, + codec: settings.codec, sequence, completed_at: Instant::now(), data: frame, })); if latest.is_closed() { return Ok(()); } } @@ -414,8 +501,8 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option, track: Arc, size: (u32, u32), display: String, portal:Option>, - mut settings: tokio::sync::watch::Receiver, idr: Arc) -> Result<(), String> { +pub async fn stream_desktop(pc: Arc, track: Arc, size: (u32, u32), display: String, portal:Option>, + mut settings: tokio::sync::watch::Receiver, applied: Arc>, idr: Arc) -> Result<(), String> { loop { if transport_closed(&pc) { return Ok(()); } if pc.connection_state() == RTCPeerConnectionState::Connected @@ -425,7 +512,12 @@ pub async fn stream_desktop(pc: Arc, track: Arc, track: Arc return result.map_err(|e| format!("Encoder reader task: {e}"))?, + result = &mut reader.0 => { + let result = result.map_err(|e| format!("Encoder reader task: {e}"))?; + if result.is_err() && current.codec == VideoCodec::H265 && track.supports(VideoCodec::H264) { + log::warn!("HEVC unavailable; falling back to negotiated H264"); + current.codec = VideoCodec::H264; + applied.send_modify(|value| value.codec = VideoCodec::H264); + current = *settings.borrow_and_update(); + while receiver.try_recv().is_ok() {} + reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone())); + continue; + } + return result; + }, _ = health.tick() => { if transport_closed(&pc) { return Ok(()); } }, changed = settings.changed(), if settings_open => { if changed.is_err() { settings_open = false; continue; } // Restart the encoder with the viewer's new 帧率/画质; it begins with an IDR, and frames // still queued from the old encoder remain a valid chain before it. let next = *settings.borrow_and_update(); + let codec_changed = next.codec != current.codec; + current = next; timeline.fps = u64::from(next.fps); - reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await; + if codec_changed { + reader.0.abort(); + let _ = (&mut reader.0).await; + // Drain only after the old producer is gone, before the new IDR can enter. + while receiver.try_recv().is_ok() {} + reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())); + } else { + reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await; + } }, frame = receiver.recv() => { // A closed queue means the reader ended; its JoinHandle carries the reason. let Some(frame) = frame else { return (&mut reader.0).await.map_err(|e| format!("Encoder reader task: {e}"))?; }; + if frame.codec != current.codec { continue; } let duration = timeline.duration_after(frame.completed_at); if let Some(last) = previous { skipped += frame.sequence.saturating_sub(last + 1); } max_age_ms = max_age_ms.max(frame.completed_at.elapsed().as_millis()); @@ -460,7 +575,7 @@ pub async fn stream_desktop(pc: Arc, track: Arc Result<()> { + media.register_codec(RTCRtpCodecParameters { capability: RTCRtpCodecCapability { + mime_type: VideoCodec::H265.mime().into(), clock_rate: 90_000, + rtcp_feedback: vec![webrtc::rtp_transceiver::RTCPFeedback { typ: "nack".into(), parameter: String::new() }, + webrtc::rtp_transceiver::RTCPFeedback { typ: "nack".into(), parameter: "pli".into() }], + ..Default::default() }, payload_type: 126, ..Default::default() }, RTPCodecType::Video) +} + +#[derive(Debug)] +struct Timeline { sequence: u16, timestamp: u32, h264: webrtc::rtp::codecs::h264::H264Payloader } +#[derive(Debug)] +pub struct VideoTrack { + h264: TrackLocalStaticRTP, h265: TrackLocalStaticRTP, + has_h264: AtomicBool, has_h265: AtomicBool, timeline: Mutex, + progress: tokio::sync::watch::Sender<(u64, VideoCodec)>, +} +impl VideoTrack { + pub fn new() -> Self { + let track = |codec: VideoCodec| TrackLocalStaticRTP::new( + RTCRtpCodecCapability { mime_type: codec.mime().into(), clock_rate: 90_000, ..Default::default() }, + "video_0".into(), "video_0".into()); + let (progress, _) = tokio::sync::watch::channel((0, VideoCodec::H264)); + Self { progress, h264: track(VideoCodec::H264), h265: track(VideoCodec::H265), has_h264: AtomicBool::new(false), + has_h265: AtomicBool::new(false), timeline: Mutex::new(Timeline { sequence: rand::random(), timestamp: rand::random(), h264: Default::default() }) } + } + pub fn supports(&self, codec: VideoCodec) -> bool { + match codec { VideoCodec::H264 => &self.has_h264, VideoCodec::H265 => &self.has_h265 }.load(Ordering::Acquire) + } + pub fn revision(&self) -> u64 { self.progress.borrow().0 } + /// Format-change RPC success follows a transmitted frame, including a negotiated fallback. + pub async fn wait_for_applied(&self, since: u64, settings: &tokio::sync::watch::Sender) -> bool { + let mut progress = self.progress.subscribe(); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let (revision, codec) = *progress.borrow_and_update(); + if revision > since && codec == settings.borrow().codec { return true; } + if progress.changed().await.is_err() { return false; } + } + }).await.unwrap_or(false) + } + pub async fn write_sample_with_extensions(&self, codec: VideoCodec, sample: &Sample, extensions: &[webrtc::rtp::extension::HeaderExtension]) -> Result<()> { + if !self.supports(codec) { return Err(Error::ErrUnsupportedCodec); } + let track = match codec { VideoCodec::H264 => &self.h264, VideoCodec::H265 => &self.h265 }; + if track.all_binding_paused().await { return Ok(()); } + let mut state = self.timeline.lock().await; + // Leave room for the RTP header and negotiated UU extensions below the 1200 byte MTU. + let payloads = match codec { + VideoCodec::H264 => state.h264.payload(1160, &sample.data).map_err(Error::Rtp)?, + VideoCodec::H265 => crate::hevc::payload(1160, &sample.data).map_err(Error::new)?, + }; + let count = payloads.len(); + let mut complete = count > 0; + for (index, payload) in payloads.into_iter().enumerate() { + let packet = Packet { header: Header { version: 2, sequence_number: state.sequence, timestamp: state.timestamp, + marker: index + 1 == count, ..Default::default() }, payload }; + state.sequence = state.sequence.wrapping_add(1); + complete &= track.write_rtp_with_extensions(&packet, extensions).await? > 0; + } + if complete { self.progress.send_modify(|value| { value.0 += 1; value.1 = codec; }); } + state.timestamp = state.timestamp.wrapping_add((sample.duration.as_secs_f64() * 90_000.0) as u32); + Ok(()) + } +} +#[async_trait] +impl TrackLocal for VideoTrack { + async fn bind(&self, context: &TrackLocalContext) -> Result { + let h264 = self.h264.bind(context).await.ok(); + let h265 = self.h265.bind(context).await.ok(); + self.has_h264.store(h264.is_some(), Ordering::Release); + self.has_h265.store(h265.is_some(), Ordering::Release); + log::info!("Video track negotiated: h264={} h265={}", h264.is_some(), h265.is_some()); + h264.or(h265).ok_or(Error::ErrUnsupportedCodec) + } + async fn unbind(&self, context: &TrackLocalContext) -> Result<()> { + if self.has_h264.swap(false, Ordering::AcqRel) { self.h264.unbind(context).await?; } + if self.has_h265.swap(false, Ordering::AcqRel) { self.h265.unbind(context).await?; } + Ok(()) + } + fn id(&self) -> &str { "video_0" } + fn stream_id(&self) -> &str { "video_0" } + fn kind(&self) -> RTPCodecType { RTPCodecType::Video } + fn as_any(&self) -> &dyn Any { self } +} + +#[cfg(test)] +mod tests { + use super::*; + use webrtc::{api::{APIBuilder, media_engine::MediaEngine}, peer_connection::configuration::RTCConfiguration}; + async fn negotiate(h264: bool, hevc: bool) { + let mut remote_media = MediaEngine::default(); + if h264 { remote_media.register_codec(RTCRtpCodecParameters { capability: RTCRtpCodecCapability { + mime_type: VideoCodec::H264.mime().into(), clock_rate: 90_000, + sdp_fmtp_line: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f".into(), ..Default::default() + }, payload_type: 102, ..Default::default() }, RTPCodecType::Video).unwrap(); } + if hevc { register_hevc(&mut remote_media).unwrap(); } + let mut setting = webrtc::api::setting_engine::SettingEngine::default(); + setting.set_ice_multicast_dns_mode(webrtc::ice::mdns::MulticastDnsMode::Disabled); + let remote = APIBuilder::new().with_setting_engine(setting.clone()).with_media_engine(remote_media).build().new_peer_connection(RTCConfiguration::default()).await.unwrap(); + remote.add_transceiver_from_kind(RTPCodecType::Video, None).await.unwrap(); + let mut media = MediaEngine::default(); media.register_default_codecs().unwrap(); register_hevc(&mut media).unwrap(); + let host = APIBuilder::new().with_setting_engine(setting).with_media_engine(media).build().new_peer_connection(RTCConfiguration::default()).await.unwrap(); + let (packets_tx, mut packets_rx) = tokio::sync::mpsc::unbounded_channel(); + remote.on_track(Box::new(move |remote_track, _, _| { + let packets_tx = packets_tx.clone(); + Box::pin(async move { + tokio::spawn(async move { + while let Ok((packet, _)) = remote_track.read_rtp().await { + if packets_tx.send(packet).is_err() { break; } + } + }); + }) + })); + let track = std::sync::Arc::new(VideoTrack::new()); + host.add_track(track.clone()).await.unwrap(); + let offer = remote.create_offer(None).await.unwrap(); + let mut gathered = remote.gathering_complete_promise().await; + remote.set_local_description(offer).await.unwrap(); + gathered.recv().await; + host.set_remote_description(remote.local_description().await.unwrap()).await.unwrap(); + let answer = host.create_answer(None).await.unwrap(); + assert_eq!(answer.sdp.contains("H265/90000"), hevc); + assert_eq!(answer.sdp.contains("H264/90000"), h264); + let mut gathered = host.gathering_complete_promise().await; + host.set_local_description(answer).await.unwrap(); + gathered.recv().await; + remote.set_remote_description(host.local_description().await.unwrap()).await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while !track.supports(VideoCodec::H264) && !track.supports(VideoCodec::H265) { tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + }).await.unwrap(); + assert_eq!(track.supports(VideoCodec::H264), h264); + assert_eq!(track.supports(VideoCodec::H265), hevc); + tokio::time::timeout(std::time::Duration::from_secs(10), async { + while host.connection_state() != webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Connected { + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }).await.unwrap(); + let codecs = if !h264 { vec![VideoCodec::H265] } else if hevc { vec![VideoCodec::H264, VideoCodec::H265, VideoCodec::H264] } else { vec![VideoCodec::H264] }; + let mut previous: Option = None; + for codec in codecs { + let data = match codec { VideoCodec::H264 => bytes::Bytes::from_static(&[0,0,1,65,1,2]), + VideoCodec::H265 => bytes::Bytes::from_static(&[0,0,1,38,1,2]) }; + let (settings, _) = tokio::sync::watch::channel(crate::capture_setting::VideoSettings { codec, ..Default::default() }); + let applied = track.wait_for_applied(track.revision(), &settings); + tokio::pin!(applied); + assert!(tokio::time::timeout(std::time::Duration::from_millis(1), &mut applied).await.is_err()); + track.write_sample_with_extensions(codec, &Sample { data, duration: std::time::Duration::from_millis(20), ..Default::default() }, &[]).await.unwrap(); + assert!(applied.await); + let packet = tokio::time::timeout(std::time::Duration::from_secs(3), packets_rx.recv()).await.unwrap().unwrap(); + assert_eq!(packet.header.payload_type, if codec == VideoCodec::H264 { 102 } else { 126 }); + assert!(packet.header.marker); + if let Some(previous) = previous { + assert_eq!(packet.header.ssrc, previous.header.ssrc); + assert_eq!(packet.header.sequence_number, previous.header.sequence_number.wrapping_add(1)); + assert_eq!(packet.header.timestamp, previous.header.timestamp.wrapping_add(1800)); + } + previous = Some(packet); + } + host.close().await.unwrap(); remote.close().await.unwrap(); + } + #[tokio::test] + async fn negotiated_codecs_only_and_legacy_h264_offer() { negotiate(true, false).await; negotiate(true, true).await; negotiate(false, true).await; } +} diff --git a/src/webrtc.rs b/src/webrtc.rs index 4490981..2ff47f9 100644 --- a/src/webrtc.rs +++ b/src/webrtc.rs @@ -53,6 +53,9 @@ impl WebRtcSession { m.register_default_codecs() .map_err(|e| format!("Register standard codecs failed: {}", e))?; + let hevc = desktop && crate::video::hevc_available().await; + if hevc { crate::video_track::register_hevc(&mut m).map_err(|e| format!("Register HEVC failed: {e}"))?; } + m.register_header_extension( webrtc::rtp_transceiver::rtp_codec::RTCRtpHeaderExtensionCapability { uri: crate::rtp_metadata::CAPTURE_INDEX_URI.into() }, webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video, None, @@ -162,7 +165,7 @@ impl WebRtcSession { tokio::spawn(async move { let ctx = ctx.lock().await.clone(); if let (Some(client), Some(ice)) = (&ctx.client_id, ctx.ice_id.as_ref().and_then(|v| v.as_str())) { - let payload = crate::capability::h264_device_capability(client, ice, 0, crate::video::FPS as i32, 5); + let payload = crate::capability::device_capability(client, ice, 0, crate::video::FPS as i32, 5, hevc); if let Err(e) = sig.emit_event(crate::capability::FORWARD_SETTING_EVENT, payload).await { log::error!("Device capability notification failed: {}", e); } @@ -179,14 +182,7 @@ impl WebRtcSession { })); // 注册视频轨 - let video_track = Arc::new(TrackLocalStaticSample::new( - RTCRtpCodecCapability { - mime_type: "video/H264".to_owned(), - ..Default::default() - }, - "video_0".to_owned(), - "video_0".to_owned(), - )); + let video_track = Arc::new(crate::video_track::VideoTrack::new()); // Set by PLI/FIR; the desktop encoder answers with an IDR on its next frame. let idr_request = Arc::new(std::sync::atomic::AtomicBool::new(false)); @@ -274,9 +270,11 @@ impl WebRtcSession { } let portal=capture.portal.clone(); // File/terminal/port-mapping connections carry no desktop video. + let control_video_track = video_track.clone(); + let stream_settings = video_settings.clone(); if desktop { tokio::spawn(async move { - if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, idr_request).await { + if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, stream_settings, idr_request).await { log::error!("Live desktop stream stopped: {}", e); } }); @@ -338,6 +336,7 @@ impl WebRtcSession { let peer_files = peer_files.clone(); let peer_port_mapping = port_mapping.clone(); let peer_video_settings = video_settings.clone(); + let peer_video_track = control_video_track.clone(); let peer_actions = peer_actions.clone(); if label == "FILE_DATA_CHANNEL" { *file_channel.lock().unwrap() = Some(d.clone()); } @@ -389,9 +388,11 @@ impl WebRtcSession { let rich_outgoing=rich_state.clone(); let open_actions = channel_actions.clone(); let open_video = channel_video_settings.clone(); + let open_track = peer_video_track.clone(); d.on_open(Box::new(move || { let init_actions = open_actions.clone(); let init_video = open_video.clone(); + let init_track = open_track.clone(); let channel = channel.clone(); let sequence = initial_sequence.clone(); let screen = screen.clone(); @@ -462,9 +463,18 @@ impl WebRtcSession { (13, Ok(crate::initialization::envelope(next_seq(), timestamp, 13, &init_actions.metrics_payload(desktop_sound, desktop_permission)))), (26, Ok(crate::initialization::envelope(next_seq(), timestamp, 26, &crate::system_metrics::remote_download_path(&home)))), + (18, Ok(crate::initialization::envelope(next_seq(), timestamp, 18, + &crate::capture_setting::codec_negotiation(screen_size, crate::video::FPS as u32, init_track.supports(crate::capture_setting::VideoCodec::H265))))), (7, crate::capture_setting::screen_sources(next_seq(), timestamp, screen_size, init_video.borrow().output, "Linux desktop")), ]; + if desktop && screen.init_error.is_none() { + let codec = crate::capture_setting::preferred_codec(init_track.supports(crate::capture_setting::VideoCodec::H264), + init_track.supports(crate::capture_setting::VideoCodec::H265)); + init_video.send_if_modified(|current| if current.codec != codec { + log::info!("Desktop codec selected: {:?}", codec); current.codec = codec; true } else { false }); + } for (field, message) in messages { + if (!desktop || screen.init_error.is_some()) && field == 18 { continue; } if desktop && screen.init_error.is_some() && matches!(field,7|8) { continue; } let message = match message { Ok(message) => message, @@ -522,6 +532,7 @@ impl WebRtcSession { let file_worker = file_session.clone(); let mapping = peer_port_mapping.clone(); let video_settings = channel_video_settings.clone(); + let negotiated_track = peer_video_track.clone(); let actions = channel_actions.clone(); Box::pin(async move { @@ -532,8 +543,17 @@ impl WebRtcSession { use crate::protocol::WireValue; let rpc = fields.iter().find_map(|f| match (f.number, &f.value) { (21, WireValue::Bytes(b)) => Some(*b), _ => None }); let rpc_fields = rpc.and_then(|r| crate::protocol::decode_wire(r).ok()); - let output_before = video_settings.borrow().output; - if let Some(response) = rpc_fields.as_deref().and_then(|r| crate::capture_setting::handle_rpc(r, &video_settings, input_size)) { + let before = *video_settings.borrow(); + let output_before = before.output; + let revision = negotiated_track.revision(); + if let Some(response) = rpc_fields.as_deref().and_then(|r| crate::capture_setting::handle_rpc_with_codecs(r, &video_settings, input_size, negotiated_track.supports(crate::capture_setting::VideoCodec::H264), negotiated_track.supports(crate::capture_setting::VideoCodec::H265))) { + let codec_changed = video_settings.borrow().codec != before.codec; + if codec_changed && !negotiated_track.wait_for_applied(revision, &video_settings).await { + // Official host replies after startVideoCaptureAndSendRpcResponse. + // Do not fabricate a codec error number or acknowledge an encoder that never started. + log::warn!("Codec switch did not produce a transmitted frame; success reply withheld"); + return; + } let ts = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs(); let next = || out_sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed); let _ = rpc_out.send(crate::initialization::envelope(next(), ts, 22, &response)); @@ -865,6 +885,10 @@ impl WebRtcSession { }; log::info!("Offer x-uuremote-quic: {}", quic_summary); log::info!("Offer includes UU capture-index extension: {}", sdp_str.lines().any(|l| l.starts_with("a=extmap:") && l.contains(crate::rtp_metadata::CAPTURE_INDEX_URI))); + let mut codecs: Vec<&str> = sdp_str.lines().filter_map(|l| l.strip_prefix("a=rtpmap:")).filter_map(|l| l.split_whitespace().nth(1)) + .filter_map(|c| c.split('/').next()).collect(); + codecs.dedup(); + log::info!("Offer codecs: {:?}", codecs); let offer = RTCSessionDescription::offer(sdp_str) .map_err(|e| format!("Invalid offer: {}", e))?;