Support H.265 (HEVC), selected by the host when both sides support it

One video track carries H.264 or H.265 (RFC 7798 payloader); the host
announces its encoders with CONTROL Message 18 CodecNegotiation and picks
H.265 when the viewer negotiated it (the official Mac viewer offers H265
first; CaptureSettingRequest field 13 only reports the current codec).
hevc_nvenc / libx265 / hevc_vaapi with fallback to software HEVC, then
H.264; UU_VIDEO_CODEC=h264 forces H.264. True color 4:4:4 applies to H.265.
Verified with the official Mac viewer on NVENC.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
VallierDevandClaude Opus 5.5 committed 2026-10-16 00:13:54 +08:00
1 parent ff84aa4fd2
commit 708c770b19
15 files changed
+672 -89

No files matched your search

+1 -1
View File
@@ -1,4 +1,4 @@
//! Opt-in guest assistance. Protocol sources: docs/validation/remote-assist-evidence.md.
//! Opt-in guest assistance. Protocol sources: AGENTS.md "Remote assistance".
//! Secret values never implement Display/Debug or enter diagnostics.
use crate::auth::{Credentials, RoomData, UUAuthClient};
use serde_json::{json, Value};
+18 -6
View File
@@ -27,17 +27,20 @@ pub fn h264_device_capability(
fps: i32,
codec_impl: i32,
) -> Value {
device_capability(client_id, ice_id, display_id, fps, codec_impl, false)
}
pub fn device_capability(client_id: &str, ice_id: &str, display_id: i32, fps: i32, codec_impl: i32, hevc: bool) -> Value {
let mut codecs = vec![json!({"video_codec": VIDEO_CODEC_H264, "chroma_sampling": 1,
"codec_impl": codec_impl, "bit_depth": 8})];
// Official VideoCodecCapability::SerializeJson uses the same fields for CodecType H265=2.
if hevc { codecs.push(json!({"video_codec": 2, "chroma_sampling": 1, "codec_impl": codec_impl, "bit_depth": 8})); }
json!({
"client_id": client_id,
"type": "device_capability",
"data": {
"display_info": [{"id": display_id, "fps": fps, "type": 0, "hdr": 0}],
"video_codec_capability": [{
"video_codec": VIDEO_CODEC_H264,
"chroma_sampling": 1,
"codec_impl": codec_impl,
"bit_depth": 8
}],
"video_codec_capability": codecs,
"ice_id": ice_id
}
})
@@ -47,6 +50,15 @@ pub fn h264_device_capability(
mod tests {
use super::*;
#[test]
fn hevc_capability_uses_official_codec_enum_and_420_only() {
let payload = device_capability("client-example", "ice-example", 0, 60, 5, true);
assert_eq!(payload["data"]["video_codec_capability"], json!([
{"video_codec":1,"chroma_sampling":1,"codec_impl":5,"bit_depth":8},
{"video_codec":2,"chroma_sampling":1,"codec_impl":5,"bit_depth":8}]));
assert_eq!(device_capability("c", "i", 0, 60, 5, false)["data"]["video_codec_capability"].as_array().unwrap().len(), 1);
}
#[test]
fn ice_finished_keeps_string_identifiers_and_exact_fields() {
let payload = ice_finished("ice-example", "control-example");
+81 -7
View File
@@ -30,9 +30,18 @@ pub const CHROMA_444: u64 = 3;
const MIN_FPS: u32 = 5;
const MAX_FPS: u32 = 144;
/// Official GvPb_CodecType, CaptureSettingRequest.codec_type (field 13).
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub enum VideoCodec { #[default] H264, H265 }
impl VideoCodec {
pub fn wire(self) -> u64 { match self { Self::H264 => 1, Self::H265 => 2 } }
pub fn mime(self) -> &'static str { match self { Self::H264 => "video/H264", Self::H265 => "video/H265" } }
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct VideoSettings {
pub fps: u32,
pub codec: VideoCodec,
pub quality: u64,
/// Viewer max_custom_bitrate (only used for QUALITY_CUSTOM): Mbit/s, or kbit/s above 1000.
pub custom_mbps: u32,
@@ -43,7 +52,7 @@ pub struct VideoSettings {
}
impl Default for VideoSettings {
fn default() -> Self { Self { fps: 60, quality: QUALITY_AUTO, custom_mbps: 0, output: None, chroma: CHROMA_420 } }
fn default() -> Self { Self { codec: VideoCodec::H264, fps: 60, quality: QUALITY_AUTO, custom_mbps: 0, output: None, chroma: CHROMA_420 } }
}
/// Selectable smaller stream sizes with the desktop's aspect ratio (even dimensions).
@@ -102,10 +111,21 @@ fn fps_from_option(value: u64) -> Option<u64> {
/// Applies a capture-setting RPC to `settings` and returns the RpcResponse payload (Message 22),
/// or None when the RPC is not a capture/fps/quality setting.
pub fn handle_rpc(rpc: &[WireField], settings: &watch::Sender<VideoSettings>, native: (u32, u32)) -> Option<Vec<u8>> {
apply_rpc(rpc, settings, native, crate::video::full_chroma_available())
handle_rpc_with_codecs(rpc, settings, native, true, false)
}
fn apply_rpc(rpc: &[WireField], settings: &watch::Sender<VideoSettings>, native: (u32, u32), full_chroma: bool) -> Option<Vec<u8>> {
/// Initial codec once the m-line is negotiated: H.265 when both sides support it (the Mac viewer
/// lists H265 first in its offer), unless UU_VIDEO_CODEC=h264.
pub fn preferred_codec(h264: bool, h265: bool) -> VideoCodec {
let forced_h264 = std::env::var("UU_VIDEO_CODEC").is_ok_and(|v| v.eq_ignore_ascii_case("h264"));
if h265 && !(forced_h264 && h264) { VideoCodec::H265 } else { VideoCodec::H264 }
}
pub fn handle_rpc_with_codecs(rpc: &[WireField], settings: &watch::Sender<VideoSettings>, native: (u32, u32), h264: bool, h265: bool) -> Option<Vec<u8>> {
apply_rpc(rpc, settings, native, h264, h265, crate::video::full_chroma_available())
}
fn apply_rpc(rpc: &[WireField], settings: &watch::Sender<VideoSettings>, native: (u32, u32), h264: bool, h265: bool, full_chroma: bool) -> Option<Vec<u8>> {
let header = message(rpc, 1).unwrap_or(&[]);
let mut next = *settings.borrow();
if let Some(request) = message(rpc, 2) {
@@ -113,6 +133,10 @@ fn apply_rpc(rpc: &[WireField], settings: &watch::Sender<VideoSettings>, native:
// Numbers only: fps, quality, bitrate, resolution and flags carry no user content.
log::info!("Capture setting request: {:?}", fields.iter().filter_map(|f| match f.value {
WireValue::Varint(v) => Some((f.number, v)), _ => None }).collect::<Vec<_>>());
// codec_type 13 (1 H264, 2 H265): like the official host (negotiateCodec), the host picks the
// codec from both capability lists; the Mac viewer sends its current codec (1 while receiving
// H.264), so only an explicit H.265 request switches, never one absent from the m-line.
if number(&fields, 13) == Some(2) && h265 { next.codec = VideoCodec::H265; }
// Field 1 is the viewer's fps option index (1 = 30, 2 = 60, ...); 18 fpsCount is the frame rate.
if let Some(fps) = number(&fields, 18).filter(|v| *v > 0).or_else(|| number(&fields, 1).and_then(fps_from_option)) { next.fps = fps as u32; }
if let Some(quality) = number(&fields, 2).filter(|v| *v > 0) { next.quality = quality; }
@@ -183,6 +207,23 @@ pub fn sync_payload(settings: &VideoSettings, native: (u32, u32)) -> Vec<u8> {
out
}
/// Message 18 CodecNegotiation {1 encoder_cap_list: EncoderCap {1 capture_mode (1 DESKTOP), 2 fps,
/// 3 codec_type, 4 width, 5 height}} (official Windows host descriptor). The viewer's own list is
/// ConnectOptions field 4 decoder_cap_list; it only selects a codec the host announces here.
pub fn codec_negotiation(size: (u32, u32), fps: u32, hevc: bool) -> Vec<u8> {
let mut out = Vec::new();
for codec in [Some(VideoCodec::H264), hevc.then_some(VideoCodec::H265)].into_iter().flatten() {
let mut cap = Vec::new();
put_varint(1, 1, &mut cap);
put_varint(2, u64::from(fps), &mut cap);
put_varint(3, codec.wire(), &mut cap);
put_varint(4, u64::from(size.0), &mut cap);
put_varint(5, u64::from(size.1), &mut cap);
put_bytes(1, &cap, &mut out);
}
out
}
#[cfg(test)]
mod tests {
use super::*;
@@ -205,7 +246,7 @@ mod tests {
put_varint(18, 30, &mut body);
let request = rpc(77, 2, &body);
let response = handle_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080)).unwrap();
assert_eq!(*rx.borrow(), VideoSettings { fps: 30, quality: QUALITY_GENERAL, custom_mbps: 0, output: None, chroma: CHROMA_420 });
assert_eq!(*rx.borrow(), VideoSettings { codec: VideoCodec::H264, fps: 30, quality: QUALITY_GENERAL, custom_mbps: 0, output: None, chroma: CHROMA_420 });
assert_eq!(rx.borrow().max_kbps(), Some(2_000));
let fields = decode_wire(&response).unwrap();
assert_eq!(message(&fields, 1), Some(&[8u8, 77][..]));
@@ -231,15 +272,24 @@ mod tests {
let (tx, rx) = watch::channel(VideoSettings::default());
// Observed Mac viewer request with 真彩 on: field 7 = 3.
let request = rpc(5, 2, &[8, 2, 16, 4, 56, 3, 144, 1, 60]);
apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), false).unwrap();
apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true, false, false).unwrap();
assert_eq!(rx.borrow().chroma, CHROMA_420);
apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true).unwrap();
apply_rpc(&decode_wire(&request).unwrap(), &tx, (1920, 1080), true, false, true).unwrap();
assert_eq!(rx.borrow().chroma, CHROMA_444);
assert_eq!(number(&decode_wire(&sync_payload(&rx.borrow(), (1920, 1080))).unwrap(), 3), Some(CHROMA_444));
apply_rpc(&decode_wire(&rpc(6, 2, &[56, 1])).unwrap(), &tx, (1920, 1080), true).unwrap();
apply_rpc(&decode_wire(&rpc(6, 2, &[56, 1])).unwrap(), &tx, (1920, 1080), true, false, true).unwrap();
assert_eq!(rx.borrow().chroma, CHROMA_420);
}
#[test]
fn codec_negotiation_lists_h264_then_optional_h265() {
assert_eq!(codec_negotiation((1920, 1080), 60, false), [10, 12, 8, 1, 16, 60, 24, 1, 32, 128, 15, 40, 184, 8]);
let both = codec_negotiation((1920, 1080), 60, true);
let caps: Vec<_> = decode_wire(&both).unwrap().iter().filter_map(|f| match f.value { WireValue::Bytes(b) => Some(b), _ => None })
.map(|b| number(&decode_wire(b).unwrap(), 3)).collect();
assert_eq!(caps, [Some(1), Some(2)]);
}
#[test]
fn resolution_requests_scale_below_native_only() {
let (tx, rx) = watch::channel(VideoSettings::default());
@@ -267,3 +317,27 @@ mod tests {
assert_eq!(number(&fields, 10), Some(1080));
}
}
#[cfg(test)]
mod codec_tests {
use super::*;
#[test]
fn codec_is_opt_in_and_requires_negotiated_support() {
let (tx, rx) = watch::channel(VideoSettings::default());
let rpc = |value| decode_wire(if value == 2 { &[18, 2, 104, 2] } else { &[18, 2, 104, 1] }).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H264);
handle_rpc_with_codecs(&rpc(2), &tx, (1920,1080), true, false).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H264);
handle_rpc_with_codecs(&rpc(2), &tx, (1920,1080), true, true).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H265);
handle_rpc_with_codecs(&rpc(1), &tx, (1920,1080), false, true).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H265); // unnegotiated H264 is never sent
// An unrelated FPS RPC must preserve the explicit codec selection.
handle_rpc_with_codecs(&decode_wire(&[26,2,8,1]).unwrap(), &tx, (1920,1080), true, true).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H265);
// The viewer reports its current codec (1) in every request; that must not undo H.265.
handle_rpc_with_codecs(&rpc(1), &tx, (1920,1080), true, true).unwrap();
assert_eq!(rx.borrow().codec, VideoCodec::H265);
assert_eq!(preferred_codec(true, false), VideoCodec::H264);
}
}
+72
View File
@@ -0,0 +1,72 @@
//! RFC 7798 §4.4.1 / §4.4.3, single-layer non-interleaved HEVC (no DONL).
//! Parameter sets travel in-band before each IDR; no aggregation is required.
use bytes::Bytes;
pub fn payload(mtu: usize, annex_b: &[u8]) -> Result<Vec<Bytes>, String> {
if mtu < 4 { return Err("HEVC payload MTU too small".into()); }
let mut starts = Vec::new();
let mut i = 0;
while i + 3 <= annex_b.len() {
let prefix = if annex_b[i..].starts_with(&[0, 0, 0, 1]) { 4 }
else if annex_b[i..].starts_with(&[0, 0, 1]) { 3 } else { i += 1; continue; };
starts.push((i, i + prefix)); i += prefix;
}
if starts.is_empty() { return Err("HEVC access unit has no Annex-B start code".into()); }
let mut out = Vec::new();
for (index, &(_, start)) in starts.iter().enumerate() {
let mut end = starts.get(index + 1).map_or(annex_b.len(), |s| s.0);
while end > start && annex_b[end - 1] == 0 { end -= 1; } // trailing_zero_8bits
let nal = &annex_b[start..end];
if nal.len() < 2 || nal[0] & 0x80 != 0 || nal[1] & 7 == 0 {
return Err("Invalid HEVC NAL header".into());
}
let kind = (nal[0] >> 1) & 63;
if kind >= 48 { return Err("HEVC Annex-B contains RTP packet type".into()); }
if nal.len() <= mtu { out.push(Bytes::copy_from_slice(nal)); continue; }
let chunks = nal[2..].chunks(mtu - 3);
let last = chunks.len() - 1;
for (n, chunk) in chunks.enumerate() {
let mut fu = Vec::with_capacity(3 + chunk.len());
// Keep F, LayerId and TID, replacing only the NAL type with FU=49.
fu.extend_from_slice(&[(nal[0] & 0x81) | (49 << 1), nal[1],
kind | if n == 0 { 0x80 } else { 0 } | if n == last { 0x40 } else { 0 }]);
fu.extend_from_slice(chunk); out.push(fu.into());
}
}
Ok(out)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parameter_sets_and_small_nals_remain_in_order_without_start_codes() {
assert_eq!(payload(1200, &[0,0,0,1,64,1,9,0,0,1,66,1,8,0,0,1,38,1,7]).unwrap(),
vec![Bytes::from_static(&[64,1,9]), Bytes::from_static(&[66,1,8]), Bytes::from_static(&[38,1,7])]);
}
#[test]
fn fu_round_trip_preserves_header_payload_and_mtu_boundaries() {
for mtu in [4, 5, 7, 1190] {
let header = [0x27, 0x2b]; // IDR type19, nonzero layer and temporal id
let data: Vec<u8> = (0..2501).map(|i| (i % 251 + 1) as u8).collect();
let mut unit = vec![0,0,0,1]; unit.extend(header); unit.extend(&data);
let packets = payload(mtu, &unit).unwrap();
assert!(packets.len() > 1);
let mut rebuilt = vec![(packets[0][0] & 0x81) | ((packets[0][2] & 63) << 1), packets[0][1]];
for (i, p) in packets.iter().enumerate() {
assert!(p.len() <= mtu); assert_eq!((p[0] >> 1) & 63, 49);
assert_eq!(p[2] & 0x80 != 0, i == 0);
assert_eq!(p[2] & 0x40 != 0, i + 1 == packets.len());
rebuilt.extend_from_slice(&p[3..]);
}
assert_eq!(&rebuilt[..2], &header); assert_eq!(&rebuilt[2..], &data);
}
}
#[test]
fn rejects_invalid_headers_and_too_small_mtu() {
for bytes in [&[0,0,1,64][..], &[0,0,1,64,0][..], &[0,0,1,128,1][..], &[64,1][..]] {
assert!(payload(1200, bytes).is_err());
}
assert!(payload(3, &[0,0,1,64,1]).is_err());
}
}
+14 -6
View File
@@ -1,4 +1,4 @@
//! In-process X11 capture and H.264 encoding through libavdevice/libavcodec.
//! In-process X11 capture and H.264/HEVC encoding through libavdevice/libavcodec.
//! It replaces the `ffmpeg` child process with the same codec options, so the viewer
//! still receives the same Annex-B access units (AUD, SPS/PPS, IDR, P).
use std::ffi::CString;
@@ -25,6 +25,7 @@ pub struct Config {
pub max_bps: i64,
pub buffer_bits: i64,
pub encoder: crate::video::Encoder,
pub codec: crate::capture_setting::VideoCodec,
/// Capture on XDamage changes (XShm) instead of x11grab; `UU_CAPTURE=grab` turns it off.
pub damage: bool,
}
@@ -71,11 +72,11 @@ fn drain(encoder: &mut encoder::Video, packet: &mut Packet, units: &mpsc::Sender
}
}
/// Opens the H.264 encoder for `config`. Returns the encoder, its input pixel format and output size.
/// Opens the selected video encoder for `config`. Returns the encoder, its input pixel format and output size.
fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), String> {
let (width, height) = config.output.unwrap_or(config.input);
let codec = encoder::find_by_name(config.encoder.name())
.ok_or("H.264 encoder is not available in this FFmpeg build")?;
let codec = encoder::find_by_name(config.encoder.name_for(config.codec))
.ok_or_else(|| format!("{} is not available in this FFmpeg build", config.encoder.name_for(config.codec)))?;
// Allocate for this codec so FFmpeg fills its defaults; libx264 rejects a context allocated without one.
let mut context = unsafe { codec::context::Context::wrap(ff::sys::avcodec_alloc_context3(codec.as_ptr()), None) }.encoder().video().map_err(text)?;
context.set_width(width);
@@ -99,15 +100,22 @@ fn open_encoder(config: &Config) -> Result<(encoder::Video, Pixel, u32, u32), St
let cq = config.cq.to_string();
let gop = config.gop.to_string();
let mut options = Dictionary::new();
let hevc = config.codec == crate::capture_setting::VideoCodec::H265;
if config.encoder == crate::video::Encoder::Vaapi {
for (key,value) in [("profile", "constrained_baseline"), ("rc_mode", "VBR"), ("async_depth", "1"), ("aud", "1")] { options.set(key,value); }
for (key,value) in [("profile", if hevc { "main" } else { "constrained_baseline" }), ("rc_mode", "VBR"), ("async_depth", "1"), ("aud", "1")] { options.set(key,value); }
options.set("qmin", &cq);
} else if config.encoder == crate::video::Encoder::Nvenc {
for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", "baseline"), ("rc", "vbr"),
for (key, value) in [("preset", "p4"), ("tune", "ull"), ("profile", if hevc { "main" } else { "baseline" }), ("rc", "vbr"),
("rc-lookahead", "0"), ("zerolatency", "1"), ("delay", "0"), ("aud", "1"), ("forced-idr", "1")] {
options.set(key, value);
}
options.set("cq", &cq);
} else if hevc {
for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "main"), ("threads", "4"), ("forced-idr", "1")] {
options.set(key, value);
}
options.set("crf", &cq);
options.set("x265-params", &crate::video::x265_params(&gop));
} else {
let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}");
for (key, value) in [("preset", "superfast"), ("tune", "zerolatency"), ("profile", "baseline"), ("threads", "4")] {
+2
View File
@@ -24,6 +24,8 @@ mod signaling;
mod terminal;
mod webrtc;
mod video;
mod video_track;
mod hevc;
#[cfg(feature = "inproc-av")]
mod damage_capture;
#[cfg(feature = "inproc-av")]
+223 -54
View File
@@ -7,8 +7,7 @@ use tokio::sync::mpsc;
use webrtc::media::Sample;
use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState;
use webrtc::peer_connection::RTCPeerConnection;
use webrtc::track::track_local::track_local_static_sample::TrackLocalStaticSample;
use crate::capture_setting::VideoSettings;
use crate::capture_setting::{VideoCodec, VideoSettings};
/// Default frame rate; the viewer's 帧率 setting overrides it per connection.
pub const FPS: u64 = 60;
@@ -28,24 +27,42 @@ fn gop_frames(fps: u32, in_process: bool) -> u64 {
}
const MAX_ACCESS_UNIT: usize = 8 * 1024 * 1024;
fn nal_offsets(bytes: &[u8], kind: u8) -> Vec<usize> {
fn nal_offsets_for(bytes: &[u8], kind: u8, codec: VideoCodec) -> Vec<usize> {
let mut result = Vec::new();
let mut i = 0;
while i + 3 < bytes.len() {
let prefix = if bytes[i..].starts_with(&[0, 0, 0, 1]) { 4 }
else if bytes[i..].starts_with(&[0, 0, 1]) { 3 } else { i += 1; continue };
if i + prefix < bytes.len() && bytes[i + prefix] & 31 == kind { result.push(i); }
let header = i + prefix;
let matches = match codec {
VideoCodec::H264 => header < bytes.len() && bytes[header] & 31 == kind,
// HEVC has a two-byte header: forbidden_zero_bit=0, temporal_id_plus1 != 0.
VideoCodec::H265 => header + 1 < bytes.len() && bytes[header] & 128 == 0
&& bytes[header + 1] & 7 != 0 && (bytes[header] >> 1) & 63 == kind,
};
if matches { result.push(i); }
i += prefix;
}
result
}
fn aud_offsets(bytes: &[u8]) -> Vec<usize> { nal_offsets(bytes, 9) }
fn is_independent_frame(bytes: &[u8]) -> bool {
!nal_offsets(bytes, 5).is_empty() && nal_offsets(bytes, 1).is_empty()
fn aud_offsets_for(bytes: &[u8], codec: VideoCodec) -> Vec<usize> {
nal_offsets_for(bytes, if codec == VideoCodec::H265 { 35 } else { 9 }, codec)
}
fn is_independent_frame_for(bytes: &[u8], codec: VideoCodec) -> bool {
match codec {
VideoCodec::H264 => !nal_offsets_for(bytes, 5, codec).is_empty() && nal_offsets_for(bytes, 1, codec).is_empty(),
// BLA/IDR/CRA are IRAP (RFC 7798 §4.1); no leading pictures with our no-B-frame encoder.
VideoCodec::H265 => (16..=21).any(|kind| !nal_offsets_for(bytes, kind, codec).is_empty())
&& !(0..16).any(|kind| !nal_offsets_for(bytes, kind, codec).is_empty()),
}
}
#[cfg(test)]
fn aud_offsets(bytes: &[u8]) -> Vec<usize> { aud_offsets_for(bytes, VideoCodec::H264) }
fn is_independent_frame(bytes: &[u8]) -> bool { is_independent_frame_for(bytes, VideoCodec::H264) }
#[derive(Debug)]
struct CompleteFrame {
codec: VideoCodec,
sequence: u64,
completed_at: Instant,
data: Vec<u8>,
@@ -63,7 +80,13 @@ impl Encoder {
match value { Some("cpu") => Self::Cpu, Some("vaapi") => Self::Vaapi, Some("nvenc") => Self::Nvenc,
_ => if nvidia { Self::Nvenc } else { Self::Cpu } }
}
pub(crate) fn name(self) -> &'static str { match self { Self::Cpu => "libx264", Self::Nvenc => "h264_nvenc", Self::Vaapi => "h264_vaapi" } }
pub(crate) fn name_for(self, codec: VideoCodec) -> &'static str {
match (self, codec) {
(Self::Cpu, VideoCodec::H264) => "libx264", (Self::Cpu, VideoCodec::H265) => "libx265",
(Self::Nvenc, VideoCodec::H264) => "h264_nvenc", (Self::Nvenc, VideoCodec::H265) => "hevc_nvenc",
(Self::Vaapi, VideoCodec::H264) => "h264_vaapi", (Self::Vaapi, VideoCodec::H265) => "hevc_vaapi",
}
}
}
fn nvidia_present() -> bool {
std::path::Path::new("/proc/driver/nvidia/version").exists() || std::path::Path::new("/dev/nvidia0").exists()
@@ -76,12 +99,14 @@ pub fn full_chroma_available() -> bool {
Encoder::requested() != Encoder::Vaapi
}
/// (-profile:v, -pix_fmt) for NVENC / libx264: 4:4:4 needs a High 4:4:4 profile.
/// (-profile:v, -pix_fmt) for NVENC / libx264 / libx265: 4:4:4 (真彩) needs a 4:4:4 profile.
fn chroma_args(settings: &VideoSettings, encoder: Encoder) -> (&'static str, &'static str) {
match (settings.chroma == crate::capture_setting::CHROMA_444, encoder) {
(true, Encoder::Nvenc) => ("high444p", "yuv444p"),
(true, _) => ("high444", "yuv444p"),
(false, _) => ("baseline", "yuv420p"),
if settings.chroma != crate::capture_setting::CHROMA_444 { return (profile_for(settings.codec, false), "yuv420p"); }
match (settings.codec, encoder) {
(VideoCodec::H264, Encoder::Nvenc) => ("high444p", "yuv444p"),
(VideoCodec::H264, _) => ("high444", "yuv444p"),
(VideoCodec::H265, Encoder::Nvenc) => ("rext", "yuv444p"),
(VideoCodec::H265, _) => ("main444-8", "yuv444p"),
}
}
pub(crate) fn vaapi_device() -> String { std::env::var("UU_VAAPI_DEVICE").unwrap_or_else(|_| "/dev/dri/renderD128".into()) }
@@ -95,12 +120,86 @@ fn vaapi_options(settings: &VideoSettings, gop: &str) -> Vec<String> {
let (maxrate, buffer) = rate_args(settings, Encoder::Vaapi);
// QVBR is not uniformly available on Intel/AMD drivers. VBR + qmin sets a quality
// floor while retaining the viewer's bitrate cap; unsupported drivers fall back to CPU.
["-c:v", "h264_vaapi", "-profile:v", "constrained_baseline", "-g", gop, "-bf", "0",
["-c:v", Encoder::Vaapi.name_for(settings.codec), "-profile:v", profile_for(settings.codec, true), "-g", gop, "-bf", "0",
"-rc_mode", "VBR", "-b:v", &settings.max_kbps().map_or("16M".into(), |k| format!("{}k",k*3/4)),
"-maxrate", &maxrate, "-bufsize", &buffer, "-qmin", &settings.cq().to_string(), "-async_depth", "1", "-aud", "1"]
.into_iter().map(str::to_owned).collect()
}
fn profile_for(codec: VideoCodec, vaapi: bool) -> &'static str {
match codec { VideoCodec::H265 => "main", VideoCodec::H264 => if vaapi { "constrained_baseline" } else { "baseline" } }
}
pub(crate) fn x265_params(gop: &str) -> String {
// Explicit no-B/no-lookahead avoids delayed frames; repeat parameter sets at each IRAP.
format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}:open-gop=0:bframes=0:rc-lookahead=0:frame-threads=1:pools=4")
}
fn cpu_hevc_options(settings: &VideoSettings, gop: &str) -> Vec<String> {
let (maxrate, buffer) = rate_args(settings, Encoder::Cpu);
let (profile, pix_fmt) = chroma_args(settings, Encoder::Cpu);
["-c:v", "libx265", "-preset", "superfast", "-tune", "zerolatency", "-profile:v", profile,
"-pix_fmt", pix_fmt, "-g", gop, "-bf", "0", "-threads", "4", "-crf", &settings.cq().to_string(),
"-maxrate", &maxrate, "-bufsize", &buffer, "-x265-params", &x265_params(gop)]
.into_iter().map(str::to_owned).collect()
}
fn encoder_options(settings: &VideoSettings, encoder: Encoder, gop: &str) -> Vec<String> {
let (maxrate, bufsize) = rate_args(settings, encoder);
let cq = settings.cq().to_string();
let target = settings.max_kbps().map_or("16M".to_string(), |k| format!("{}k", k * 3 / 4));
if encoder == Encoder::Vaapi { return vaapi_options(settings, gop); }
let (profile, pix_fmt) = chroma_args(settings, encoder);
if encoder == Encoder::Nvenc {
return ["-c:v", encoder.name_for(settings.codec), "-preset", "p4", "-tune", "ull",
"-profile:v", profile, "-pix_fmt", pix_fmt, "-g", gop, "-bf", "0",
"-rc", "vbr", "-cq", &cq, "-b:v", &target, "-maxrate", &maxrate, "-bufsize", &bufsize,
"-rc-lookahead", "0", "-zerolatency", "1", "-delay", "0", "-aud", "1"]
.into_iter().map(str::to_owned).collect();
}
if settings.codec == VideoCodec::H265 { return cpu_hevc_options(settings, gop); }
let params = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}");
["-c:v", "libx264", "-preset", "superfast", "-tune", "zerolatency",
"-profile:v", profile, "-pix_fmt", pix_fmt, "-g", gop, "-threads", "4",
"-crf", &cq, "-maxrate", &maxrate, "-bufsize", &bufsize, "-x264-params", &params]
.into_iter().map(str::to_owned).collect()
}
fn ffmpeg_command() -> Command {
match crate::config::ffmpeg_root() {
Some(root) => { let mut command = Command::new(root.join("bin/ffmpeg"));
command.env("LD_LIBRARY_PATH", root.join("lib/x86_64-linux-gnu")); command }
None => Command::new("ffmpeg"),
}
}
/// A compiled-in encoder name does not prove that its driver can initialize.
/// Probe one synthetic 4:2:0 frame once per process before advertising HEVC.
pub async fn hevc_available() -> bool {
static AVAILABLE: tokio::sync::OnceCell<bool> = tokio::sync::OnceCell::const_new();
*AVAILABLE.get_or_init(|| async {
let requested = Encoder::requested();
let mut candidates = vec![requested];
if requested != Encoder::Cpu { candidates.push(Encoder::Cpu); }
for encoder in candidates {
let settings = VideoSettings { codec: VideoCodec::H265, ..Default::default() };
let mut command = ffmpeg_command();
command.args(["-hide_banner", "-loglevel", "error", "-nostdin"]);
if encoder == Encoder::Vaapi { command.args(["-vaapi_device", &vaapi_device()]); }
command.args(["-f", "lavfi", "-i", "color=black:size=256x256:rate=1", "-frames:v", "1", "-an"]);
if encoder == Encoder::Vaapi { command.args(["-vf", &vaapi_filter(None)]); }
command.args(encoder_options(&settings, encoder, "30"));
command.args(["-f", "hevc", "pipe:1"]).kill_on_drop(true).stderr(std::process::Stdio::null());
if let Ok(Ok(output)) = tokio::time::timeout(Duration::from_secs(5), command.output()).await {
if output.status.success() && is_independent_frame_for(&output.stdout, VideoCodec::H265) {
log::info!("HEVC encoder probe: {} ok", encoder.name_for(VideoCodec::H265));
return true;
}
}
}
log::info!("HEVC encoder probe: unavailable");
false
}).await
}
/// Round UP (+1ns float guard) so webrtc's floor(duration*90000) yields exactly `ticks`.
fn ticks_to_duration(ticks: u64) -> Duration {
Duration::from_nanos((ticks * 1_000_000_000 + RTP_CLOCK - 1) / RTP_CLOCK + 1)
@@ -131,7 +230,7 @@ struct DependencyGate {needs_idr:bool}
impl Default for DependencyGate {fn default()->Self{Self{needs_idr:true}}}
impl DependencyGate{
fn submit(&mut self,tx:&mpsc::Sender<Arc<CompleteFrame>>,frame:Arc<CompleteFrame>){
if self.needs_idr&&!is_independent_frame(&frame.data){return;}
if self.needs_idr&&!is_independent_frame_for(&frame.data, frame.codec){return;}
match tx.try_send(frame){
Ok(())=>self.needs_idr=false,
Err(mpsc::error::TrySendError::Full(_))=>self.needs_idr=true,
@@ -169,7 +268,7 @@ async fn read_encoder(
match read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,sequence.clone(),settings,idr.clone()).await {
Ok(())=>Ok(()),
Err(error) if !latest.is_closed()=>{
log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name(),error);
log::warn!("{} stream unavailable; switching scoped encoder to CPU: {}",encoder.name_for(settings.codec),error);
read_encoder_with(size,display,portal,latest,Encoder::Cpu,sequence,settings,idr).await
},
Err(error)=>Err(error),
@@ -182,6 +281,7 @@ async fn read_inprocess(
config: crate::inproc_encoder::Config, latest: mpsc::Sender<Arc<CompleteFrame>>,
sequence: Arc<std::sync::atomic::AtomicU64>, idr: Arc<AtomicBool>,
) -> Result<(), String> {
let codec = config.codec;
let (units_tx, mut units) = mpsc::channel::<Vec<u8>>(4);
let stop = Arc::new(AtomicBool::new(false));
// If this future is dropped (reader task aborted), the blocking encoder stops at its next packet.
@@ -193,7 +293,7 @@ async fn read_inprocess(
let mut gate = DependencyGate::default();
while let Some(data) = units.recv().await {
let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
gate.submit(&latest, Arc::new(CompleteFrame { sequence, completed_at: Instant::now(), data }));
gate.submit(&latest, Arc::new(CompleteFrame { codec, sequence, completed_at: Instant::now(), data }));
if latest.is_closed() { break; }
}
// Dropping the receiver also unblocks the encoder if it is waiting to send.
@@ -211,9 +311,15 @@ fn buffer_kbits(max_kbps: u32, encoder: Encoder) -> u32 {
if encoder == Encoder::Cpu { max_kbps.clamp(500, 8000) } else { (max_kbps / 8).clamp(100, 500) }
}
fn buffer_for(settings: &VideoSettings, encoder: Encoder) -> u32 {
if settings.codec == VideoCodec::H265 && encoder == Encoder::Cpu {
settings.max_kbps().unwrap_or(30_000) // one second of HEVC bitrate, including high presets
} else { buffer_kbits(settings.max_kbps().unwrap_or(30_000), encoder) }
}
fn rate_args(settings: &VideoSettings, encoder: Encoder) -> (String, String) {
let max_kbps = settings.max_kbps().unwrap_or(30_000);
(format!("{max_kbps}k"), format!("{}k", buffer_kbits(max_kbps, encoder)))
(format!("{max_kbps}k"), format!("{}k", buffer_for(settings, encoder)))
}
async fn read_encoder_with(
@@ -234,27 +340,22 @@ async fn read_encoder_with(
display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps),
gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32,
target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000,
max_bps: max_kbps * 1000, buffer_bits: i64::from(buffer_kbits(max_kbps as u32, encoder)) * 1000, encoder,
max_bps: max_kbps * 1000, buffer_bits: i64::from(buffer_for(&settings, encoder)) * 1000, encoder, codec: settings.codec,
damage: crate::inproc_encoder::damage_enabled(),
};
log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process",
display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name());
display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name_for(settings.codec));
return read_inprocess(config, latest, sequence, idr).await;
}
let size = format!("{}x{}", size.0, size.1);
let fps = settings.fps.to_string();
// Keep the IDR interval at 2 s whatever the frame rate.
let gop = gop_frames(settings.fps, false).to_string();
let (maxrate, bufsize) = rate_args(&settings, encoder);
let (maxrate, _) = rate_args(&settings, encoder);
let cq = settings.cq().to_string();
let target = settings.max_kbps().map_or("16M".to_string(), |k| format!("{}k", k * 3 / 4));
let (profile, pix_fmt) = chroma_args(&settings, encoder);
log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} pix_fmt={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,encoder.name(),if encoder == Encoder::Vaapi { "nv12" } else { pix_fmt });
log::info!("Desktop encoder: display={} source={} output={:?} fps={} cq={} maxrate={} encoder={} pix_fmt={} low_latency=true",display,size,settings.output,settings.fps,cq,maxrate,encoder.name_for(settings.codec),if encoder == Encoder::Vaapi { "nv12" } else { chroma_args(&settings, encoder).1 });
// A bundled FFmpeg prefix (config ffmpeg_root / UU_FFMPEG_ROOT) or `ffmpeg` on PATH.
let mut command=match crate::config::ffmpeg_root() {
Some(root) => { let mut c=Command::new(root.join("bin/ffmpeg")); c.env("LD_LIBRARY_PATH",root.join("lib/x86_64-linux-gnu")); c }
None => Command::new("ffmpeg"),
};
let mut command = ffmpeg_command();
command.args(["-hide_banner","-loglevel","error","-nostdin"]);
if encoder == Encoder::Vaapi { command.args(["-vaapi_device", &vaapi_device()]); }
if wayland {
@@ -271,27 +372,13 @@ async fn read_encoder_with(
} else if let Some((width, height)) = settings.output {
command.args(["-vf", &format!("scale={width}:{height}:flags=bilinear")]);
}
if encoder == Encoder::Vaapi {
command.args(vaapi_options(&settings, &gop));
} else if encoder == Encoder::Nvenc {
command.args(["-c:v","h264_nvenc","-preset","p4","-tune","ull",
"-profile:v",profile,"-pix_fmt",pix_fmt,"-g",&gop,"-bf","0",
"-rc","vbr","-cq",&cq,"-b:v",&target,"-maxrate",&maxrate,"-bufsize",&bufsize,
"-rc-lookahead","0","-zerolatency","1","-delay","0","-aud","1"]);
}else{
let x264 = format!("aud=1:repeat-headers=1:scenecut=0:keyint={gop}:min-keyint={gop}");
command.args([
"-c:v", "libx264", "-preset", "superfast", "-tune", "zerolatency",
"-profile:v", profile, "-pix_fmt", pix_fmt, "-g", &gop, "-threads", "4",
"-crf", &cq, "-maxrate", &maxrate, "-bufsize", &bufsize,
"-x264-params", &x264]);
}
command.args(encoder_options(&settings, encoder, &gop));
// x11grab paces against an absolute schedule; after a capture stall it grabs
// back-to-back to "catch up" (measured 69 frames in 200ms). vfr on a 1/FPS
// timebase drops those same-slot frames instead of encoding a burst.
let time_base = format!("1/{fps}");
if wayland { command.stdin(std::process::Stdio::piped()); }
let mut child=command.args(["-fps_mode","vfr","-enc_time_base",&time_base,"-f","h264","pipe:1"])
let mut child=command.args(["-fps_mode","vfr","-enc_time_base",&time_base,"-f",if settings.codec == VideoCodec::H265 { "hevc" } else { "h264" },"pipe:1"])
.kill_on_drop(true)
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::inherit())
@@ -359,14 +446,14 @@ async fn read_encoder_with(
pending.extend_from_slice(&chunk[..count]);
// A chunk can include the next AUD, so split before enforcing the bound.
loop {
let aud = aud_offsets(&pending);
let aud = aud_offsets_for(&pending, settings.codec);
if aud.len() < 2 { break; }
let next = aud[1];
if next > MAX_ACCESS_UNIT { return Err("Encoder access unit exceeds limit".into()); }
let frame: Vec<u8> = pending.drain(..next).collect();
let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1;
gate.submit(&latest, Arc::new(CompleteFrame {
sequence, completed_at: Instant::now(), data: frame,
codec: settings.codec, sequence, completed_at: Instant::now(), data: frame,
}));
if latest.is_closed() { return Ok(()); }
}
@@ -414,8 +501,8 @@ pub async fn probe(size:(u32,u32),display:String,portal:Option<Arc<crate::portal
Ok(())
}
pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalStaticSample>, size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
mut settings: tokio::sync::watch::Receiver<VideoSettings>, idr: Arc<AtomicBool>) -> Result<(), String> {
pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<crate::video_track::VideoTrack>, size: (u32, u32), display: String, portal:Option<Arc<crate::portal::PortalSession>>,
mut settings: tokio::sync::watch::Receiver<VideoSettings>, applied: Arc<tokio::sync::watch::Sender<VideoSettings>>, idr: Arc<AtomicBool>) -> Result<(), String> {
loop {
if transport_closed(&pc) { return Ok(()); }
if pc.connection_state() == RTCPeerConnectionState::Connected
@@ -425,7 +512,12 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
crate::capture::keep_display_awake()?;
let (latest, mut receiver) = mpsc::channel(4);
let sequence = Arc::new(std::sync::atomic::AtomicU64::new(0));
let current = *settings.borrow_and_update();
let mut current = *settings.borrow_and_update();
if !track.supports(current.codec) && track.supports(VideoCodec::H265) {
current.codec = VideoCodec::H265; // H265-only remote video offer.
applied.send_replace(current);
settings.borrow_and_update();
}
let mut reader = EncoderTask(tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone())));
let mut previous = None;
let mut timeline = RtpTimeline { fps: u64::from(current.fps), ..Default::default() };
@@ -439,19 +531,42 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
let mut stats = Instant::now();
loop {
tokio::select! {
result = &mut reader.0 => return result.map_err(|e| format!("Encoder reader task: {e}"))?,
result = &mut reader.0 => {
let result = result.map_err(|e| format!("Encoder reader task: {e}"))?;
if result.is_err() && current.codec == VideoCodec::H265 && track.supports(VideoCodec::H264) {
log::warn!("HEVC unavailable; falling back to negotiated H264");
current.codec = VideoCodec::H264;
applied.send_modify(|value| value.codec = VideoCodec::H264);
current = *settings.borrow_and_update();
while receiver.try_recv().is_ok() {}
reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), current, sequence.clone(), idr.clone()));
continue;
}
return result;
},
_ = health.tick() => { if transport_closed(&pc) { return Ok(()); } },
changed = settings.changed(), if settings_open => {
if changed.is_err() { settings_open = false; continue; }
// Restart the encoder with the viewer's new 帧率/画质; it begins with an IDR, and frames
// still queued from the old encoder remain a valid chain before it.
let next = *settings.borrow_and_update();
let codec_changed = next.codec != current.codec;
current = next;
timeline.fps = u64::from(next.fps);
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await;
if codec_changed {
reader.0.abort();
let _ = (&mut reader.0).await;
// Drain only after the old producer is gone, before the new IDR can enter.
while receiver.try_recv().is_ok() {}
reader.0 = tokio::spawn(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone()));
} else {
reader.restart(read_encoder(size, display.clone(), portal.clone(), latest.clone(), next, sequence.clone(), idr.clone())).await;
}
},
frame = receiver.recv() => {
// A closed queue means the reader ended; its JoinHandle carries the reason.
let Some(frame) = frame else { return (&mut reader.0).await.map_err(|e| format!("Encoder reader task: {e}"))?; };
if frame.codec != current.codec { continue; }
let duration = timeline.duration_after(frame.completed_at);
if let Some(last) = previous { skipped += frame.sequence.saturating_sub(last + 1); }
max_age_ms = max_age_ms.max(frame.completed_at.elapsed().as_millis());
@@ -460,7 +575,7 @@ pub async fn stream_desktop(pc: Arc<RTCPeerConnection>, track: Arc<TrackLocalSta
data: frame.data.clone().into(), duration, ..Default::default()
};
let extensions = [crate::rtp_metadata::capture_index(0), crate::rtp_metadata::new_frame()];
let send = track.write_sample_with_extensions(&sample, &extensions);
let send = track.write_sample_with_extensions(frame.codec, &sample, &extensions);
let send_started=Instant::now();
tokio::pin!(send);
loop {
@@ -533,7 +648,7 @@ mod tests {
let old_tx = tx.clone();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let frame = |sequence, idr| Arc::new(CompleteFrame {
sequence, completed_at: Instant::now(),
sequence, codec: VideoCodec::H264, completed_at: Instant::now(),
data: vec![0, 0, 1, 9, 240, 0, 0, 1, if idr { 5 } else { 1 }, 42],
});
let mut reader = EncoderTask(tokio::spawn(async move {
@@ -627,7 +742,7 @@ mod tests {
async fn bounded_queue_never_sends_broken_prediction_chain_after_overload(){
let(tx,mut rx)=mpsc::channel(1);
let mut gate=DependencyGate::default();
let frame=|sequence,idr|Arc::new(CompleteFrame{sequence,completed_at:Instant::now(),data:vec![0,0,1,9,240,0,0,1,if idr{5}else{1},42]});
let frame=|sequence,idr|Arc::new(CompleteFrame{sequence,codec:VideoCodec::H264,completed_at:Instant::now(),data:vec![0,0,1,9,240,0,0,1,if idr{5}else{1},42]});
gate.submit(&tx,frame(1,false));assert!(rx.try_recv().is_err()); // no IDR yet
gate.submit(&tx,frame(2,true));
gate.submit(&tx,frame(3,false)); // queue full: P3 lost
@@ -637,6 +752,60 @@ mod tests {
gate.submit(&tx,frame(31,true));assert_eq!(rx.recv().await.unwrap().sequence,31);
gate.submit(&tx,frame(32,false));assert_eq!(rx.recv().await.unwrap().sequence,32);
}
#[tokio::test]
#[ignore = "requires installed FFmpeg with a working HEVC encoder"]
async fn hevc_capability_probe_initializes_real_encoder() {
assert!(hevc_available().await);
}
#[test]
fn hevc_annex_b_boundaries_and_irap_use_two_byte_headers() {
let bytes = [0,0,0,1,70,1,80,0,0,1,64,1,42,0,0,1,38,1,42,0,0,1,70,1,80];
assert_eq!(aud_offsets_for(&bytes, VideoCodec::H265), vec![0,19]);
assert!(is_independent_frame_for(&bytes[..19], VideoCodec::H265));
for kind in 16..=21 {
assert!(is_independent_frame_for(&[0,0,1,kind << 1,1,42], VideoCodec::H265));
}
assert!(!is_independent_frame_for(&[0,0,1,2,1,42], VideoCodec::H265));
assert!(!is_independent_frame_for(&[0,0,1,38], VideoCodec::H265));
assert!(!is_independent_frame_for(&[0,0,1,38,0,42], VideoCodec::H265));
assert!(!is_independent_frame_for(&[0,0,1,166,1,42], VideoCodec::H265));
assert!(!is_independent_frame_for(&[0,0,1,38,1,42,0,0,1,2,1,42], VideoCodec::H265));
}
#[tokio::test]
async fn hevc_dependency_gate_recovers_only_at_irap_after_overflow() {
let (tx, mut rx) = mpsc::channel(1);
let mut gate = DependencyGate::default();
let frame = |sequence, irap| Arc::new(CompleteFrame { codec: VideoCodec::H265,
sequence, completed_at: Instant::now(), data: vec![0,0,1,if irap { 38 } else { 2 },1,42] });
gate.submit(&tx, frame(1, false));
assert!(rx.try_recv().is_err());
gate.submit(&tx, frame(2, true));
gate.submit(&tx, frame(3, false));
assert_eq!(rx.recv().await.unwrap().sequence, 2);
gate.submit(&tx, frame(4, false));
assert!(rx.try_recv().is_err());
gate.submit(&tx, frame(5, true));
assert_eq!(rx.recv().await.unwrap().sequence, 5);
}
#[test]
fn hevc_options_keep_420_low_latency_and_existing_rate_policy() {
let settings = VideoSettings { codec: VideoCodec::H265, ..Default::default() };
let cpu = cpu_hevc_options(&settings, "30");
for pair in [["-c:v", "libx265"], ["-profile:v", "main"], ["-pix_fmt", "yuv420p"],
["-bf", "0"], ["-bufsize", "30000k"]] {
assert!(cpu.windows(2).any(|window| window == pair));
}
let params = x265_params("30");
for option in ["aud=1", "repeat-headers=1", "bframes=0", "rc-lookahead=0", "keyint=30", "min-keyint=30"] {
assert!(params.split(':').any(|value| value == option));
}
let vaapi = vaapi_options(&settings, "30");
for pair in [["-c:v", "hevc_vaapi"], ["-profile:v", "main"], ["-bf", "0"], ["-bufsize", "500k"]] {
assert!(vaapi.windows(2).any(|window| window == pair));
}
assert_eq!(Encoder::Nvenc.name_for(VideoCodec::H265), "hevc_nvenc");
assert_eq!(Encoder::Cpu.name_for(VideoCodec::H264), "libx264");
}
#[test]
fn nvenc_idr_with_parameter_sets_is_independent(){
// AUD, SPS, PPS, IDR slice as h264_nvenc emits every GOP (verified on the 3090).
+173
View File
@@ -0,0 +1,173 @@
//! One negotiated video m-line/SSRC, with codec-specific payload types.
//! A single RTP sequence/timestamp survives format changes (never two independent SSRC timelines).
use std::{any::Any, sync::atomic::{AtomicBool, Ordering}};
use async_trait::async_trait;
use tokio::sync::Mutex;
use webrtc::{error::{Error, Result}, media::Sample, rtp::{packet::Packet, header::Header, packetizer::Payloader}};
use webrtc::rtp_transceiver::rtp_codec::{RTCRtpCodecCapability, RTCRtpCodecParameters, RTPCodecType};
use webrtc::track::track_local::{TrackLocal, TrackLocalContext, track_local_static_rtp::TrackLocalStaticRTP};
use crate::capture_setting::VideoCodec;
/// Official SDK GetSupportedFormats constructs H265 with an empty parameter map.
pub fn register_hevc(media: &mut webrtc::api::media_engine::MediaEngine) -> Result<()> {
media.register_codec(RTCRtpCodecParameters { capability: RTCRtpCodecCapability {
mime_type: VideoCodec::H265.mime().into(), clock_rate: 90_000,
rtcp_feedback: vec![webrtc::rtp_transceiver::RTCPFeedback { typ: "nack".into(), parameter: String::new() },
webrtc::rtp_transceiver::RTCPFeedback { typ: "nack".into(), parameter: "pli".into() }],
..Default::default() }, payload_type: 126, ..Default::default() }, RTPCodecType::Video)
}
#[derive(Debug)]
struct Timeline { sequence: u16, timestamp: u32, h264: webrtc::rtp::codecs::h264::H264Payloader }
#[derive(Debug)]
pub struct VideoTrack {
h264: TrackLocalStaticRTP, h265: TrackLocalStaticRTP,
has_h264: AtomicBool, has_h265: AtomicBool, timeline: Mutex<Timeline>,
progress: tokio::sync::watch::Sender<(u64, VideoCodec)>,
}
impl VideoTrack {
pub fn new() -> Self {
let track = |codec: VideoCodec| TrackLocalStaticRTP::new(
RTCRtpCodecCapability { mime_type: codec.mime().into(), clock_rate: 90_000, ..Default::default() },
"video_0".into(), "video_0".into());
let (progress, _) = tokio::sync::watch::channel((0, VideoCodec::H264));
Self { progress, h264: track(VideoCodec::H264), h265: track(VideoCodec::H265), has_h264: AtomicBool::new(false),
has_h265: AtomicBool::new(false), timeline: Mutex::new(Timeline { sequence: rand::random(), timestamp: rand::random(), h264: Default::default() }) }
}
pub fn supports(&self, codec: VideoCodec) -> bool {
match codec { VideoCodec::H264 => &self.has_h264, VideoCodec::H265 => &self.has_h265 }.load(Ordering::Acquire)
}
pub fn revision(&self) -> u64 { self.progress.borrow().0 }
/// Format-change RPC success follows a transmitted frame, including a negotiated fallback.
pub async fn wait_for_applied(&self, since: u64, settings: &tokio::sync::watch::Sender<crate::capture_setting::VideoSettings>) -> bool {
let mut progress = self.progress.subscribe();
tokio::time::timeout(std::time::Duration::from_secs(10), async {
loop {
let (revision, codec) = *progress.borrow_and_update();
if revision > since && codec == settings.borrow().codec { return true; }
if progress.changed().await.is_err() { return false; }
}
}).await.unwrap_or(false)
}
pub async fn write_sample_with_extensions(&self, codec: VideoCodec, sample: &Sample, extensions: &[webrtc::rtp::extension::HeaderExtension]) -> Result<()> {
if !self.supports(codec) { return Err(Error::ErrUnsupportedCodec); }
let track = match codec { VideoCodec::H264 => &self.h264, VideoCodec::H265 => &self.h265 };
if track.all_binding_paused().await { return Ok(()); }
let mut state = self.timeline.lock().await;
// Leave room for the RTP header and negotiated UU extensions below the 1200 byte MTU.
let payloads = match codec {
VideoCodec::H264 => state.h264.payload(1160, &sample.data).map_err(Error::Rtp)?,
VideoCodec::H265 => crate::hevc::payload(1160, &sample.data).map_err(Error::new)?,
};
let count = payloads.len();
let mut complete = count > 0;
for (index, payload) in payloads.into_iter().enumerate() {
let packet = Packet { header: Header { version: 2, sequence_number: state.sequence, timestamp: state.timestamp,
marker: index + 1 == count, ..Default::default() }, payload };
state.sequence = state.sequence.wrapping_add(1);
complete &= track.write_rtp_with_extensions(&packet, extensions).await? > 0;
}
if complete { self.progress.send_modify(|value| { value.0 += 1; value.1 = codec; }); }
state.timestamp = state.timestamp.wrapping_add((sample.duration.as_secs_f64() * 90_000.0) as u32);
Ok(())
}
}
#[async_trait]
impl TrackLocal for VideoTrack {
async fn bind(&self, context: &TrackLocalContext) -> Result<RTCRtpCodecParameters> {
let h264 = self.h264.bind(context).await.ok();
let h265 = self.h265.bind(context).await.ok();
self.has_h264.store(h264.is_some(), Ordering::Release);
self.has_h265.store(h265.is_some(), Ordering::Release);
log::info!("Video track negotiated: h264={} h265={}", h264.is_some(), h265.is_some());
h264.or(h265).ok_or(Error::ErrUnsupportedCodec)
}
async fn unbind(&self, context: &TrackLocalContext) -> Result<()> {
if self.has_h264.swap(false, Ordering::AcqRel) { self.h264.unbind(context).await?; }
if self.has_h265.swap(false, Ordering::AcqRel) { self.h265.unbind(context).await?; }
Ok(())
}
fn id(&self) -> &str { "video_0" }
fn stream_id(&self) -> &str { "video_0" }
fn kind(&self) -> RTPCodecType { RTPCodecType::Video }
fn as_any(&self) -> &dyn Any { self }
}
#[cfg(test)]
mod tests {
use super::*;
use webrtc::{api::{APIBuilder, media_engine::MediaEngine}, peer_connection::configuration::RTCConfiguration};
async fn negotiate(h264: bool, hevc: bool) {
let mut remote_media = MediaEngine::default();
if h264 { remote_media.register_codec(RTCRtpCodecParameters { capability: RTCRtpCodecCapability {
mime_type: VideoCodec::H264.mime().into(), clock_rate: 90_000,
sdp_fmtp_line: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f".into(), ..Default::default()
}, payload_type: 102, ..Default::default() }, RTPCodecType::Video).unwrap(); }
if hevc { register_hevc(&mut remote_media).unwrap(); }
let mut setting = webrtc::api::setting_engine::SettingEngine::default();
setting.set_ice_multicast_dns_mode(webrtc::ice::mdns::MulticastDnsMode::Disabled);
let remote = APIBuilder::new().with_setting_engine(setting.clone()).with_media_engine(remote_media).build().new_peer_connection(RTCConfiguration::default()).await.unwrap();
remote.add_transceiver_from_kind(RTPCodecType::Video, None).await.unwrap();
let mut media = MediaEngine::default(); media.register_default_codecs().unwrap(); register_hevc(&mut media).unwrap();
let host = APIBuilder::new().with_setting_engine(setting).with_media_engine(media).build().new_peer_connection(RTCConfiguration::default()).await.unwrap();
let (packets_tx, mut packets_rx) = tokio::sync::mpsc::unbounded_channel();
remote.on_track(Box::new(move |remote_track, _, _| {
let packets_tx = packets_tx.clone();
Box::pin(async move {
tokio::spawn(async move {
while let Ok((packet, _)) = remote_track.read_rtp().await {
if packets_tx.send(packet).is_err() { break; }
}
});
})
}));
let track = std::sync::Arc::new(VideoTrack::new());
host.add_track(track.clone()).await.unwrap();
let offer = remote.create_offer(None).await.unwrap();
let mut gathered = remote.gathering_complete_promise().await;
remote.set_local_description(offer).await.unwrap();
gathered.recv().await;
host.set_remote_description(remote.local_description().await.unwrap()).await.unwrap();
let answer = host.create_answer(None).await.unwrap();
assert_eq!(answer.sdp.contains("H265/90000"), hevc);
assert_eq!(answer.sdp.contains("H264/90000"), h264);
let mut gathered = host.gathering_complete_promise().await;
host.set_local_description(answer).await.unwrap();
gathered.recv().await;
remote.set_remote_description(host.local_description().await.unwrap()).await.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(10), async {
while !track.supports(VideoCodec::H264) && !track.supports(VideoCodec::H265) { tokio::time::sleep(std::time::Duration::from_millis(10)).await; }
}).await.unwrap();
assert_eq!(track.supports(VideoCodec::H264), h264);
assert_eq!(track.supports(VideoCodec::H265), hevc);
tokio::time::timeout(std::time::Duration::from_secs(10), async {
while host.connection_state() != webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState::Connected {
tokio::time::sleep(std::time::Duration::from_millis(10)).await;
}
}).await.unwrap();
let codecs = if !h264 { vec![VideoCodec::H265] } else if hevc { vec![VideoCodec::H264, VideoCodec::H265, VideoCodec::H264] } else { vec![VideoCodec::H264] };
let mut previous: Option<Packet> = None;
for codec in codecs {
let data = match codec { VideoCodec::H264 => bytes::Bytes::from_static(&[0,0,1,65,1,2]),
VideoCodec::H265 => bytes::Bytes::from_static(&[0,0,1,38,1,2]) };
let (settings, _) = tokio::sync::watch::channel(crate::capture_setting::VideoSettings { codec, ..Default::default() });
let applied = track.wait_for_applied(track.revision(), &settings);
tokio::pin!(applied);
assert!(tokio::time::timeout(std::time::Duration::from_millis(1), &mut applied).await.is_err());
track.write_sample_with_extensions(codec, &Sample { data, duration: std::time::Duration::from_millis(20), ..Default::default() }, &[]).await.unwrap();
assert!(applied.await);
let packet = tokio::time::timeout(std::time::Duration::from_secs(3), packets_rx.recv()).await.unwrap().unwrap();
assert_eq!(packet.header.payload_type, if codec == VideoCodec::H264 { 102 } else { 126 });
assert!(packet.header.marker);
if let Some(previous) = previous {
assert_eq!(packet.header.ssrc, previous.header.ssrc);
assert_eq!(packet.header.sequence_number, previous.header.sequence_number.wrapping_add(1));
assert_eq!(packet.header.timestamp, previous.header.timestamp.wrapping_add(1800));
}
previous = Some(packet);
}
host.close().await.unwrap(); remote.close().await.unwrap();
}
#[tokio::test]
async fn negotiated_codecs_only_and_legacy_h264_offer() { negotiate(true, false).await; negotiate(true, true).await; negotiate(false, true).await; }
}
+36 -12
View File
@@ -53,6 +53,9 @@ impl WebRtcSession {
m.register_default_codecs()
.map_err(|e| format!("Register standard codecs failed: {}", e))?;
let hevc = desktop && crate::video::hevc_available().await;
if hevc { crate::video_track::register_hevc(&mut m).map_err(|e| format!("Register HEVC failed: {e}"))?; }
m.register_header_extension(
webrtc::rtp_transceiver::rtp_codec::RTCRtpHeaderExtensionCapability { uri: crate::rtp_metadata::CAPTURE_INDEX_URI.into() },
webrtc::rtp_transceiver::rtp_codec::RTPCodecType::Video, None,
@@ -162,7 +165,7 @@ impl WebRtcSession {
tokio::spawn(async move {
let ctx = ctx.lock().await.clone();
if let (Some(client), Some(ice)) = (&ctx.client_id, ctx.ice_id.as_ref().and_then(|v| v.as_str())) {
let payload = crate::capability::h264_device_capability(client, ice, 0, crate::video::FPS as i32, 5);
let payload = crate::capability::device_capability(client, ice, 0, crate::video::FPS as i32, 5, hevc);
if let Err(e) = sig.emit_event(crate::capability::FORWARD_SETTING_EVENT, payload).await {
log::error!("Device capability notification failed: {}", e);
}
@@ -179,14 +182,7 @@ impl WebRtcSession {
}));
// 注册视频轨
let video_track = Arc::new(TrackLocalStaticSample::new(
RTCRtpCodecCapability {
mime_type: "video/H264".to_owned(),
..Default::default()
},
"video_0".to_owned(),
"video_0".to_owned(),
));
let video_track = Arc::new(crate::video_track::VideoTrack::new());
// Set by PLI/FIR; the desktop encoder answers with an IDR on its next frame.
let idr_request = Arc::new(std::sync::atomic::AtomicBool::new(false));
@@ -274,9 +270,11 @@ impl WebRtcSession {
}
let portal=capture.portal.clone();
// File/terminal/port-mapping connections carry no desktop video.
let control_video_track = video_track.clone();
let stream_settings = video_settings.clone();
if desktop {
tokio::spawn(async move {
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, idr_request).await {
if let Err(e) = crate::video::stream_desktop(streaming_pc, video_track, capture_size, capture_display, portal, video_settings_rx, stream_settings, idr_request).await {
log::error!("Live desktop stream stopped: {}", e);
}
});
@@ -338,6 +336,7 @@ impl WebRtcSession {
let peer_files = peer_files.clone();
let peer_port_mapping = port_mapping.clone();
let peer_video_settings = video_settings.clone();
let peer_video_track = control_video_track.clone();
let peer_actions = peer_actions.clone();
if label == "FILE_DATA_CHANNEL" { *file_channel.lock().unwrap() = Some(d.clone()); }
@@ -389,9 +388,11 @@ impl WebRtcSession {
let rich_outgoing=rich_state.clone();
let open_actions = channel_actions.clone();
let open_video = channel_video_settings.clone();
let open_track = peer_video_track.clone();
d.on_open(Box::new(move || {
let init_actions = open_actions.clone();
let init_video = open_video.clone();
let init_track = open_track.clone();
let channel = channel.clone();
let sequence = initial_sequence.clone();
let screen = screen.clone();
@@ -462,9 +463,18 @@ impl WebRtcSession {
(13, Ok(crate::initialization::envelope(next_seq(), timestamp, 13, &init_actions.metrics_payload(desktop_sound, desktop_permission)))),
(26, Ok(crate::initialization::envelope(next_seq(), timestamp, 26,
&crate::system_metrics::remote_download_path(&home)))),
(18, Ok(crate::initialization::envelope(next_seq(), timestamp, 18,
&crate::capture_setting::codec_negotiation(screen_size, crate::video::FPS as u32, init_track.supports(crate::capture_setting::VideoCodec::H265))))),
(7, crate::capture_setting::screen_sources(next_seq(), timestamp, screen_size, init_video.borrow().output, "Linux desktop")),
];
if desktop && screen.init_error.is_none() {
let codec = crate::capture_setting::preferred_codec(init_track.supports(crate::capture_setting::VideoCodec::H264),
init_track.supports(crate::capture_setting::VideoCodec::H265));
init_video.send_if_modified(|current| if current.codec != codec {
log::info!("Desktop codec selected: {:?}", codec); current.codec = codec; true } else { false });
}
for (field, message) in messages {
if (!desktop || screen.init_error.is_some()) && field == 18 { continue; }
if desktop && screen.init_error.is_some() && matches!(field,7|8) { continue; }
let message = match message {
Ok(message) => message,
@@ -522,6 +532,7 @@ impl WebRtcSession {
let file_worker = file_session.clone();
let mapping = peer_port_mapping.clone();
let video_settings = channel_video_settings.clone();
let negotiated_track = peer_video_track.clone();
let actions = channel_actions.clone();
Box::pin(async move {
@@ -532,8 +543,17 @@ impl WebRtcSession {
use crate::protocol::WireValue;
let rpc = fields.iter().find_map(|f| match (f.number, &f.value) { (21, WireValue::Bytes(b)) => Some(*b), _ => None });
let rpc_fields = rpc.and_then(|r| crate::protocol::decode_wire(r).ok());
let output_before = video_settings.borrow().output;
if let Some(response) = rpc_fields.as_deref().and_then(|r| crate::capture_setting::handle_rpc(r, &video_settings, input_size)) {
let before = *video_settings.borrow();
let output_before = before.output;
let revision = negotiated_track.revision();
if let Some(response) = rpc_fields.as_deref().and_then(|r| crate::capture_setting::handle_rpc_with_codecs(r, &video_settings, input_size, negotiated_track.supports(crate::capture_setting::VideoCodec::H264), negotiated_track.supports(crate::capture_setting::VideoCodec::H265))) {
let codec_changed = video_settings.borrow().codec != before.codec;
if codec_changed && !negotiated_track.wait_for_applied(revision, &video_settings).await {
// Official host replies after startVideoCaptureAndSendRpcResponse.
// Do not fabricate a codec error number or acknowledge an encoder that never started.
log::warn!("Codec switch did not produce a transmitted frame; success reply withheld");
return;
}
let ts = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default().as_secs();
let next = || out_sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let _ = rpc_out.send(crate::initialization::envelope(next(), ts, 22, &response));
@@ -865,6 +885,10 @@ impl WebRtcSession {
};
log::info!("Offer x-uuremote-quic: {}", quic_summary);
log::info!("Offer includes UU capture-index extension: {}", sdp_str.lines().any(|l| l.starts_with("a=extmap:") && l.contains(crate::rtp_metadata::CAPTURE_INDEX_URI)));
let mut codecs: Vec<&str> = sdp_str.lines().filter_map(|l| l.strip_prefix("a=rtpmap:")).filter_map(|l| l.split_whitespace().nth(1))
.filter_map(|c| c.split('/').next()).collect();
codecs.dedup();
log::info!("Offer codecs: {:?}", codecs);
let offer = RTCSessionDescription::offer(sdp_str)
.map_err(|e| format!("Invalid offer: {}", e))?;