diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd39a57..67c95a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -106,7 +106,7 @@ jobs: ASAN_OPTIONS: detect_leaks=1 UBSAN_OPTIONS: halt_on_error=1 run: | - for source in tests/portal_engine.c tests/portal_input.c tests/portal_eis.c tests/desktop_power.c tests/wayland_transform.c; do + for source in tests/portal_engine.c tests/portal_dmabuf.c tests/portal_input.c tests/portal_eis.c tests/desktop_power.c tests/wayland_transform.c; do target="/tmp/$(basename "$source" .c)" cc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer \ -Wall -Wextra -Wno-unused-parameter "$source" -o "$target" \ @@ -123,10 +123,19 @@ jobs: sudo apt-get update sudo apt-get install -y build-essential pkg-config \ libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \ - libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage0-dev + libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage0-dev \ + libpulse-dev libopus-dev xvfb xauth \ + libwayland-dev libxkbcommon-dev libglib2.0-dev libpipewire-0.3-dev libei-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 + - name: Read-only uinput text keymap regression + run: | + cc -O1 -g -fsanitize=address,undefined -fno-omit-frame-pointer \ + -Wall -Wextra tests/uinput_keymap.c -o /tmp/uu-uinput-keymap-test -lX11 + ASAN_OPTIONS=detect_leaks=1 UBSAN_OPTIONS=halt_on_error=1 \ + xvfb-run -a /tmp/uu-uinput-keymap-test - run: cargo test --locked --features inproc-av + - run: cargo test --locked --features inproc-av,wayland,portal test-linux-arm64: # aarch64 hosts (Ampere A1, arm64 VMs, Apple-silicon VMs) build from source, so keep that diff --git a/README.en.md b/README.en.md index 200a171..32b0090 100644 --- a/README.en.md +++ b/README.en.md @@ -210,6 +210,8 @@ sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard sudo apt install libglib2.0-dev libpipewire-0.3-dev libei-dev # optional: headers for the inproc-av VAAPI/DRI3 engines (libva itself is loaded at runtime) sudo apt install libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage0-dev +# In-process audio capture and Opus encoding for inproc-av +sudo apt install libpulse-dev libopus-dev ``` - A Rust toolchain is required (). @@ -228,6 +230,9 @@ cargo build --release --features wayland,portal # + GNOME/KDE on Wayland The viewer's `viewer-ffmpeg` feature supports **FFmpeg 6.x only** (`pkg-config --modversion libavcodec` = 60.x); FFmpeg 7+ fails inside `ffmpeg-sys-next` with `avfft.h file not found` (#63). The host's optional `--features inproc-av` encodes with hardware inside the process on X11. It does not link FFmpeg development libraries; it loads the NVIDIA driver libraries (`libcuda.so.1`, `libnvidia-encode.so.1`) at runtime for NVENC (needs a driver with NVENC API 12.1, 530 or newer) and `libva.so.2`/`libva-drm.so.2` for VAAPI. When the capture and encoder sizes match, NVENC reads the XShm buffer directly, saving one host-to-device copy; VAAPI runs in process under the same feature (DRI3 DMA-BUF zero copy, or XShm upload without DRI3, then VPP to NV12 and VCN; H.264/HEVC). CPU encoding and `UU_CAPTURE=grab` keep the FFmpeg process; if the in-process hardware session fails it falls back to the matching FFmpeg process pipeline, then to the CPU. The default build and the release packages need neither feature. +On GNOME/KDE Wayland, enable in-process encoding with `cargo build --release --features wayland,portal,inproc-av` and `set-encoder vaapi`; support is limited to Portal + VAAPI. +`UU_PORTAL_DMABUF=0` disables DMA-BUF in favor of in-process host upload; `UU_VIDEO_PIPELINE=process` restores external FFmpeg pipelines. + ### 3. Log in Run inside your graphical session: @@ -292,6 +297,8 @@ sudo udevadm control --reload && sudo udevadm trigger --name-match=uinput `uaccess` grants access only to the user sitting at the machine. `UU_INPUT_BACKEND=uinput|native` forces a backend. X11 keeps XTest by default because it handles held keys correctly on high-latency links. +ASCII text mapping through uinput requires X11/XWayland XKB; unmappable text keeps the existing text/clipboard path. + ## Remote assistance and the main window (experimental) As in the official client, this host shows a **device ID** and a **verification code**; whoever you give them to can diff --git a/README.md b/README.md index a297146..0d49f90 100644 --- a/README.md +++ b/README.md @@ -193,6 +193,8 @@ sudo apt install libwayland-dev libxkbcommon-dev wl-clipboard sudo apt install libglib2.0-dev libpipewire-0.3-dev libei-dev # 可选:inproc-av 的 VAAPI/DRI3 引擎编译所需(libva 本体运行时加载) sudo apt install libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage0-dev +# inproc-av 的进程内音频采集与 Opus 编码 +sudo apt install libpulse-dev libopus-dev ``` - 需要 Rust 工具链()。 @@ -210,6 +212,9 @@ cargo build --release --features wayland,portal # + GNOME/KDE Wayland 控制端的 `viewer-ffmpeg` 只支持 **FFmpeg 6.x**(`pkg-config --modversion libavcodec` 为 60.x);FFmpeg 7 及以上会在依赖 `ffmpeg-sys-next` 里报 `avfft.h file not found`(#63)。 服务端可选的 `--features inproc-av` 是 X11 下的进程内硬件编码:不链接 FFmpeg 开发库,NVENC 运行时加载 NVIDIA 驱动库(`libcuda.so.1`、`libnvidia-encode.so.1`,需要支持 NVENC API 12.1 的驱动,530 及以上),VAAPI 运行时加载 `libva.so.2`/`libva-drm.so.2`。截屏与编码尺寸一致时 NVENC 直接读取 XShm 缓冲,省去一次主机到显存的拷贝;VAAPI 在同一特性下也进程内运行(DRI3 DMA-BUF 零拷贝,无 DRI3 时 XShm 上传,VPP 转 NV12 后送 VCN,支持 H.264/H.265)。CPU 编码或 `UU_CAPTURE=grab` 仍走外部 FFmpeg 进程;进程内硬件编码启动失败时先回退到对应的 FFmpeg 进程管线,再回退到 CPU。默认构建和预编译包都不需要该特性。 +GNOME/KDE Wayland 的进程内编码用 `cargo build --release --features wayland,portal,inproc-av` 配合 `set-encoder vaapi` 启用,目前仅支持 Portal + VAAPI。 +`UU_PORTAL_DMABUF=0` 禁用 DMA-BUF、改用进程内上传;`UU_VIDEO_PIPELINE=process` 恢复外部 FFmpeg 管线。 + ### 3. 登录 在图形桌面里运行: @@ -272,6 +277,8 @@ sudo udevadm control --reload && sudo udevadm trigger --name-match=uinput `uaccess` 只把权限给当前坐在这台机器前的登录用户。`UU_INPUT_BACKEND=uinput|native` 可以强制指定后端。 X11 默认继续用 XTest,因为它能正确处理网络延迟下按住不放的按键。 +uinput 的 ASCII 文本映射依赖 X11/XWayland XKB;不可映射的文字保留原有文字输入/剪贴板路径。 + ## 远程协助与主窗口(实验性) 和官方客户端的「远程协助」一样:本机显示**设备 ID** 和**验证码**,把它们发给信任的人,对方在 UU 远程的「开始协助」里输入即可控制本机。 diff --git a/build.rs b/build.rs index d339b77..cfefe31 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,10 @@ fn main() { + println!("cargo:rerun-if-changed=c_engine/dmabuf_frame.h"); println!("cargo:rerun-if-changed=c_engine/wayland_transform.h"); println!("cargo:rerun-if-changed=c_engine/portal_clipboard.inc"); println!("cargo:rerun-if-changed=c_engine/x11_engine.c"); + println!("cargo:rerun-if-changed=c_engine/uinput_keymap.c"); + println!("cargo:rerun-if-changed=c_engine/uinput_keymap.h"); println!("cargo:rerun-if-changed=c_engine/nvenc_engine.c"); println!("cargo:rerun-if-changed=c_engine/nvenc_engine.h"); println!("cargo:rerun-if-changed=vendor/ffnvcodec/ffnvcodec/dynlink_cuda.h"); @@ -30,9 +33,16 @@ fn main() { } let mut x11_build = cc::Build::new(); x11_build.file("c_engine/x11_engine.c") + .file("c_engine/uinput_keymap.c") .file("c_engine/clipboard.c") .file("c_engine/pam_check.c"); if std::env::var_os("CARGO_FEATURE_INPROC_AV").is_some() { + println!("cargo:rerun-if-changed=c_engine/audio_engine.c"); + x11_build.file("c_engine/audio_engine.c"); + let audio_flags = std::process::Command::new("pkg-config").args(["--cflags", "libpulse", "opus"]) + .output().expect("inproc-av requires libpulse and opus development libraries"); + assert!(audio_flags.status.success(), "inproc-av requires libpulse-dev and libopus-dev"); + for flag in String::from_utf8_lossy(&audio_flags.stdout).split_whitespace() { x11_build.flag(flag); } x11_build.file("c_engine/nvenc_engine.c"); x11_build.file("c_engine/vaapi_engine.c"); x11_build.file("c_engine/dri3_capture.c"); @@ -47,6 +57,8 @@ fn main() { if std::env::var_os("CARGO_FEATURE_INPROC_AV").is_some() { // Emit them after the static C archive: GNU ld --as-needed would otherwise // discard the shared libraries before libx11_engine.a references their symbols. + println!("cargo:rustc-link-lib=pulse"); + println!("cargo:rustc-link-lib=opus"); println!("cargo:rustc-link-lib=gbm"); println!("cargo:rustc-link-lib=X11-xcb"); println!("cargo:rustc-link-lib=xcb"); diff --git a/c_engine/audio_engine.c b/c_engine/audio_engine.c new file mode 100644 index 0000000..3131cb8 --- /dev/null +++ b/c_engine/audio_engine.c @@ -0,0 +1,124 @@ +#include +#include +#include +#include +#include +#include "i18n.h" + +typedef struct { + pa_mainloop *loop; + pa_context *context; + pa_stream *stream; + OpusEncoder *encoder; + char *source; + int failed, connecting; + int16_t pcm[960 * 2]; + size_t filled, offset; +} UuAudio; + +void uu_audio_close(UuAudio *audio) { + if (!audio) return; + if (audio->stream) { + pa_stream_disconnect(audio->stream); + pa_stream_unref(audio->stream); + } + if (audio->context) { + pa_context_disconnect(audio->context); + pa_context_unref(audio->context); + } + if (audio->loop) pa_mainloop_free(audio->loop); + if (audio->encoder) opus_encoder_destroy(audio->encoder); + free(audio->source); + free(audio); +} + +static void connect_source(UuAudio *audio, const char *source) { + if (!source || !*source) { audio->failed = 1; return; } + pa_sample_spec spec = { PA_SAMPLE_S16NE, 48000, 2 }; + audio->stream = pa_stream_new(audio->context, L("UU 远程音频", "UU Remote audio"), &spec, NULL); + pa_buffer_attr attr = { (uint32_t)-1, (uint32_t)-1, (uint32_t)-1, (uint32_t)-1, sizeof(audio->pcm) }; + if (!audio->stream || pa_stream_connect_record(audio->stream, source, &attr, + PA_STREAM_ADJUST_LATENCY | PA_STREAM_DONT_MOVE) < 0) audio->failed = 1; +} + +static void source_info(pa_context *context, const pa_source_info *info, int end, void *data) { + (void)context; + UuAudio *audio = data; + if (info && !audio->stream) connect_source(audio, info->name); + if (end && !audio->stream) audio->failed = 1; +} + +static void sink_info(pa_context *context, const pa_sink_info *info, int end, void *data) { + (void)context; + UuAudio *audio = data; + if (info && !audio->stream) connect_source(audio, info->monitor_source_name); + if (end && !audio->stream) audio->failed = 1; +} + +static void server_info(pa_context *context, const pa_server_info *info, void *data) { + UuAudio *audio = data; + if (!info || !info->default_sink_name) { audio->failed = 1; return; } + pa_operation *op = pa_context_get_sink_info_by_name(context, info->default_sink_name, sink_info, audio); + if (op) pa_operation_unref(op); + else audio->failed = 1; +} + +UuAudio *uu_audio_open(const char *source) { + UuAudio *audio = calloc(1, sizeof(*audio)); + if (!audio) return NULL; + int error; + audio->source = strdup(source); + audio->encoder = opus_encoder_create(48000, 2, OPUS_APPLICATION_RESTRICTED_LOWDELAY, &error); + audio->loop = pa_mainloop_new(); + if (!audio->source || !audio->encoder || !audio->loop) goto fail; + if (opus_encoder_ctl(audio->encoder, OPUS_SET_BITRATE(128000)) != OPUS_OK) goto fail; + audio->context = pa_context_new(pa_mainloop_get_api(audio->loop), "UU Remote"); + if (!audio->context || pa_context_connect(audio->context, NULL, PA_CONTEXT_NOAUTOSPAWN, NULL) < 0) goto fail; + return audio; +fail: + uu_audio_close(audio); + return NULL; +} + +/* Nonblocking: the Rust worker checks cancellation between calls, including on a suspended sink. */ +int uu_audio_packet(UuAudio *audio, uint8_t *packet, int capacity) { + int result; + if (pa_mainloop_iterate(audio->loop, 0, &result) < 0) return -1; + pa_context_state_t state = pa_context_get_state(audio->context); + if (!PA_CONTEXT_IS_GOOD(state) || audio->failed) return -2; + if (state != PA_CONTEXT_READY) return 0; + if (!audio->connecting) { + audio->connecting = 1; + if (strcmp(audio->source, "@DEFAULT_MONITOR@") == 0) { + pa_operation *op = pa_context_get_server_info(audio->context, server_info, audio); + if (op) pa_operation_unref(op); + else return -3; + } else { + pa_operation *op = pa_context_get_source_info_by_name(audio->context, audio->source, source_info, audio); + if (op) pa_operation_unref(op); + else return -3; + } + } + if (!audio->stream) return 0; + pa_stream_state_t stream_state = pa_stream_get_state(audio->stream); + if (!PA_STREAM_IS_GOOD(stream_state)) return -4; + if (stream_state != PA_STREAM_READY) return 0; + const void *buffer; + size_t length; + if (pa_stream_peek(audio->stream, &buffer, &length) < 0) return -5; + if (!length) return 0; + if (audio->offset > length) return -6; + size_t copy = sizeof(audio->pcm) - audio->filled; + if (copy > length - audio->offset) copy = length - audio->offset; + if (buffer) memcpy((uint8_t *)audio->pcm + audio->filled, (const uint8_t *)buffer + audio->offset, copy); + else memset((uint8_t *)audio->pcm + audio->filled, 0, copy); + audio->filled += copy; + audio->offset += copy; + if (audio->offset == length) { + if (pa_stream_drop(audio->stream) < 0) return -7; + audio->offset = 0; + } + if (audio->filled < sizeof(audio->pcm)) return 0; + audio->filled = 0; + return opus_encode(audio->encoder, audio->pcm, 960, packet, capacity); +} diff --git a/c_engine/dmabuf_frame.h b/c_engine/dmabuf_frame.h new file mode 100644 index 0000000..cc399e7 --- /dev/null +++ b/c_engine/dmabuf_frame.h @@ -0,0 +1,10 @@ +#ifndef UU_DMABUF_FRAME_H +#define UU_DMABUF_FRAME_H +#include +/* Borrowed single-plane RGB buffer. The producer retains ownership until the lease ends. */ +typedef struct { + int32_t fd; + uint32_t fourcc, width, height, stride, offset, size; + uint64_t modifier; +} UuDmaFrame; +#endif diff --git a/c_engine/portal_engine.c b/c_engine/portal_engine.c index 71eaa4a..927c291 100644 --- a/c_engine/portal_engine.c +++ b/c_engine/portal_engine.c @@ -16,6 +16,7 @@ #include #include #include +#include #define DEST "org.freedesktop.portal.Desktop" #define PATH "/org/freedesktop/portal/desktop" @@ -71,6 +72,11 @@ struct uu_portal { size_t frame_length; uint64_t sequence; uint64_t revision; + uint32_t buffers_seen, last_buffer_type; + int dma_requested, dma_negotiated, dma_format, dma_bad, dma_blocked, dma_neutral, dma_seen; + uint64_t modifier; + gint64 dma_requested_at; + struct pw_buffer *dma_pending, *dma_current; }; #include "portal_clipboard.inc" @@ -223,11 +229,17 @@ static void stream_state(void *data, enum pw_stream_state old, enum pw_stream_st pthread_mutex_unlock(&p->lock); if (state == PW_STREAM_STATE_ERROR || state == PW_STREAM_STATE_UNCONNECTED) fail(data, 9); } +static void return_dma_buffer(struct uu_portal *p, struct pw_buffer **buffer); static void stream_format(void *data, uint32_t id, const struct spa_pod *param) { struct uu_portal *p = data; if (id != SPA_PARAM_Format) return; if (!param) { pthread_mutex_lock(&p->lock); + if (p->dma_format) { + return_dma_buffer(p, &p->dma_pending); + return_dma_buffer(p, &p->dma_current); + p->sequence = 0; + } p->format_valid = 0; update_interruption(p); pthread_mutex_unlock(&p->lock); @@ -239,6 +251,11 @@ static void stream_format(void *data, uint32_t id, const struct spa_pod *param) (info.format != SPA_VIDEO_FORMAT_BGRA && info.format != SPA_VIDEO_FORMAT_BGRx && info.format != SPA_VIDEO_FORMAT_RGBA && info.format != SPA_VIDEO_FORMAT_RGBx)) { fail(p, 8); return; } pthread_mutex_lock(&p->lock); + if (p->dma_format) { + return_dma_buffer(p, &p->dma_pending); + return_dma_buffer(p, &p->dma_current); + p->sequence = 0; + } if (p->width && (p->width != info.size.width || p->height != info.size.height)) { p->error = 8; pthread_cond_broadcast(&p->changed); pthread_mutex_unlock(&p->lock); return; } @@ -246,12 +263,70 @@ static void stream_format(void *data, uint32_t id, const struct spa_pod *param) if (!p->frame) p->frame = malloc(length); if (!p->frame) { p->error = 8; pthread_mutex_unlock(&p->lock); return; } p->width = info.size.width; p->height = info.size.height; p->format = info.format; + p->dma_format = spa_pod_find_prop(param, NULL, SPA_FORMAT_VIDEO_modifier) != NULL; + p->modifier = info.modifier; + p->dma_negotiated = p->dma_requested; + p->buffers_seen = 0; p->frame_length = length; p->format_valid = 1; p->display_changed_at = 0; update_interruption(p); pthread_cond_broadcast(&p->changed); pthread_mutex_unlock(&p->lock); + fprintf(stderr, "Portal capture format: format=%u size=%ux%u dma_buf=%d dma_requested=%d modifier=%llu\n", + info.format, info.size.width, info.size.height, p->dma_format, p->dma_requested, (unsigned long long)info.modifier); + /* PipeWire DMA-BUF sharing: modifier formats require DMA buffers exclusively. + * Keep a current and a pending frame; at least four buffers prevent producer starvation. */ + if (p->stream) { + uint8_t storage[256]; + struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(storage, sizeof(storage)); + const struct spa_pod *buffers = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, + SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(4, 4, 8), + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int(p->dma_format ? + (1 << SPA_DATA_DmaBuf) : ((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd)))); + pw_stream_update_params(p->stream, &buffers, 1); + } +} + +/* All pending/current buffer changes run under the PipeWire loop lock, then p->lock. */ +static void return_dma_buffer(struct uu_portal *p, struct pw_buffer **buffer) { + if (*buffer && p->stream) pw_stream_queue_buffer(p->stream, *buffer); + *buffer = NULL; +} +static int describe_dma_buffer(struct uu_portal *p, struct spa_buffer *spa, UuDmaFrame *frame) { + if (p->error || atomic_load(&p->closed)) return 10; + if (p->modifier != 0) return 19; + if (!p->format_valid || !p->width || !p->height || spa->n_datas != 1) return 14; + struct spa_data *d = &spa->datas[0]; + struct spa_meta_header *header = spa_buffer_find_meta_data(spa, SPA_META_Header, sizeof(*header)); + if (!d->chunk || (d->chunk->flags & SPA_CHUNK_FLAG_CORRUPTED) || + (header && (header->flags & SPA_META_HEADER_FLAG_CORRUPTED))) return 10; + if (d->chunk->flags & SPA_CHUNK_FLAG_EMPTY) return 15; + struct spa_meta_region *crop = spa_buffer_find_meta_data(spa, SPA_META_VideoCrop, sizeof(*crop)); + if (crop && spa_meta_region_is_valid(crop) && (crop->region.position.x || crop->region.position.y || + crop->region.size.width != p->width || crop->region.size.height != p->height)) return 14; + /* Explicit sync is not advertised; require implicit DMA-BUF fences understood by VAAPI. */ +#if PW_CHECK_VERSION(1, 2, 0) + if (spa_buffer_find_meta(spa, SPA_META_SyncTimeline)) return 18; +#endif + if (d->type != SPA_DATA_DmaBuf || d->fd < 0 || d->fd > INT_MAX || d->chunk->stride < (int64_t)p->width * 4) + return 14; + uint64_t offset = (uint64_t)d->mapoffset + d->chunk->offset; + uint64_t size = (uint64_t)d->mapoffset + d->maxsize; + uint64_t required = offset + (uint64_t)(p->height - 1) * d->chunk->stride + (uint64_t)p->width * 4; + if (required > size || size > UINT32_MAX) return 14; + if (required - offset > d->chunk->size) return 10; + uint32_t fourcc; + switch (p->format) { + case SPA_VIDEO_FORMAT_BGRx: fourcc=0x34325258; break; + case SPA_VIDEO_FORMAT_BGRA: fourcc=0x34325241; break; + case SPA_VIDEO_FORMAT_RGBx: fourcc=0x34324258; break; + case SPA_VIDEO_FORMAT_RGBA: fourcc=0x34324241; break; + default: return 14; + } + *frame = (UuDmaFrame){ .fd=d->fd, .fourcc=fourcc, .width=p->width, .height=p->height, + .stride=d->chunk->stride, .offset=offset, .size=size, .modifier=p->modifier }; + return 0; } /* Called with the frame mutex held. Invalid transient chunks are discarded; * allocation capacity is not evidence that valid pixels have been supplied. */ @@ -301,14 +376,131 @@ static void stream_process(void *data) { } if (!newest) return; pthread_mutex_lock(&p->lock); - copy_video_buffer(p, newest->buffer); + p->buffers_seen++; + p->last_buffer_type = newest->buffer->n_datas ? newest->buffer->datas[0].type : 0; + if (p->dma_format) { + UuDmaFrame frame; + int result = describe_dma_buffer(p, newest->buffer, &frame); + if (result == 0 && p->dma_requested && !p->dma_blocked) { + return_dma_buffer(p, &p->dma_pending); + p->dma_pending = newest; + newest = NULL; + p->dma_seen = 1; + p->sequence++; p->revision++; p->dma_neutral = 0; + } else if (result == 15) { + return_dma_buffer(p, &p->dma_pending); + return_dma_buffer(p, &p->dma_current); + p->dma_neutral = 1; p->sequence++; p->revision++; + } else if (result != 10) p->dma_bad = result; + } else copy_video_buffer(p, newest->buffer); pthread_cond_broadcast(&p->changed); pthread_mutex_unlock(&p->lock); - pw_stream_queue_buffer(p->stream, newest); + if (newest) pw_stream_queue_buffer(p->stream, newest); +} +static void stream_remove_buffer(void *data, struct pw_buffer *buffer) { + struct uu_portal *p = data; + pthread_mutex_lock(&p->lock); + if (p->dma_pending == buffer) p->dma_pending = NULL; + if (p->dma_current == buffer) p->dma_current = NULL; + pthread_mutex_unlock(&p->lock); } static const struct pw_stream_events stream_events = { - PW_VERSION_STREAM_EVENTS, .state_changed = stream_state, .param_changed = stream_format, .process = stream_process + PW_VERSION_STREAM_EVENTS, .state_changed = stream_state, .param_changed = stream_format, .process = stream_process, + .remove_buffer = stream_remove_buffer }; + +static unsigned capture_formats(struct uu_portal *p, struct spa_pod_builder *b, const struct spa_pod **params, int dma) { + unsigned count = 0; + /* Only advertise linear RGB: arbitrary tiling/compression cannot be inferred from an FD. + * PipeWire page_dma_buf: put the modifier format before the shared-memory fallback. */ + for (int with_modifier = dma; with_modifier >= 0; with_modifier--) { + struct spa_pod_frame frame; + spa_pod_builder_push_object(b, &frame, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat); + spa_pod_builder_add(b, + SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), + SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), + SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(4, SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBA, SPA_VIDEO_FORMAT_RGBx), + 0); + if (p->width && p->height) spa_pod_builder_add(b, + SPA_FORMAT_VIDEO_size, SPA_POD_Rectangle(&SPA_RECTANGLE(p->width,p->height)), 0); + else spa_pod_builder_add(b, SPA_FORMAT_VIDEO_size, + SPA_POD_CHOICE_RANGE_Rectangle(&SPA_RECTANGLE(1920,1080), &SPA_RECTANGLE(1,1), &SPA_RECTANGLE(8192,8192)), 0); + spa_pod_builder_add(b, + SPA_FORMAT_VIDEO_framerate, SPA_POD_CHOICE_RANGE_Fraction(&SPA_FRACTION(60,1), &SPA_FRACTION(0,1), &SPA_FRACTION(60,1)), 0); + if (with_modifier) { + spa_pod_builder_prop(b, SPA_FORMAT_VIDEO_modifier, SPA_POD_PROP_FLAG_MANDATORY); + spa_pod_builder_long(b, 0); /* DRM_FORMAT_MOD_LINEAR */ + } + params[count++] = spa_pod_builder_pop(b, &frame); + } + params[count++] = spa_pod_builder_add_object(b, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, + SPA_PARAM_BUFFERS_buffers, SPA_POD_CHOICE_RANGE_Int(4, 4, 8), + SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd) | + (dma ? (1 << SPA_DATA_DmaBuf) : 0))); + return count; +} +static int request_capture_formats(struct uu_portal *p, int dma) { + uint8_t storage[2048]; + struct spa_pod_builder b = SPA_POD_BUILDER_INIT(storage, sizeof(storage)); + const struct spa_pod *params[3]; + unsigned count = capture_formats(p, &b, params, dma); + return pw_stream_update_params(p->stream, params, count); +} + +int uu_portal_dmabuf_acquire(struct uu_portal *p, UuDmaFrame *frame, uint64_t *revision) { + if (!p || !p->pw_loop || !p->stream || !frame || !revision) return 14; + pw_thread_loop_lock(p->pw_loop); + pthread_mutex_lock(&p->lock); + int result = 10; + if (atomic_load(&p->closed) || p->error || p->dma_blocked) result = 14; + else if (p->dma_bad) result = p->dma_bad; + else if (!p->dma_requested) { + p->dma_requested = 1; p->dma_negotiated = 0; p->dma_seen = 0; + p->dma_requested_at = g_get_monotonic_time(); + pthread_mutex_unlock(&p->lock); + int status = request_capture_formats(p, 1); + pw_thread_loop_unlock(p->pw_loop); + return status < 0 ? 14 : 10; + } else if (p->dma_negotiated && !p->dma_format) result = 16; + else if (!p->dma_seen && !p->dma_neutral && + g_get_monotonic_time() - p->dma_requested_at > 2 * G_TIME_SPAN_SECOND) result = 17; + else if (p->dma_neutral) result = 15; + else if (p->dma_format && p->format_valid) { + if (p->dma_pending) { + return_dma_buffer(p, &p->dma_current); + p->dma_current = p->dma_pending; p->dma_pending = NULL; + } + if (p->dma_current) { + result = describe_dma_buffer(p, p->dma_current->buffer, frame); + if (!result) { + result = *revision == p->revision ? 12 : 0; + *revision = p->revision; + } + } + } + pthread_mutex_unlock(&p->lock); + /* A lease pins both the pw_buffer and its format across VAAPI's completion fence. + * No producer buffer may be returned, removed or renegotiated while the GPU reads it. */ + if (result != 0 && result != 12) pw_thread_loop_unlock(p->pw_loop); + return result; +} +void uu_portal_dmabuf_release(struct uu_portal *p) { + pw_thread_loop_unlock(p->pw_loop); +} +void uu_portal_dmabuf_disable(struct uu_portal *p) { + if (!p || !p->pw_loop || !p->stream) return; + pw_thread_loop_lock(p->pw_loop); + pthread_mutex_lock(&p->lock); + int requested = p->dma_requested; + if (requested) { + p->dma_requested = 0; p->dma_blocked = 1; p->dma_neutral = 0; + return_dma_buffer(p, &p->dma_pending); return_dma_buffer(p, &p->dma_current); + if (p->dma_format) { p->sequence = 0; p->revision++; } + } + pthread_mutex_unlock(&p->lock); + if (requested) request_capture_formats(p, 0); + pw_thread_loop_unlock(p->pw_loop); +} static int connect_pipewire(struct uu_portal *p) { if (open_cancelled(p)) return 13; GUnixFDList *fds = NULL; @@ -337,27 +529,33 @@ static int connect_pipewire(struct uu_portal *p) { PW_KEY_MEDIA_TYPE, "Video", PW_KEY_MEDIA_CATEGORY, "Capture", PW_KEY_MEDIA_ROLE, "Screen", NULL)); if (!p->stream) return 7; pw_stream_add_listener(p->stream, &p->stream_listener, &stream_events, p); - uint8_t storage[1024]; struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(storage, sizeof(storage)); - const struct spa_pod *params[2]; - params[0] = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_Format, SPA_PARAM_EnumFormat, - SPA_FORMAT_mediaType, SPA_POD_Id(SPA_MEDIA_TYPE_video), SPA_FORMAT_mediaSubtype, SPA_POD_Id(SPA_MEDIA_SUBTYPE_raw), - SPA_FORMAT_VIDEO_format, SPA_POD_CHOICE_ENUM_Id(4, SPA_VIDEO_FORMAT_BGRA, SPA_VIDEO_FORMAT_BGRx, SPA_VIDEO_FORMAT_RGBA, SPA_VIDEO_FORMAT_RGBx), - SPA_FORMAT_VIDEO_size, SPA_POD_CHOICE_RANGE_Rectangle(&SPA_RECTANGLE(1920,1080), &SPA_RECTANGLE(1,1), &SPA_RECTANGLE(8192,8192)), - SPA_FORMAT_VIDEO_framerate, SPA_POD_CHOICE_RANGE_Fraction(&SPA_FRACTION(60,1), &SPA_FRACTION(0,1), &SPA_FRACTION(60,1))); - params[1] = spa_pod_builder_add_object(&builder, SPA_TYPE_OBJECT_ParamBuffers, SPA_PARAM_Buffers, - SPA_PARAM_BUFFERS_dataType, SPA_POD_CHOICE_FLAGS_Int((1 << SPA_DATA_MemPtr) | (1 << SPA_DATA_MemFd))); + uint8_t storage[2048]; struct spa_pod_builder builder = SPA_POD_BUILDER_INIT(storage, sizeof(storage)); + const struct spa_pod *params[3]; + unsigned param_count = capture_formats(p, &builder, params, p->dma_requested); + p->dma_requested_at = g_get_monotonic_time(); /* Node ID belongs to this portal's restricted remote. Do not put it into * target.object, whose numeric value means object.serial on modern PW. */ if (pw_stream_connect(p->stream, PW_DIRECTION_INPUT, p->node, - PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS, params, 2) < 0) return 7; + PW_STREAM_FLAG_AUTOCONNECT | PW_STREAM_FLAG_MAP_BUFFERS, params, param_count) < 0) return 7; if (pw_thread_loop_start(p->pw_loop) < 0) return 7; - return wait_for_first_frame(p); + int result = wait_for_first_frame(p); + pthread_mutex_lock(&p->lock); + int fallback = (result == 10 || result == 14) && p->dma_requested && !p->error && + !atomic_load(&p->closed) && !open_cancelled(p); + int dma_status = p->dma_bad; + pthread_mutex_unlock(&p->lock); + if (fallback) { + fprintf(stderr, "Portal initial DMA-BUF unavailable: status=%d; falling back to shared-memory capture\n", dma_status); + uu_portal_dmabuf_disable(p); + result = wait_for_first_frame(p); + } + return result; } static int wait_for_first_frame(struct uu_portal *p) { struct timespec until; gint64 deadline = g_get_monotonic_time() + 10 * G_TIME_SPAN_SECOND; pthread_mutex_lock(&p->lock); - while (!p->sequence && !p->error && !open_cancelled(p)) { + while (!p->sequence && !p->error && !open_cancelled(p) && !(p->dma_requested && p->dma_bad)) { gint64 remaining = deadline - g_get_monotonic_time(); if (remaining <= 0) break; clock_gettime(CLOCK_REALTIME, &until); @@ -366,7 +564,10 @@ static int wait_for_first_frame(struct uu_portal *p) { if (until.tv_nsec >= 1000000000) { until.tv_sec++; until.tv_nsec -= 1000000000; } pthread_cond_timedwait(&p->changed, &p->lock, &until); } - int result = p->error ? p->error : open_cancelled(p) ? 13 : p->sequence ? 0 : 10; + int result = p->error ? p->error : open_cancelled(p) ? 13 : p->sequence ? 0 : + p->dma_requested && p->dma_bad ? 14 : 10; + if (result == 10) fprintf(stderr, "Portal first-frame timeout: format_valid=%d dma_buf=%d dma_requested=%d buffers=%u last_type=%u dma_status=%d\n", + p->format_valid, p->dma_format, p->dma_requested, p->buffers_seen, p->last_buffer_type, p->dma_bad); pthread_mutex_unlock(&p->lock); return result; } @@ -395,12 +596,13 @@ static int read_stream_result(struct uu_portal *p, GVariant *results) { } static struct uu_portal *open_persist(const char *restore, void (*save_token)(const char *, void *), void *save_data, uu_portal_cancelled_fn cancelled, void *cancel_data, - int *error, uint32_t *width, uint32_t *height) { + int prefer_dmabuf, int *error, uint32_t *width, uint32_t *height) { *error = 0; *width = *height = 0; struct uu_portal *p = calloc(1, sizeof(*p)); if (!p) { *error = 8; return NULL; } p->cancelled = cancelled; p->cancel_data = cancel_data; + p->dma_requested = !!prefer_dmabuf; /* GNOME forwards NotifyPointerMotionAbsolute into Mutter's stream-to-stage * transform. It expects stream pixels, unlike the compositor-sized KDE path. * Do not apply the compositor scale twice on fractional-scale displays. */ @@ -505,11 +707,17 @@ struct uu_portal *uu_portal_open_persist_cancellable(const char *restore, void (*save_token)(const char *, void *), void *save_data, uu_portal_cancelled_fn cancelled, void *cancel_data, int *error, uint32_t *width, uint32_t *height) { - return open_persist(restore, save_token, save_data, cancelled, cancel_data, error, width, height); + return open_persist(restore, save_token, save_data, cancelled, cancel_data, 0, error, width, height); +} +struct uu_portal *uu_portal_open_persist_capture(const char *restore, + void (*save_token)(const char *, void *), void *save_data, + uu_portal_cancelled_fn cancelled, void *cancel_data, int prefer_dmabuf, + int *error, uint32_t *width, uint32_t *height) { + return open_persist(restore, save_token, save_data, cancelled, cancel_data, prefer_dmabuf, error, width, height); } struct uu_portal *uu_portal_open_persist(const char *restore, void (*save_token)(const char *, void *), void *data, int *error, uint32_t *width, uint32_t *height) { - return open_persist(restore, save_token, data, NULL, NULL, error, width, height); + return open_persist(restore, save_token, data, NULL, NULL, 0, error, width, height); } struct uu_portal *uu_portal_open_restore(const char *restore, int *error, uint32_t *width, uint32_t *height) { return uu_portal_open_persist(restore, NULL, NULL, error, width, height); @@ -556,11 +764,15 @@ void uu_portal_close(struct uu_portal *p) { } void uu_portal_reset_frame(struct uu_portal *p) { if (!p) return; + if (p->pw_loop) pw_thread_loop_lock(p->pw_loop); pthread_mutex_lock(&p->lock); + return_dma_buffer(p, &p->dma_pending); return_dma_buffer(p, &p->dma_current); + p->dma_neutral = 0; p->sequence = 0; p->revision++; if (p->frame) memset(p->frame, 0, p->frame_length); pthread_mutex_unlock(&p->lock); + if (p->pw_loop) pw_thread_loop_unlock(p->pw_loop); } int uu_portal_capture(struct uu_portal *p, uint8_t *frame, size_t length) { return uu_portal_capture_since(p, frame, length, NULL); @@ -577,6 +789,7 @@ static int stream_stalled(struct uu_portal *p) { } int uu_portal_capture_since(struct uu_portal *p, uint8_t *frame, size_t length, uint64_t *revision) { if (!p || !frame) return 11; + uu_portal_dmabuf_disable(p); pthread_mutex_lock(&p->lock); int result = atomic_load(&p->closed) ? 9 : p->error ? p->error : stream_stalled(p) ? 9 : !p->sequence ? 10 : length != p->frame_length ? 11 : 0; diff --git a/c_engine/portal_engine.h b/c_engine/portal_engine.h index f0b1b25..d0aa951 100644 --- a/c_engine/portal_engine.h +++ b/c_engine/portal_engine.h @@ -2,6 +2,7 @@ #define UU_PORTAL_ENGINE_H #include #include +#include "dmabuf_frame.h" struct uu_portal; typedef int (*uu_portal_cancelled_fn)(void *data); /* One object owns one consent/session and one PipeWire stream. */ @@ -10,6 +11,8 @@ struct uu_portal *uu_portal_open(int *error, uint32_t *width, uint32_t *height); struct uu_portal *uu_portal_open_persist(const char *restore, void (*save_token)(const char *, void *), void *data, int *error, uint32_t *width, uint32_t *height); /* The cancellation callback and data are used only until this call returns. */ struct uu_portal *uu_portal_open_persist_cancellable(const char *restore, void (*save_token)(const char *, void *), void *save_data, uu_portal_cancelled_fn cancelled, void *cancel_data, int *error, uint32_t *width, uint32_t *height); +/* Select the capture transport before connecting, including on same-process recovery. */ +struct uu_portal *uu_portal_open_persist_capture(const char *restore, void (*save_token)(const char *, void *), void *save_data, uu_portal_cancelled_fn cancelled, void *cancel_data, int prefer_dmabuf, int *error, uint32_t *width, uint32_t *height); struct uu_portal *uu_portal_open_restore(const char *restore, int *error, uint32_t *width, uint32_t *height); const char *uu_portal_restore_token(struct uu_portal *portal); void uu_portal_close(struct uu_portal *portal); @@ -23,6 +26,12 @@ void uu_portal_logical_size(struct uu_portal *portal, uint32_t *width, uint32_t void uu_portal_reset_frame(struct uu_portal *portal); int uu_portal_capture(struct uu_portal *portal, uint8_t *bgra, size_t length); int uu_portal_capture_since(struct uu_portal *portal, uint8_t *bgra, size_t length, uint64_t *revision); +/* acquire holds the PipeWire loop until release, including GPU completion. + * 0/12 = leased new/unchanged frame, 10 = pending, 15 = neutral black. + * CPU fallback: 14 layout/disabled, 16 shared memory, 17 timeout, 18 explicit sync, 19 modifier. */ +int uu_portal_dmabuf_acquire(struct uu_portal *portal, UuDmaFrame *frame, uint64_t *revision); +void uu_portal_dmabuf_release(struct uu_portal *portal); +void uu_portal_dmabuf_disable(struct uu_portal *portal); int uu_portal_move(struct uu_portal *portal, double x, double y, int relative); int uu_portal_button(struct uu_portal *portal, int button, int pressed); int uu_portal_scroll(struct uu_portal *portal, double dx, double dy, int finish); diff --git a/c_engine/uinput_keymap.c b/c_engine/uinput_keymap.c new file mode 100644 index 0000000..1853656 --- /dev/null +++ b/c_engine/uinput_keymap.c @@ -0,0 +1,58 @@ +#include "uinput_keymap.h" +#include +#include +#include +#include + +static size_t text_plan(XkbDescPtr map, const XkbStateRec *state, + const uint8_t *text, size_t length, UuTextKey *plan, size_t capacity) { + if (!map || !map->map || !map->map->modmap || !state || state->base_mods || state->latched_mods) return 0; + /* Prove the XKB -> evdev offset from physical key names rather than assuming a US layout. */ + if (!map->names || !map->names->keys || map->min_key_code > 10 || map->max_key_code < 50 || + memcmp(map->names->keys[10].name,"AE01",4) || + memcmp(map->names->keys[38].name,"AC01",4) || + memcmp(map->names->keys[50].name,"LFSH",4)) return 0; + uint16_t shift_code=0; + for (unsigned code=9; code<=map->max_key_code; code++) { + if (code>=map->min_key_code && code-8<=248 && (map->map->modmap[code] & ShiftMask)) { + shift_code=code-8; break; + } + } + UuTextKey lookup[128]={0}; + for (unsigned shift=0; shift<2; shift++) { + if (shift && !shift_code) continue; + for (unsigned code=9; code<=map->max_key_code; code++) { + if (codemin_key_code || code-8>248) continue; + KeySym symbol=NoSymbol; + unsigned consumed=0; + unsigned modifiers=XkbBuildCoreState(state->locked_mods | (shift ? ShiftMask : 0),state->group); + if (!XkbTranslateKeyCode(map,code,modifiers,&consumed,&symbol)) continue; + UuTextKey key={.code=code-8,.shift=shift ? shift_code : 0}; + if (symbol>=0x20 && symbol<=0x7e && !lookup[symbol].code) lookup[symbol]=key; + else if (symbol==XK_Return && !lookup['\n'].code) lookup['\n']=lookup['\r']=key; + else if (symbol==XK_Tab && !lookup['\t'].code) lookup['\t']=key; + } + } + size_t count=0; + while (count=128) return 0; + XInitThreads(); + Display *display=XOpenDisplay(NULL); + if (!display) return 0; + XkbDescPtr map=XkbGetMap(display,XkbAllClientInfoMask,XkbUseCoreKbd); + XkbStateRec state; + size_t count=0; + if (map && XkbGetState(display,XkbUseCoreKbd,&state)==Success && + XkbGetNames(display,XkbKeyNamesMask,map)==Success) + count=text_plan(map,&state,text,length,plan,capacity); + if (map) XkbFreeKeyboard(map,XkbAllComponentsMask,True); + XCloseDisplay(display); + return count; +} diff --git a/c_engine/uinput_keymap.h b/c_engine/uinput_keymap.h new file mode 100644 index 0000000..e5e6e0b --- /dev/null +++ b/c_engine/uinput_keymap.h @@ -0,0 +1,9 @@ +#ifndef UU_UINPUT_KEYMAP_H +#define UU_UINPUT_KEYMAP_H +#include +#include +typedef struct { uint16_t code, shift; } UuTextKey; +/* Read-only XKB lookup; emits no input and never changes the keymap or clipboard. + * Return the number of initial ASCII characters representable by physical keys. */ +size_t uu_uinput_text_plan(const uint8_t *text, size_t length, UuTextKey *plan, size_t capacity); +#endif diff --git a/c_engine/vaapi_engine.c b/c_engine/vaapi_engine.c index 16e228f..5ec532b 100644 --- a/c_engine/vaapi_engine.c +++ b/c_engine/vaapi_engine.c @@ -268,33 +268,57 @@ VaapiEngine* vaapi_create(const VaapiConfig* config) { } int vaapi_import_dma_buffer(VaapiEngine* engine, int dma_fd, uint32_t stride) { - if (!engine || dma_fd < 0) return -1; + if (!engine) return -1; + UuDmaFrame frame = { + .fd=dma_fd, .fourcc=0x34325258, .width=engine->cap_width, .height=engine->cap_height, + .stride=stride, .size=stride * engine->cap_height, .modifier=0 + }; + return vaapi_import_frame(engine, &frame); +} - if (engine->in_rgb_surf != VA_INVALID_ID) { - RT.vaDestroySurfaces(engine->va_dpy, &engine->in_rgb_surf, 1); - engine->in_rgb_surf = VA_INVALID_ID; - } +static void vaapi_clear_input(VaapiEngine *engine) { if (engine->vpp_buf != VA_INVALID_ID) { RT.vaDestroyBuffer(engine->va_dpy, engine->vpp_buf); engine->vpp_buf = VA_INVALID_ID; } + if (engine->in_rgb_surf != VA_INVALID_ID) { + RT.vaDestroySurfaces(engine->va_dpy, &engine->in_rgb_surf, 1); + engine->in_rgb_surf = VA_INVALID_ID; + } + engine->dma_fd = -1; +} - engine->dma_fd = dma_fd; - engine->stride = stride; +int vaapi_import_frame(VaapiEngine *engine, const UuDmaFrame *frame) { + if (!engine || !frame || frame->fd < 0 || frame->width != engine->cap_width || + frame->height != engine->cap_height || !frame->height || frame->stride < (uint64_t)frame->width * 4 || + (uint64_t)frame->offset + (uint64_t)(frame->height - 1) * frame->stride + (uint64_t)frame->width * 4 > frame->size) + return -1; + uint32_t va_fourcc; + switch (frame->fourcc) { + case 0x34325258: va_fourcc=VA_FOURCC_BGRX; break; /* DRM_FORMAT_XRGB8888 */ + case 0x34325241: va_fourcc=VA_FOURCC_BGRA; break; /* DRM_FORMAT_ARGB8888 */ + case 0x34324258: va_fourcc=VA_FOURCC_RGBX; break; /* DRM_FORMAT_XBGR8888 */ + case 0x34324241: va_fourcc=VA_FOURCC_RGBA; break; /* DRM_FORMAT_ABGR8888 */ + default: return -1; + } + vaapi_clear_input(engine); + engine->dma_fd = frame->fd; + engine->stride = frame->stride; VADRMPRIMESurfaceDescriptor prime_desc; memset(&prime_desc, 0, sizeof(prime_desc)); - prime_desc.fourcc = VA_FOURCC_BGRX; + prime_desc.fourcc = va_fourcc; prime_desc.width = engine->cap_width; prime_desc.height = engine->cap_height; prime_desc.num_objects = 1; - prime_desc.objects[0].fd = dma_fd; - prime_desc.objects[0].size = stride * engine->cap_height; - prime_desc.objects[0].drm_format_modifier = 0; + prime_desc.objects[0].fd = frame->fd; + prime_desc.objects[0].size = frame->size; + prime_desc.objects[0].drm_format_modifier = frame->modifier; prime_desc.num_layers = 1; - prime_desc.layers[0].drm_format = 0x34325258; // DRM_FORMAT_XRGB8888 + prime_desc.layers[0].drm_format = frame->fourcc; prime_desc.layers[0].num_planes = 1; - prime_desc.layers[0].pitch[0] = stride; + prime_desc.layers[0].pitch[0] = frame->stride; + prime_desc.layers[0].offset[0] = frame->offset; VASurfaceAttrib attribs[2]; attribs[0].type = VASurfaceAttribMemoryType; @@ -322,6 +346,8 @@ int vaapi_import_dma_buffer(VaapiEngine* engine, int dma_fd, uint32_t stride) { int vaapi_upload_host_buffer(VaapiEngine* engine, const uint8_t* host_ptr, uint32_t pitch) { if (!engine || !host_ptr) return -1; + /* Never write a fallback image into the compositor's imported buffer. */ + if (engine->dma_fd >= 0) vaapi_clear_input(engine); if (engine->in_rgb_surf == VA_INVALID_ID) { VAStatus st = RT.vaCreateSurfaces(engine->va_dpy, VA_RT_FORMAT_RGB32, engine->cap_width, engine->cap_height, &engine->in_rgb_surf, 1, NULL, 0); diff --git a/c_engine/vaapi_engine.h b/c_engine/vaapi_engine.h index 567b9fc..92c7040 100644 --- a/c_engine/vaapi_engine.h +++ b/c_engine/vaapi_engine.h @@ -3,6 +3,7 @@ #include #include +#include "dmabuf_frame.h" #ifdef __cplusplus extern "C" { @@ -28,6 +29,7 @@ VaapiEngine* vaapi_create(const VaapiConfig* config); // Import a DRI3 / GBM DMA-BUF fd as the zero-copy input surface int vaapi_import_dma_buffer(VaapiEngine* engine, int dma_fd, uint32_t stride); +int vaapi_import_frame(VaapiEngine* engine, const UuDmaFrame *frame); // Host fallback: upload ARGB host memory if DMA-BUF is unavailable int vaapi_upload_host_buffer(VaapiEngine* engine, const uint8_t* host_ptr, uint32_t pitch); diff --git a/src/audio.rs b/src/audio.rs index ccb4a86..01a09bf 100644 --- a/src/audio.rs +++ b/src/audio.rs @@ -1,6 +1,6 @@ //! Desktop sound for the viewer's 声音 menu: the default output's monitor (PulseAudio or //! PipeWire-pulse) is encoded to Opus and sent on the negotiated audio m-line. -//! An FFmpeg child process does the encoding. +//! inproc-av uses libpulse/libopus directly; other builds and native failures use FFmpeg. //! PipeWire monitors ignore the sink's mute by default, so 被控端静音运行 keeps the stream audible. //! `UU_AUDIO=0` disables it; `UU_AUDIO_SOURCE` picks another Pulse source. @@ -56,12 +56,78 @@ fn read_packets(stdout: impl Read, tx: &tokio::sync::mpsc::Sender) fn start(source: &str, tx: tokio::sync::mpsc::Sender) -> Result<(Stopper, tokio::task::JoinHandle>), String> { let stop = Arc::new(AtomicBool::new(false)); + #[cfg(all(target_os = "linux", feature = "inproc-av"))] + if !crate::inproc_encoder::disabled() { + let source = std::ffi::CString::new(source).map_err(|_| "Audio source contains NUL")?; + let worker_stop = stop.clone(); + let job = tokio::task::spawn_blocking(move || { + match native_packets(&source, &tx, &worker_stop) { + Ok(()) => Ok(()), + Err(error) if !worker_stop.load(Ordering::Relaxed) && !tx.is_closed() => { + log::warn!("In-process audio unavailable: {error}; falling back to FFmpeg Opus"); + let mut child = spawn_encoder(source.to_str().map_err(|_| "Invalid audio source")?)?; + let stdout = child.stdout.take().ok_or("Encoder stdout missing")?; + // Supervise the process separately so cancellation also interrupts a blocked read. + let (done_tx, done_rx) = std::sync::mpsc::channel(); + let reader_tx = tx.clone(); + let reader = std::thread::spawn(move || { + let result = read_packets(stdout, &reader_tx); + let _ = done_tx.send(result); + }); + let result = loop { + match done_rx.recv_timeout(Duration::from_millis(20)) { + Ok(result) => break result, + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break Err("Audio reader ended".into()), + Err(_) => if worker_stop.load(Ordering::Relaxed) || tx.is_closed() { break Ok(()); }, + } + }; + let _ = child.kill(); + let _ = child.wait(); + let _ = reader.join(); + result + }, + Err(_) => Ok(()), + } + }); + return Ok((Stopper { stop, child: None }, job)); + } let mut child = spawn_encoder(source)?; let stdout = child.stdout.take().ok_or("Encoder stdout missing")?; let job = tokio::task::spawn_blocking(move || read_packets(stdout, &tx)); Ok((Stopper { stop, child: Some(child) }, job)) } +#[cfg(all(target_os = "linux", feature = "inproc-av"))] +fn native_packets(source: &std::ffi::CStr, tx: &tokio::sync::mpsc::Sender, stop: &AtomicBool) -> Result<(), String> { + extern "C" { + fn uu_audio_open(source: *const libc::c_char) -> *mut libc::c_void; + fn uu_audio_packet(audio: *mut libc::c_void, packet: *mut u8, capacity: libc::c_int) -> libc::c_int; + fn uu_audio_close(audio: *mut libc::c_void); + } + struct Capture(*mut libc::c_void); + impl Drop for Capture { + fn drop(&mut self) { unsafe { uu_audio_close(self.0) }; } + } + let capture = Capture(unsafe { uu_audio_open(source.as_ptr()) }); + if capture.0.is_null() { return Err("Native audio initialization failed".into()); } + let mut packet = [0; 4000]; + let mut last_packet = std::time::Instant::now(); + let mut announced = false; + while !stop.load(Ordering::Relaxed) && !tx.is_closed() { + let size = unsafe { uu_audio_packet(capture.0, packet.as_mut_ptr(), packet.len() as libc::c_int) }; + if size < 0 { return Err(format!("Native audio capture/encode failed: {size}")); } + if size == 0 { + if last_packet.elapsed() >= Duration::from_secs(5) { return Err("Native audio capture timed out".into()); } + std::thread::sleep(Duration::from_millis(2)); + continue; + } + if !announced { log::info!("Desktop audio pipeline=in-process opus 48k stereo"); announced = true; } + last_packet = std::time::Instant::now(); + if tx.blocking_send(bytes::Bytes::copy_from_slice(&packet[..size as usize])).is_err() { break; } + } + Ok(()) +} + fn closed(pc: &RTCPeerConnection) -> bool { matches!(pc.connection_state(), RTCPeerConnectionState::Closed | RTCPeerConnectionState::Failed) } @@ -140,3 +206,36 @@ pub async fn probe(seconds: u64) -> Result<(), String> { let result = job.await.map_err(|e| e.to_string())?; if packets == 0 { result } else { Ok(()) } } + +#[cfg(all(test, target_os = "linux", feature = "inproc-av"))] +mod native_tests { + use super::*; + + #[tokio::test] + #[ignore = "requires a running PulseAudio or PipeWire-pulse server"] + async fn native_opus_packets_decode_and_capture_stops_promptly() { + extern "C" { + fn opus_decoder_create(rate: i32, channels: i32, error: *mut i32) -> *mut libc::c_void; + fn opus_decode(decoder: *mut libc::c_void, data: *const u8, length: i32, pcm: *mut i16, size: i32, fec: i32) -> i32; + fn opus_decoder_destroy(decoder: *mut libc::c_void); + } + let source = std::ffi::CString::new("@DEFAULT_MONITOR@").unwrap(); + let (tx, mut rx) = tokio::sync::mpsc::channel(2); + let stop = Arc::new(AtomicBool::new(false)); + let worker_stop = stop.clone(); + let job = tokio::task::spawn_blocking(move || native_packets(&source, &tx, &worker_stop)); + let stopper = Stopper { stop, child: None }; + let packet = tokio::time::timeout(Duration::from_secs(6), rx.recv()).await.unwrap().unwrap(); + let mut error = 0; + let decoder = unsafe { opus_decoder_create(48000, 2, &mut error) }; + assert_eq!(error, 0); + assert!(!decoder.is_null()); + let mut pcm = [0; 960 * 2]; + let samples = unsafe { opus_decode(decoder, packet.as_ptr(), packet.len() as i32, pcm.as_mut_ptr(), 960, 0) }; + unsafe { opus_decoder_destroy(decoder) }; + assert_eq!(samples, 960); + drop(rx); + drop(stopper); + tokio::time::timeout(Duration::from_secs(1), job).await.unwrap().unwrap().unwrap(); + } +} diff --git a/src/desktop_control_tests.rs b/src/desktop_control_tests.rs index f32a883..c42acd5 100644 --- a/src/desktop_control_tests.rs +++ b/src/desktop_control_tests.rs @@ -79,6 +79,53 @@ fn mapping(kind: u8, payload: &[u8]) -> Vec { crate::initialization::envelope(1, 0, 27, &frame) } +#[tokio::test] +async fn ready_portal_control_input_is_forwarded_while_locked() { + tokio::time::timeout(Duration::from_secs(20), async { + let input = Arc::new(InputInjector::without_desktop()); + let mut screen = ScreenCapture::unavailable("synthetic ready desktop".into()); + screen.backend = crate::desktop_backend::DesktopBackend::Portal; + screen.init_error = None; + let (lock, locked) = tokio::sync::watch::channel(Some(true)); + let portal = Arc::new(crate::desktop_session::PortalDesktop::with_lock_state(Arc::new(screen), locked)); + let (host, mut viewer) = connect(input.clone(), true, Some(portal.clone()), &["CONTROL_DATA_CHANNEL"]).await; + receive(&mut viewer.messages, "CONTROL_DATA_CHANNEL", 13).await; + for (index, state) in [Some(true), None].into_iter().enumerate() { + lock.send(state).unwrap(); + let before = input.portal_input_counts_for_test(); + for message in [ + br#"{"action":"kbd_press","key":65}"#.as_slice(), + br#"{"action":"kbd_release","key":65}"#.as_slice(), + br#"{"action":"mouse_press","button":1}"#.as_slice(), + br#"{"action":"mouse_release","button":1}"#.as_slice(), + ] { + send(&viewer, "CONTROL_DATA_CHANNEL", message.to_vec()).await; + } + let sequence = 310 + index as u64; + send(&viewer, "CONTROL_DATA_CHANNEL", crate::initialization::envelope(sequence, 0, 3, &[8, 0])).await; + loop { + let response = receive(&mut viewer.messages, "CONTROL_DATA_CHANNEL", 3).await; + let args: serde_json::Value = serde_json::from_slice(body(body(&response, 3).unwrap(), 2).unwrap()).unwrap(); + if args["seq"] == sequence { break; } + } + let after = input.portal_input_counts_for_test(); + assert_eq!(after[2] - before[2], 2, "keyboard dropped for lock state {state:?}"); + assert_eq!(after[1] - before[1], 2, "mouse buttons dropped for lock state {state:?}"); + } + let before = input.portal_input_counts_for_test(); + let mut unavailable = ScreenCapture::unavailable("synthetic closed source".into()); + unavailable.backend = crate::desktop_backend::DesktopBackend::Portal; + portal.publish_test_screen(Arc::new(unavailable)); + send(&viewer, "CONTROL_DATA_CHANNEL", br#"{"action":"kbd_click","key":65}"#.to_vec()).await; + send(&viewer, "CONTROL_DATA_CHANNEL", crate::initialization::envelope(320, 0, 3, &[8, 0])).await; + receive(&mut viewer.messages, "CONTROL_DATA_CHANNEL", 3).await; + assert_eq!(input.portal_input_counts_for_test(), before, "unavailable source accepted input"); + host.pc.close().await.unwrap(); + viewer.pc.close().await.unwrap(); + viewer.router.lock().await.stop().await.unwrap(); + }).await.expect("locked CONTROL input regression exceeded deadline"); +} + #[tokio::test] async fn closing_unnegotiated_peers_releases_callback_state() { for desktop in [false, true] { diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index a4a6d77..a2b5370 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -2,6 +2,7 @@ //! With UU_VIDEO_ENCODER=vaapi, H.264 / HEVC go through VAAPI instead: a DRI3 DMA-BUF of the root window is //! imported as a VA surface and converted/scaled by VPP, so the CPU never touches the pixels (XShm upload //! when DRI3 is unavailable). +//! Portal sessions can instead upload their existing BGRA frames, sharing the same GPU VPP and encoder. //! When capture and encoder sizes match, the XShm buffer is registered as CUDA pinned memory and //! NVENC reads it by DMA, so there is no host-to-device copy (the X server still writes XShm). @@ -77,6 +78,7 @@ extern "C" { fn vaapi_create(config: *const VaapiConfigFFI) -> *mut VaapiEngineOpaque; fn vaapi_import_dma_buffer(engine: *mut VaapiEngineOpaque, dma_fd: libc::c_int, stride: u32) -> libc::c_int; + fn vaapi_import_frame(engine: *mut VaapiEngineOpaque, frame: *const crate::portal::DmaFrame) -> libc::c_int; fn vaapi_upload_host_buffer(engine: *mut VaapiEngineOpaque, host_ptr: *const u8, pitch: u32) -> libc::c_int; fn vaapi_encode_frame( engine: *mut VaapiEngineOpaque, @@ -130,6 +132,7 @@ impl Drop for SafeDri3Capture { } pub struct Config { + pub portal: Option>, pub display: String, pub input: (u32, u32), pub output: Option<(u32, u32)>, @@ -218,6 +221,28 @@ pub fn run( } } +pub fn black_frame(codec: crate::capture_setting::VideoCodec) -> Result { + let device = CString::new(crate::video::vaapi_device()).map_err(|_| "Invalid VAAPI device")?; + let config = VaapiConfigFFI { + width: 1280, height: 720, cap_width: 1280, cap_height: 720, fps: 1, gop: 1, + target_bps: 1_000_000, max_bps: 2_000_000, buffer_bits: 500_000, + is_hevc: libc::c_int::from(codec == crate::capture_setting::VideoCodec::H265), + device: device.as_ptr(), + }; + let engine = SafeVaapiEngine(unsafe { vaapi_create(&config) }); + if engine.0.is_null() { return Err("Locked video VAAPI initialization failed".into()); } + let pixels = vec![0; 1280 * 720 * 4]; + if unsafe { vaapi_upload_host_buffer(engine.0, pixels.as_ptr(), 1280 * 4) } != 0 { + return Err("Locked video VAAPI upload failed".into()); + } + let mut data = std::ptr::null(); + let mut length = 0; + if unsafe { vaapi_encode_frame(engine.0, 1, &mut data, &mut length) } != 0 || data.is_null() || length == 0 { + return Err("Locked video VAAPI encode failed".into()); + } + Ok(bytes::Bytes::copy_from_slice(unsafe { std::slice::from_raw_parts(data, length as usize) })) +} + fn run_vaapi( config: Config, units: mpsc::Sender, @@ -257,6 +282,9 @@ fn run_vaapi( return Err("Failed to create native VAAPI engine".into()); } let engine = SafeVaapiEngine(raw_engine); + if let Some(portal) = config.portal.as_ref() { + return run_portal_vaapi(&config, portal, &engine, units, stop, idr, budget); + } // Capture stays at the full input size; VPP scales (or pads) to the encoder size. let display = CString::new(config.display.as_str()).map_err(|_| "DISPLAY contains NUL".to_string())?; @@ -392,6 +420,115 @@ fn run_vaapi( keepalive = random_keepalive(&mut seed); } } +fn vaapi_packet(engine: &SafeVaapiEngine, idr: &AtomicBool) -> Result, String> { + let mut data = std::ptr::null(); + let mut length = 0; + let force_idr = libc::c_int::from(idr.swap(false, Ordering::AcqRel)); + let result = unsafe { vaapi_encode_frame(engine.0, force_idr, &mut data, &mut length) }; + if result != 0 { return Err(format!("VAAPI encode failed: {result}")); } + if data.is_null() || length == 0 { return Ok(Vec::new()); } + Ok(unsafe { std::slice::from_raw_parts(data, length as usize) }.to_vec()) +} + +fn run_portal_vaapi( + config: &Config, portal: &crate::portal::PortalSession, engine: &SafeVaapiEngine, + units: mpsc::Sender, stop: Arc, idr: Arc, budget: Arc, +) -> Result<(), String> { + use crate::portal::DmaCapture; + let interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps)); + let mut next_poll = Instant::now(); + let mut last_send = Instant::now(); + let mut seed = 1; + let mut keepalive = random_keepalive(&mut seed); + let mut dma_version = None; + let mut cpu_version = None; + let mut cpu_frame = Vec::new(); + let mut uploaded = false; + let mut input_dma = false; + let mut blank_frame = false; + let mut dma_disabled = std::env::var("UU_PORTAL_DMABUF").is_ok_and(|v| v == "0"); + if dma_disabled { log::info!("Portal VAAPI DMA-BUF disabled by setting; pipeline=host-upload"); } + loop { + if stop.load(Ordering::Relaxed) || units.is_closed() { return Ok(()); } + std::thread::sleep(next_poll.saturating_duration_since(Instant::now()).min(Duration::from_millis(20))); + let now = Instant::now(); + if now < next_poll { continue; } + next_poll = (next_poll + interval).max(now); + if portal.size() != config.input { return Err("Portal size changed; restart encoder".into()); } + let Some(permit) = acquire(&budget, &units) else { return Ok(()); }; + if stop.load(Ordering::Relaxed) { return Ok(()); } + let due = last_send.elapsed() >= keepalive || idr.load(Ordering::Acquire); + let captured = if dma_disabled { Ok(DmaCapture::Cpu) } else { + portal.with_dmabuf(&mut dma_version, |frame, changed| { + if !changed && input_dma && !due { return Ok(None); } + if changed || !input_dma { + let result = unsafe { vaapi_import_frame(engine.0, frame) }; + if result != 0 { return Err(format!("VAAPI DMA-BUF import failed: {result}")); } + } + // vaapi_encode_frame waits for VA completion before the PipeWire lease is released. + let packet = vaapi_packet(engine, &idr)?; + if !input_dma { + log::info!("Portal VAAPI capture pipeline=dma-buf zero_cpu_copy=true source={}x{} fourcc={} modifier={}", + frame.width, frame.height, frame.fourcc, frame.modifier); + } + input_dma = true; + blank_frame = false; + uploaded = true; + Ok(Some(packet)) + }) + }; + let captured = match captured { + Ok(DmaCapture::Unavailable(code)) => { + let reason = match code { + 14 => "buffer layout unsupported or DMA-BUF disabled for this session", + 16 => "compositor selected shared memory", + 17 => "DMA-BUF negotiation or first frame timed out", + 18 => "explicit synchronization unsupported", + 19 => "nonlinear modifier unsupported", + _ => "DMA-BUF interface unavailable", + }; + log::warn!("Portal DMA-BUF unavailable: {reason} (code={code}); falling back to native VAAPI host upload"); + dma_disabled = true; portal.disable_dmabuf(); + DmaCapture::Cpu + }, + Err(error) => { + log::warn!("Portal DMA-BUF unavailable: {error}; falling back to native VAAPI host upload"); + dma_disabled = true; portal.disable_dmabuf(); + DmaCapture::Cpu + }, + Ok(value) => value, + }; + let data = match captured { + DmaCapture::Frame(Some(packet)) => packet, + DmaCapture::Frame(None) | DmaCapture::Pending => continue, + state @ (DmaCapture::Cpu | DmaCapture::Blank) => { + cpu_frame.resize(config.input.0 as usize * config.input.1 as usize * 4, 0); + let changed = if matches!(state, DmaCapture::Blank) { + let changed = !blank_frame; + if changed { cpu_frame.fill(0); } + blank_frame = true; + changed + } else { + blank_frame = false; + portal.capture_changed(&mut cpu_frame, &mut cpu_version)? + }; + if !changed && !input_dma && uploaded && !due { continue; } + if changed || input_dma || !uploaded { + let result = unsafe { vaapi_upload_host_buffer(engine.0, cpu_frame.as_ptr(), config.input.0 * 4) }; + if result != 0 { return Err(format!("Portal VAAPI upload failed: {result}")); } + input_dma = false; uploaded = true; + } + vaapi_packet(engine, &idr)? + }, + DmaCapture::Unavailable(_) => unreachable!(), + }; + if data.is_empty() { continue; } + if units.blocking_send(EncodedUnit { data, permit, completed_at: Instant::now() }).is_err() { return Ok(()); } + last_send = Instant::now(); + keepalive = random_keepalive(&mut seed); + } +} + fn vaapi_wait(dri3: &Option, capture: &mut Option, timeout: Duration) -> Result { match (dri3, capture) { (Some(c), _) => match unsafe { dri3_capture_wait_damage(c.0, timeout.as_millis().min(1000) as i32) } { @@ -664,6 +801,24 @@ fn pad_argb32(src: &[u8], src_w: u32, src_h: u32, dst: &mut [u8], dst_w: u32, ds #[cfg(test)] mod budget_tests { use super::*; + #[test] + #[ignore = "requires a VAAPI render node and FFmpeg for independent decode verification"] + fn vaapi_locked_frames_decode_as_black_in_both_codecs() { + use std::io::Write; + use std::process::{Command, Stdio}; + for codec in [crate::capture_setting::VideoCodec::H264, crate::capture_setting::VideoCodec::H265] { + let frame = black_frame(codec).unwrap(); + let mut decoder = Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-i", "pipe:0", + "-frames:v", "1", "-vf", "scale=1:1", "-pix_fmt", "rgb24", "-f", "rawvideo", "pipe:1"]) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + decoder.stdin.take().unwrap().write_all(&frame).unwrap(); + let decoded = decoder.wait_with_output().unwrap(); + assert!(decoded.status.success(), "decode failed for {codec:?}"); + assert_eq!(decoded.stdout.len(), 3); + assert!(decoded.stdout.iter().all(|&value| value <= 3), "frame is not black for {codec:?}"); + } + } + #[tokio::test] async fn closing_encoder_output_wakes_a_blocked_capture() { let budget = Arc::new(Semaphore::new(0)); @@ -726,3 +881,238 @@ mod frame_tests { } } } + +#[cfg(all(test, target_os = "linux", feature = "portal"))] +mod portal_hardware_tests { + // Run on an unlocked desktop with saved Portal consent and VAAPI selected. + // FFmpeg is used only for independent decode verification; desktop contents + // are neither saved nor printed. These tests do not lock the real desktop + // or replace system lock-screen acceptance in an isolated VM. + use super::*; + use crate::capture_setting::VideoCodec; + + struct Runner { + stop: Arc, + idr: Arc, + rx: mpsc::Receiver, + job: Option>>, + } + impl Drop for Runner { + fn drop(&mut self) { self.stop.store(true, Ordering::Release); self.rx.close(); } + } + impl Runner { + fn start(portal: Arc, codec: VideoCodec) -> Self { + let config = Config { + input: portal.size(), portal: Some(portal), display: "wayland:portal".into(), + output: None, fps: 60, gop: 120, cq: 18, target_bps: 16_000_000, + max_bps: 30_000_000, buffer_bits: 500_000, codec, + chroma: crate::capture_setting::CHROMA_420, vaapi: true, + }; + let (tx, rx) = mpsc::channel(4); + let stop = Arc::new(AtomicBool::new(false)); + let idr = Arc::new(AtomicBool::new(false)); + let worker_stop = stop.clone(); + let worker_idr = idr.clone(); + let job = tokio::task::spawn_blocking(move || run(config, tx, worker_stop, worker_idr, Arc::new(Semaphore::new(4)))); + Self { stop, idr, rx, job: Some(job) } + } + async fn frame(&mut self) -> Vec { + tokio::time::timeout(Duration::from_secs(10), self.rx.recv()).await.unwrap() + .expect("native capture stopped").data + } + async fn hevc_idr(&mut self) -> Vec { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let frame = self.frame().await; + let nals = crate::hevc::payload(usize::MAX, &frame).unwrap(); + if nals.iter().any(|nal| (16..=21).contains(&((nal[0] >> 1) & 63))) { return frame; } + } + }).await.expect("requested HEVC IDR did not arrive") + } + async fn finish(mut self) { + self.stop.store(true, Ordering::Release); + self.rx.close(); + tokio::time::timeout(Duration::from_secs(2), self.job.take().unwrap()).await + .expect("native encoder did not stop").unwrap().unwrap(); + } + } + fn decoded_is_black(data: &[u8], codec: VideoCodec) -> bool { + use std::io::Write; + use std::process::{Command, Stdio}; + let format = if codec == VideoCodec::H265 { "hevc" } else { "h264" }; + let mut decoder = Command::new("ffmpeg").args(["-hide_banner", "-loglevel", "error", "-f", format, + "-i", "pipe:0", "-frames:v", "1", "-pix_fmt", "rgb24", "-f", "rawvideo", "pipe:1"]) + .stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap(); + // Concurrent writing avoids filling stdin while the decoder is blocked on raw RGB output. + let mut input = decoder.stdin.take().unwrap(); + let encoded = data.to_vec(); + let writer = std::thread::spawn(move || input.write_all(&encoded)); + let decoded = decoder.wait_with_output().unwrap(); + writer.join().unwrap().unwrap(); + assert!(decoded.status.success() && !decoded.stdout.is_empty(), "native frame decode failed"); + decoded.stdout.iter().all(|&value| value <= 8) + } + fn resources() -> (usize, u64) { + let fds = std::fs::read_dir("/proc/self/fd").unwrap().count(); + let rss = std::fs::read_to_string("/proc/self/status").unwrap().lines() + .find(|line| line.starts_with("VmRSS:")).unwrap().split_whitespace().nth(1).unwrap().parse().unwrap(); + (fds, rss) + } + async fn reopen(portal: Arc) { + tokio::task::spawn_blocking(move || { + portal.end(); + assert!(portal.recover(&AtomicBool::new(false)).unwrap()); + }).await.unwrap(); + } + + #[tokio::test] + #[ignore = "requires an unlocked Portal desktop and saved consent"] + async fn portal_snapshot_after_dma_initialization() { + let _ = env_logger::builder().is_test(true).try_init(); + tokio::task::spawn_blocking(|| { + let mut portal = crate::portal::PortalSession::open().unwrap(); + let (locked, watch) = tokio::sync::watch::channel(Some(false)); + portal.test_lock_watch(watch); + assert!(matches!(portal.with_dmabuf(&mut None, |_, _| Ok(())).unwrap(), crate::portal::DmaCapture::Frame(()))); + let (width, height) = portal.size(); + let mut pixels = vec![0; width as usize * height as usize * 4]; + for state in [Some(true), None, Some(false)] { + locked.send(state).unwrap(); + portal.pause(); + portal.capture(&mut pixels).unwrap(); + assert!(pixels.chunks_exact(4).any(|p| p[..3].iter().any(|&v| v > 8)), + "healthy snapshot was blanked for lock state {state:?}"); + println!("snapshot_lock_state={state:?} valid_frame=true"); + } + }).await.unwrap(); + } + + // RUST_LOG=info cargo test --release --locked --features wayland,portal,inproc-av portal_five_minute_endurance_with_full_decode -- --ignored --nocapture + // Run for five minutes, independently decode every output frame, require equal + // encoded/decoded counts and no decode errors, and report CPU, RSS and FDs. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[ignore = "five-minute real Portal/VAAPI run; FFmpeg is used only as an independent decoder"] + async fn portal_five_minute_endurance_with_full_decode() { + use std::process::Stdio; + use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; + let _ = env_logger::builder().is_test(true).try_init(); + let session = tokio::task::spawn_blocking(crate::portal::PortalSession::open).await.unwrap().unwrap(); + let mut runner = Runner::start(Arc::new(session), VideoCodec::H265); + let mut decoder = tokio::process::Command::new("ffmpeg").args([ + "-hide_banner", "-loglevel", "error", "-nostats", "-threads", "2", "-f", "hevc", "-i", "pipe:0", + "-progress", "pipe:1", "-fps_mode", "passthrough", "-f", "null", "/dev/null", + ]).stdin(Stdio::piped()).stdout(Stdio::piped()).stderr(Stdio::piped()).kill_on_drop(true).spawn().unwrap(); + let mut input = decoder.stdin.take().unwrap(); + let mut errors = decoder.stderr.take().unwrap(); + let error_reader = tokio::spawn(async move { + let mut total = 0usize; + let mut buffer = [0; 2048]; + loop { + let count = errors.read(&mut buffer).await.unwrap(); + if count == 0 { return total; } + total += count; + } + }); + let output = decoder.stdout.take().unwrap(); + let progress_reader = tokio::spawn(async move { + let mut lines = BufReader::new(output).lines(); + let mut frames = 0u64; + while let Some(line) = lines.next_line().await.unwrap() { + if let Some(count) = line.strip_prefix("frame=") { frames = count.trim().parse().unwrap(); } + } + frames + }); + let ticks = || { + let stat = std::fs::read_to_string("/proc/self/stat").unwrap(); + let fields: Vec<_> = stat.split_once(") ").unwrap().1.split_whitespace().collect(); + fields[11].parse::().unwrap() + fields[12].parse::().unwrap() + }; + let hz = unsafe { libc::sysconf(libc::_SC_CLK_TCK) } as f64; + let started = Instant::now(); + let mut last_report = started; + let mut last_ticks = ticks(); + let mut frames = 0u64; + while started.elapsed() < Duration::from_secs(300) { + let data = runner.frame().await; + tokio::time::timeout(Duration::from_secs(5), input.write_all(&data)).await + .expect("verification decoder stopped consuming").unwrap(); + frames += 1; + if last_report.elapsed() >= Duration::from_secs(30) { + let now_ticks = ticks(); + let cpu = (now_ticks - last_ticks) as f64 / hz / last_report.elapsed().as_secs_f64() * 100.0; + let (fds, rss_kb) = resources(); + println!("endurance elapsed_s={} encoded_frames={} cpu_percent={cpu:.2} fds={fds} rss_kb={rss_kb}", + started.elapsed().as_secs(), frames); + last_ticks = now_ticks; last_report = Instant::now(); + } + } + runner.finish().await; + drop(input); + assert!(tokio::time::timeout(Duration::from_secs(15), decoder.wait()).await.unwrap().unwrap().success()); + assert_eq!(error_reader.await.unwrap(), 0, "verification decoder reported errors"); + let decoded = progress_reader.await.unwrap(); + assert_eq!(decoded, frames, "decoder lost encoded frames"); + println!("portal_endurance_passed duration_s={:.2} encoded_frames={} decoded_frames={} decode_errors=0", + started.elapsed().as_secs_f64(), frames, decoded); + } + + // RUST_LOG=info cargo test --release --locked --features wayland,portal,inproc-av portal_restart_resources_and_injected_lock_recovery -- --ignored --nocapture + // Recreate the Portal session 20 times, check resource growth, then inject + // locked/unknown/unlocked observations to check healthy-frame passthrough. + // Actually close/restore the source to check placeholder black and recovery. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[ignore = "requires an unlocked Portal desktop, saved consent, VAAPI and FFmpeg decode verification"] + async fn portal_restart_resources_and_injected_lock_recovery() { + let _ = env_logger::builder().is_test(true).try_init(); + let mut session = tokio::task::spawn_blocking(crate::portal::PortalSession::open).await.unwrap().unwrap(); + // Inject only the lock observation; actual session closure/recovery and GPU encoding remain real. + // This does not lock the user's desktop or bypass the isolated-VM acceptance guard. + let (locked, watch) = tokio::sync::watch::channel(Some(false)); + session.test_lock_watch(watch); + let portal = Arc::new(session); + let mut snapshots = Vec::new(); + for cycle in 0..20 { + let codec = if cycle % 2 == 0 { VideoCodec::H264 } else { VideoCodec::H265 }; + let mut runner = Runner::start(portal.clone(), codec); + let frame = runner.frame().await; + assert!(!decoded_is_black(&frame, codec), "restarted stream stayed black"); + runner.finish().await; + reopen(portal.clone()).await; + let (fds, rss_kb) = resources(); + snapshots.push((fds, rss_kb)); + println!("restart_cycle={} fds={} rss_kb={}", cycle + 1, fds, rss_kb); + } + let warm = snapshots[3]; + let last = *snapshots.last().unwrap(); + assert!(last.0 <= warm.0 + 2, "file descriptors grew across reconnects"); + assert!(last.1 <= warm.1 + 32 * 1024, "resident memory grew across reconnects"); + + let mut runner = Runner::start(portal.clone(), VideoCodec::H265); + assert!(!decoded_is_black(&runner.hevc_idr().await, VideoCodec::H265)); + for state in [Some(true), None, Some(false)] { + locked.send(state).unwrap(); + portal.pause(); + runner.idr.store(true, Ordering::Release); + assert!(!decoded_is_black(&runner.hevc_idr().await, VideoCodec::H265), + "healthy DMA source was blanked for lock state {state:?}"); + assert!(matches!(portal.with_dmabuf(&mut None, |_, _| Ok(())).unwrap(), crate::portal::DmaCapture::Frame(()))); + println!("dma_lock_state={state:?} valid_frame=true"); + } + for cycle in 0..3 { + locked.send(Some(true)).unwrap(); + portal.end(); + runner.idr.store(true, Ordering::Release); + assert!(decoded_is_black(&runner.hevc_idr().await, VideoCodec::H265), "closed Portal source did not produce placeholder frame"); + assert!(matches!(portal.with_dmabuf(&mut None, |_, _| Ok(())).unwrap(), crate::portal::DmaCapture::Cpu)); + locked.send(Some(false)).unwrap(); + let session = portal.clone(); + tokio::task::spawn_blocking(move || { session.recover(&AtomicBool::new(false)).unwrap(); }).await.unwrap(); + runner.idr.store(true, Ordering::Release); + assert!(!decoded_is_black(&runner.hevc_idr().await, VideoCodec::H265), "desktop did not recover after unlock observation"); + println!("injected_lock_cycle={} source_closed=true placeholder_black=true same_arc_recovered=true", cycle + 1); + } + runner.finish().await; + println!("portal_restart_resources_passed cycles=20 fd_growth={} rss_growth_kb={} injected_lock_cycles=3", + last.0 as i64 - warm.0 as i64, last.1 as i64 - warm.1 as i64); + } +} diff --git a/src/input.rs b/src/input.rs index ac16537..215d1aa 100644 --- a/src/input.rs +++ b/src/input.rs @@ -99,7 +99,22 @@ impl Controller { #[derive(Default)] struct CmdState { held: bool, pending: bool, passthrough: bool, combos: std::collections::BTreeMap } -pub struct InputInjector { mac_controller: std::sync::atomic::AtomicBool, cmd: std::sync::Mutex, serial:std::sync::Mutex<()>, owner:std::sync::Mutex, shortcuts: std::sync::Mutex, uinput: Option, screen: std::sync::Mutex<(u32,u32)>, portal_failures: std::sync::atomic::AtomicUsize, portal_outcomes: [std::sync::atomic::AtomicUsize; 3], portal:std::sync::Mutex>>, backend: crate::desktop_backend::DesktopBackend, pending: std::sync::Mutex, held: std::sync::Mutex, touch: std::sync::Mutex } +pub struct InputInjector { mac_controller: std::sync::atomic::AtomicBool, cmd: std::sync::Mutex, serial:std::sync::Mutex<()>, owner:std::sync::Mutex, shortcuts: std::sync::Mutex, uinput: Option, uinput_pointer: bool, screen: std::sync::Mutex<(u32,u32)>, portal_failures: std::sync::atomic::AtomicUsize, portal_outcomes: [std::sync::atomic::AtomicUsize; 3], portal:std::sync::Mutex>>, backend: crate::desktop_backend::DesktopBackend, pending: std::sync::Mutex, held: std::sync::Mutex, touch: std::sync::Mutex } + +fn active_output_count(json: &[u8]) -> Option { + let data: serde_json::Value = serde_json::from_slice(json).ok()?; + Some(data.get("outputs")?.as_array()?.iter() + .filter(|o| o["enabled"] == true && o["connected"] == true).count()) +} +fn single_portal_output() -> bool { + if cfg!(test) { return false; } + let monitors = crate::portal::monitors(); + if !monitors.is_empty() { return monitors.len() == 1; } + // KDE has no Mutter monitor API. Query once at injector creation, never during input. + std::process::Command::new("timeout").args(["2s", "kscreen-doctor", "-j"]).output().ok() + .filter(|output| output.status.success()) + .and_then(|output| active_output_count(&output.stdout)) == Some(1) +} impl InputInjector { pub fn new() -> Self { @@ -110,7 +125,11 @@ impl InputInjector { Err(e) => { log::warn!("uinput unavailable ({}); using the desktop input path. See README for the udev rule.", e.kind()); None } } } else { None }; - Self { mac_controller: std::sync::atomic::AtomicBool::new(false), cmd: std::sync::Mutex::new(CmdState::default()), serial:std::sync::Mutex::new(()), owner:std::sync::Mutex::new(DesktopOwner::default()), shortcuts: std::sync::Mutex::new(Shortcuts::default()), uinput, screen: std::sync::Mutex::new((0,0)), portal_failures: std::sync::atomic::AtomicUsize::new(0), portal_outcomes: std::array::from_fn(|_| std::sync::atomic::AtomicUsize::new(0)), portal:std::sync::Mutex::new(None), backend:crate::desktop_backend::detect().unwrap_or(crate::desktop_backend::DesktopBackend::X11), pending: std::sync::Mutex::new(PendingPointer::default()), held:std::sync::Mutex::new(HeldInput::default()), touch:std::sync::Mutex::new(TouchDrag::default()) } + let uinput_pointer = uinput.is_some() && (std::env::var("UU_INPUT_BACKEND").as_deref() == Ok("uinput") + || (backend == crate::desktop_backend::DesktopBackend::Portal && single_portal_output())); + if uinput_pointer { log::info!("Input pointer backend: uinput (motion, buttons and wheel)"); } + else if backend == crate::desktop_backend::DesktopBackend::Portal { log::info!("Input pointer backend: portal (monitor-local coordinates)"); } + Self { mac_controller: std::sync::atomic::AtomicBool::new(false), cmd: std::sync::Mutex::new(CmdState::default()), serial:std::sync::Mutex::new(()), owner:std::sync::Mutex::new(DesktopOwner::default()), shortcuts: std::sync::Mutex::new(Shortcuts::default()), uinput, uinput_pointer, screen: std::sync::Mutex::new((0,0)), portal_failures: std::sync::atomic::AtomicUsize::new(0), portal_outcomes: std::array::from_fn(|_| std::sync::atomic::AtomicUsize::new(0)), portal:std::sync::Mutex::new(None), backend, pending: std::sync::Mutex::new(PendingPointer::default()), held:std::sync::Mutex::new(HeldInput::default()), touch:std::sync::Mutex::new(TouchDrag::default()) } } pub fn bind_portal(&self,session:Option>) { @@ -251,7 +270,7 @@ impl InputInjector { *self.cmd.lock().unwrap() = CmdState::default(); let mut touch = self.touch.lock().unwrap(); self.lift_finger(&mut touch); - let mode = if controller != Controller::Mobile { TouchMode::Mouse } else { + let mode = if controller != Controller::Mobile || self.uinput_pointer { TouchMode::Mouse } else { match std::env::var("UU_TOUCH_INPUT").as_deref() { Ok("0") => TouchMode::Mouse, Ok("touch") => TouchMode::Touch, _ => TouchMode::Gesture } }; *touch = TouchDrag { mode, ..TouchDrag::default() }; @@ -478,6 +497,18 @@ impl InputInjector { } } fn inject_move(&self,x:i32,y:i32,relative:bool)->i32 { + if self.uinput_pointer && self.backend != crate::desktop_backend::DesktopBackend::X11 { + if let Some(uinput) = &self.uinput { + if relative { return Self::uinput_result(uinput.move_relative(x,y)); } + let (w,h) = *self.screen.lock().unwrap(); + if w > 1 && h > 1 { + let scale = |v: i32, extent: u32| (i64::from(v).clamp(0, i64::from(extent - 1)) + * i64::from(crate::uinput::ABS_MAX) / i64::from(extent - 1)) as i32; + return Self::uinput_result(uinput.move_absolute(scale(x,w),scale(y,h))); + } + return -1; + } + } // uinput ABS axes span the compositor's whole desktop. A captured output // uses its own pixel space; bind absolute motion to that Wayland output, // including when uinput remains in use for keyboard/buttons/relative input. @@ -512,7 +543,7 @@ impl InputInjector { fn inject_button(&self,button:i32,pressed:bool)->i32 { // Portal: clicks follow absolute motion on the same session, so a tap cannot land // before the compositor has moved the pointer there (uinput is a separate queue). - if self.backend==crate::desktop_backend::DesktopBackend::Portal { + if !self.uinput_pointer && self.backend==crate::desktop_backend::DesktopBackend::Portal { if let Some(portal)=self.portal.lock().unwrap().as_ref() { return self.portal_result("button", portal.button(button,pressed)); } } if let Some(uinput)=&self.uinput { @@ -619,11 +650,60 @@ impl InputInjector { true } - /// Mobile input-method text (`text_input`). X11 types it directly; Wayland re-maps a spare + fn emit_uinput_text(&self, plan: &[crate::uinput::TextKey]) -> usize { + let Some(devices) = self.uinput.as_ref() else { return 0 }; + let (mut typed, mut shift) = (0, 0); + for key in plan { + if key.code == 0 || key.code > 248 || key.shift > 248 { break; } + if shift != key.shift { + if shift != 0 { + if Self::uinput_result(devices.key(shift, false)) != 0 { break; } + shift = 0; + } + if key.shift != 0 { + if Self::uinput_result(devices.key(key.shift, true)) != 0 { break; } + shift = key.shift; + } + } + let pressed = Self::uinput_result(devices.key(key.code, true)) == 0; + let released = Self::uinput_result(devices.key(key.code, false)) == 0; + if pressed { typed += 1; } + if !pressed || !released { break; } + } + if shift != 0 { Self::uinput_result(devices.key(shift, false)); } + typed + } + fn type_uinput_text(&self, text: &str) -> usize { + self.type_uinput_text_when(text, || { + uinput_layout_types_ascii(&std::env::var("XDG_CURRENT_DESKTOP").unwrap_or_default(), gnome_layout_types_ascii) + }) + } + fn type_uinput_text_when(&self, text: &str, layout_allowed: impl FnOnce() -> bool) -> usize { + if self.uinput.is_none() || !self.held.lock().unwrap().keys.is_empty() { return 0; } + // XKB can still report US while IBus consumes physical keys for composition. + // Leave committed text intact for the existing native/clipboard fallback. + if !layout_allowed() { return 0; } + self.emit_uinput_text(&crate::uinput::text_plan(text)) + } + + /// Mobile input-method text (`text_input`). uinput types the layout-mappable ASCII prefix + /// without clipboard access. X11 types directly; native Wayland re-maps a spare /// keycode in the virtual keyboard's keymap. Portal and other Wayland environments inject /// via clipboard paste (Shift+Insert). Set UU_WAYLAND_TEXT=0 to disable Wayland text injection. pub fn text_input(&self, text: &str) { let _serial=self.serial.lock().unwrap(); + if self.backend != crate::desktop_backend::DesktopBackend::X11 { + if std::env::var("UU_WAYLAND_TEXT").is_ok_and(|v| v == "0") { return; } + let typed = self.type_uinput_text(text); + if typed > 0 { log::info!("Desktop text input (uinput): chars={typed}"); } + if typed == text.len() { return; } + // The uinput planner consumes ASCII only, so the character count is also a byte offset. + let rest = &text[typed..]; + return self.desktop_text_input(rest); + } + self.desktop_text_input(text); + } + fn desktop_text_input(&self, text: &str) { let Ok(c_text)=std::ffi::CString::new(text.replace('\0',"")) else { return }; if self.backend==crate::desktop_backend::DesktopBackend::X11 { let typed=unsafe { x11_type_utf8(c_text.as_ptr()) }; @@ -716,6 +796,149 @@ mod motion_tests { } } +#[cfg(all(test, target_os = "linux"))] +mod kernel_pointer_tests { + use super::*; + #[test] + fn committed_text_with_gnome_ime_emits_no_uinput_keys() { + for source in ["[('ibus', 'libpinyin'), ('xkb', 'us')]", "@a(ss) []"] { + let (devices, mut keyboard, _) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); input.uinput = Some(devices); + let checked = std::cell::Cell::new(false); + let text = "fixture"; + let typed = input.type_uinput_text_when(text, || { + checked.set(true); + uinput_layout_types_ascii("GNOME", || first_source_types_ascii(source)) + }); + assert!(checked.get()); + assert_eq!(typed, 0); + assert!(crate::uinput::Devices::recorded_events_for_test(&mut keyboard).is_empty()); + } + } + + #[test] + fn text_plan_emits_balanced_modifiers_without_portal_or_clipboard() { + let (devices, mut keyboard, _) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); input.uinput = Some(devices); + let plan = [ + crate::uinput::TextKey { code:30, shift:42 }, + crate::uinput::TextKey { code:48, shift:42 }, + crate::uinput::TextKey { code:2, shift:0 }, + ]; + assert_eq!(input.emit_uinput_text(&plan), 3); + let keys: Vec<_> = crate::uinput::Devices::recorded_events_for_test(&mut keyboard).into_iter() + .filter(|event| event.0 == 1).collect(); + assert_eq!(keys, [(1,42,1),(1,30,1),(1,30,0),(1,48,1),(1,48,0),(1,42,0),(1,2,1),(1,2,0)]); + assert_eq!(input.portal_input_counts_for_test(), [0; 3]); + } + + #[test] + fn partial_text_plan_releases_shift_and_does_not_disturb_held_keys() { + let (devices, mut keyboard, _) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); input.uinput = Some(devices); + assert_eq!(input.emit_uinput_text(&[ + crate::uinput::TextKey { code:30, shift:42 }, + crate::uinput::TextKey { code:0, shift:0 }, + ]), 1); + let keys: Vec<_> = crate::uinput::Devices::recorded_events_for_test(&mut keyboard).into_iter() + .filter(|event| event.0 == 1).collect(); + assert_eq!(keys, [(1,42,1),(1,30,1),(1,30,0),(1,42,0)]); + input.held.lock().unwrap().keys.insert(0xffe3); + assert_eq!(input.type_uinput_text("fixture"), 0); + assert_eq!(input.held.lock().unwrap().keys.len(), 1); + } + + #[test] + #[ignore = "requires the current X11/XWayland layout; records events without desktop injection"] + fn committed_ascii_text_bypasses_portal_with_live_keymap() { + let fixture = "aAzZ09!@#$%^&*()_+-=[]{};:'\",.<>/?\\|`~\t\n"; + assert_eq!(crate::uinput::text_plan(fixture).len(), fixture.len(), "fixture not available in idle host keymap"); + let (devices, mut keyboard, _) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); input.uinput = Some(devices); + let input = std::sync::Arc::new(input); + let binding = input.portal.lock().unwrap(); + let worker_input = input.clone(); + let (done, received) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { worker_input.text_input(fixture); let _ = done.send(()); }); + // On failure panic with the binding held: poisoning prevents a fallback from touching the clipboard. + assert!(received.recv_timeout(std::time::Duration::from_secs(2)).is_ok(), "text path waited for Portal/clipboard"); + drop(binding); + worker.join().unwrap(); + let keys = crate::uinput::Devices::recorded_events_for_test(&mut keyboard); + let down = keys.iter().filter(|e| e.0 == 1 && e.2 == 1).count(); + let up = keys.iter().filter(|e| e.0 == 1 && e.2 == 0).count(); + assert!(down >= fixture.len()); + assert_eq!(down, up); + assert_eq!(input.portal_input_counts_for_test(), [0; 3]); + } + + #[test] + fn full_uinput_bypasses_portal_binding_and_preserves_move_click_order() { + for controller in [Controller::Other, Controller::Mobile] { + let (devices, mut keyboard, mut pointer) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); + input.uinput = Some(devices); + input.uinput_pointer = true; + let input = std::sync::Arc::new(input); + // A blocked Portal must not stall any kernel-routed input. The recording + // files exercise the real event writer without typing or clicking on the desktop. + let binding = input.portal.lock().unwrap(); + let worker_input = input.clone(); + let (done, received) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + worker_input.set_controller(controller); + assert_eq!(worker_input.touch.lock().unwrap().mode, TouchMode::Mouse); + worker_input.windows_key_event(65, true); + worker_input.windows_key_event(65, false); + worker_input.mouse_absolute_normalized(1.0, 0.0, (3072, 1920)); + worker_input.wire_mouse_button(1, true); + worker_input.wire_mouse_button(1, false); + worker_input.mouse_scroll(120, 240); + worker_input.flush_pointer(); + let _ = done.send(()); + }); + let completed = received.recv_timeout(std::time::Duration::from_secs(1)); + drop(binding); + worker.join().unwrap(); + assert!(completed.is_ok(), "kernel input waited for a Portal binding"); + assert_eq!(input.portal_input_counts_for_test(), [0; 3]); + let keys = crate::uinput::Devices::recorded_events_for_test(&mut keyboard); + assert_eq!(keys, [(1, 30, 1), (0, 0, 0), (1, 30, 0), (0, 0, 0)]); + let mouse = crate::uinput::Devices::recorded_events_for_test(&mut pointer); + assert_eq!(&mouse[..7], &[ + (3, 0, 65535), (3, 1, 0), (0, 0, 0), + (1, 0x110, 1), (0, 0, 0), (1, 0x110, 0), (0, 0, 0), + ]); + assert_eq!(mouse.iter().filter(|e| e.0 == 2 && e.1 == 8).map(|e| e.2).sum::(), 2); + assert_eq!(mouse.iter().filter(|e| e.0 == 2 && e.1 == 6).map(|e| e.2).sum::(), 1); + } + } + + #[test] + fn full_uinput_clamps_coordinates_and_keeps_relative_motion_on_kernel() { + let (devices, mut keyboard, mut pointer) = crate::uinput::Devices::recording_for_test(); + let mut input = InputInjector::without_desktop(); + input.uinput = Some(devices); input.uinput_pointer = true; + *input.screen.lock().unwrap() = (3072, 1920); + assert_eq!(input.inject_move(-5, 4000, false), 0); + assert_eq!(crate::uinput::Devices::recorded_events_for_test(&mut pointer), + [(3, 0, 0), (3, 1, 65535), (0, 0, 0)]); + assert_eq!(input.inject_move(4, -3, true), 0); + assert_eq!(crate::uinput::Devices::recorded_events_for_test(&mut keyboard), + [(2, 0, 4), (2, 1, -3), (0, 0, 0)]); + assert_eq!(input.portal_input_counts_for_test(), [0; 3]); + } + + #[test] + fn kde_output_count_ignores_disabled_and_disconnected_outputs() { + assert_eq!(active_output_count(br#"{"outputs":[{"enabled":true,"connected":true},{"enabled":false,"connected":true}]}"#), Some(1)); + assert_eq!(active_output_count(br#"{"outputs":[{"enabled":true,"connected":true},{"enabled":true,"connected":true}]}"#), Some(2)); + assert_eq!(active_output_count(br#"{"outputs":[{"enabled":true,"connected":false}]}"#), Some(0)); + assert_eq!(active_output_count(br#"{}"#), None); + assert_eq!(active_output_count(b"invalid"), None); + } +} + #[cfg(test)] mod shortcut_tests { use super::*; @@ -807,11 +1030,14 @@ impl Drop for DesktopInputSession {fn drop(&mut self) {self.disconnected();}} #[cfg(test)] impl InputInjector { + pub fn portal_input_counts_for_test(&self) -> [usize; 3] { + std::array::from_fn(|i| self.portal_outcomes[i].load(std::sync::atomic::Ordering::Relaxed)) + } pub fn without_desktop() -> Self { Self { mac_controller:std::sync::atomic::AtomicBool::new(false), cmd:std::sync::Mutex::new(CmdState::default()), serial:std::sync::Mutex::new(()), owner:std::sync::Mutex::new(DesktopOwner::default()), shortcuts:std::sync::Mutex::new(Shortcuts::default()), - uinput:None, screen:std::sync::Mutex::new((0,0)), portal_failures:std::sync::atomic::AtomicUsize::new(0), + uinput:None, uinput_pointer:false, screen:std::sync::Mutex::new((0,0)), portal_failures:std::sync::atomic::AtomicUsize::new(0), portal_outcomes:std::array::from_fn(|_|std::sync::atomic::AtomicUsize::new(0)), portal:std::sync::Mutex::new(None), backend:crate::desktop_backend::DesktopBackend::Portal, pending:std::sync::Mutex::new(PendingPointer::default()), held:std::sync::Mutex::new(HeldInput::default()), @@ -835,6 +1061,9 @@ fn first_source_types_ascii(sources: &str) -> bool { const NON_LATIN: &[&str]=&["af","am","ara","bd","bg","bt","by","eg","et","ge","gr","il","in","iq","ir","kg","kh","kz","la","lk","mk","mm","mn","mv","np","ru","sy","th","tj","ua"]; kind=="xkb" && !NON_LATIN.contains(&base) && !variant.contains("cyrillic") && (base!="rs" || variant.contains("latin")) } +fn uinput_layout_types_ascii(desktop: &str, gnome_layout: impl FnOnce() -> bool) -> bool { + !desktop.split(':').any(|name| name.eq_ignore_ascii_case("gnome")) || gnome_layout() +} /// GNOME's active input source is the first of `mru-sources` (else of `sources`). Cached for 2 s. fn gnome_layout_types_ascii() -> bool { #[cfg(test)] @@ -857,6 +1086,17 @@ fn gnome_layout_types_ascii() -> bool { mod text_keysym_tests { use super::*; #[test] + fn uinput_preserves_gnome_input_source_guard_without_gating_other_desktops() { + for desktop in ["GNOME", "ubuntu:GNOME", "gnome:GNOME-Classic"] { + assert!(!uinput_layout_types_ascii(desktop, || first_source_types_ascii("[('ibus', 'libpinyin'), ('xkb', 'us')]"))); + assert!(!uinput_layout_types_ascii(desktop, || false)); + assert!(uinput_layout_types_ascii(desktop, || first_source_types_ascii("[('xkb', 'us'), ('ibus', 'libpinyin')]"))); + } + for desktop in ["KDE", "Hyprland", ""] { + assert!(uinput_layout_types_ascii(desktop, || panic!("GNOME input-source state must not gate another desktop"))); + } + } + #[test] fn ascii_text_becomes_keysyms_and_other_text_does_not() { assert_eq!(ascii_keysyms("Hi!\n\t~"), Some(vec![0x48,0x69,0x21,0xff0d,0xff09,0x7e])); assert_eq!(ascii_keysyms("čaj"), None); diff --git a/src/locked_video.rs b/src/locked_video.rs index 2ee8bab..98c5002 100644 --- a/src/locked_video.rs +++ b/src/locked_video.rs @@ -5,6 +5,14 @@ use webrtc::{media::Sample, peer_connection::{RTCPeerConnection, peer_connection use crate::{capture_setting::VideoCodec, desktop_session::PortalDesktop, video_track::VideoTrack}; async fn black_frame(codec: VideoCodec) -> Result { + #[cfg(feature = "inproc-av")] + if crate::video::native_engine_eligible(crate::video_select::selection().encoder, crate::video::NativeDisplayKind::Portal) { + match tokio::task::spawn_blocking(move || crate::inproc_encoder::black_frame(codec)).await { + Ok(Ok(frame)) => return Ok(frame), + Ok(Err(error)) => log::warn!("Locked video in-process VAAPI unavailable: {error}; falling back to FFmpeg"), + Err(_) => log::warn!("Locked video in-process worker failed; falling back to FFmpeg"), + } + } let mut command = crate::video::ffmpeg_command(); command.args(["-hide_banner", "-loglevel", "error", "-nostdin", "-f", "lavfi", "-i", "color=c=black:s=1280x720:r=1", "-frames:v", "1", diff --git a/src/portal.rs b/src/portal.rs index 2b03c29..aa8c62a 100644 --- a/src/portal.rs +++ b/src/portal.rs @@ -15,6 +15,20 @@ impl std::error::Error for PortalError {} #[derive(Clone, Copy, PartialEq, Eq)] pub struct FrameVersion(usize, u64); + +#[repr(C)] +#[derive(Default)] +pub struct DmaFrame { + pub fd: i32, + pub fourcc: u32, + pub width: u32, + pub height: u32, + pub stride: u32, + pub offset: u32, + pub size: u32, + pub modifier: u64, +} +pub enum DmaCapture { Frame(T), Pending, Cpu, Blank, Unavailable(i32) } #[cfg(unix)] #[path = "portal_permissions.rs"] mod permissions; @@ -25,6 +39,7 @@ mod native { crate::engine_ffi! { crate::engine_plugin::Engine::Portal; fn uu_portal_open_persist(restore: *const libc::c_char, save: extern "C" fn(*const libc::c_char, *mut c_void), data: *mut c_void, error: *mut i32, width: *mut u32, height: *mut u32) -> *mut c_void; fn uu_portal_open_persist_cancellable(restore: *const libc::c_char, save: extern "C" fn(*const libc::c_char, *mut c_void), save_data: *mut c_void, cancelled: extern "C" fn(*mut c_void) -> i32, cancel_data: *mut c_void, error: *mut i32, width: *mut u32, height: *mut u32) -> *mut c_void; + fn uu_portal_open_persist_capture(restore: *const libc::c_char, save: extern "C" fn(*const libc::c_char, *mut c_void), save_data: *mut c_void, cancelled: Option i32>, cancel_data: *mut c_void, prefer_dmabuf: i32, error: *mut i32, width: *mut u32, height: *mut u32) -> *mut c_void; fn uu_portal_clipboard_supported(portal: *mut c_void) -> i32; fn uu_portal_clipboard_ready(portal: *mut c_void) -> i32; fn uu_portal_clipboard_set(portal: *mut c_void, text: *const u8, length: usize) -> i32; @@ -35,6 +50,9 @@ mod native { fn uu_portal_input_dispatch(portal: *mut c_void); fn uu_portal_reset_frame(portal: *mut c_void); fn uu_portal_capture_since(portal: *mut c_void, bgra: *mut u8, length: usize, revision: *mut u64) -> i32; + fn uu_portal_dmabuf_acquire(portal: *mut c_void, frame: *mut super::DmaFrame, revision: *mut u64) -> i32; + fn uu_portal_dmabuf_release(portal: *mut c_void); + fn uu_portal_dmabuf_disable(portal: *mut c_void); fn uu_portal_move(portal: *mut c_void, x: f64, y: f64, relative: i32) -> i32; fn uu_portal_button(portal: *mut c_void, button: i32, pressed: i32) -> i32; fn uu_portal_key(portal: *mut c_void, keysym: u32, keycode: i32, pressed: i32) -> i32; @@ -149,7 +167,21 @@ mod native { if result.is_err() { log::warn!("Portal permission cache callback failed"); } } let restore = token.as_ref().map_or(std::ptr::null(), |value| value.as_ptr()); - let pointer = unsafe { + #[allow(unused_mut)] + let mut pointer: *mut c_void = std::ptr::null_mut(); + #[cfg(feature = "inproc-av")] + if crate::video::native_engine_eligible(crate::video_select::selection().encoder, crate::video::NativeDisplayKind::Portal) + && !std::env::var("UU_PORTAL_DMABUF").is_ok_and(|v| v == "0") { + let callback = cancelled.map(|_| is_cancelled as extern "C" fn(*mut c_void) -> i32); + let cancel_data = cancelled.map_or(std::ptr::null_mut(), |value| value as *const _ as *mut c_void); + pointer = unsafe { uu_portal_open_persist_capture( + restore, save_token, &mut cache as *mut _ as *mut c_void, callback, cancel_data, 1, + &mut error, &mut width, &mut height, + ) }; + if pointer.is_null() && error != 0 { return Err(super::PortalError { code: error }); } + if pointer.is_null() { log::warn!("Portal engine lacks DMA-BUF initialization; falling back to legacy initialization"); } + } + let pointer = if !pointer.is_null() { pointer } else { unsafe { if let Some(cancelled) = cancelled { uu_portal_open_persist_cancellable( restore, save_token, &mut cache as *mut _ as *mut c_void, @@ -159,7 +191,7 @@ mod native { } else { uu_portal_open_persist(restore, save_token, &mut cache as *mut _ as *mut c_void, &mut error, &mut width, &mut height) } - }; + } }; let handle = NonNull::new(pointer).ok_or(super::PortalError { code: error })?; let (mut x, mut y) = (0, 0); let granted = (unsafe { uu_portal_stream_position(handle.as_ptr(), &mut x, &mut y) } == 0) @@ -172,9 +204,8 @@ mod native { if super::permissions::save(dir, &saved, &token).is_err() { log::warn!("Portal permission cache could not be saved"); } } if wanted.is_some() && granted != wanted { log::warn!("Portal granted a different monitor than requested"); } - // A freshly opened session holds no lock-screen frame: only an observed lock (or a - // closed stream) must discard the cached frame. Starting "locked" cleared the first - // valid frame, and a static desktop sends no new one, so the viewer stayed black (#2). + // Keep a freshly granted compositor frame regardless of lock state. Discarding it + // before new damage can leave a static desktop permanently blank (#2). Ok(Self { handle: Mutex::new(Handle(handle)), locked: crate::session_lock::watch_locked(), was_locked: std::sync::atomic::AtomicBool::new(false), recovery: Mutex::new(Recovery::default()), size: pack(width, height).into(), monitor: Mutex::new(granted) }) } /// Frame pixels of the monitor the stream currently follows. @@ -211,6 +242,10 @@ mod native { pub fn open() -> Result { Self::open_inner(None, None, false).map_err(|error| error.to_string()) } + #[cfg(test)] + pub(crate) fn test_lock_watch(&mut self, locked: tokio::sync::watch::Receiver>) { + self.locked = locked; + } /// A consent saved before touch support restores keyboard and pointer only. Ask the host once /// more (GNOME shows its share dialog; the consent timeout closes it) and swap the new session /// in when it grants the touchscreen for this monitor at this size; until then, and on any @@ -276,6 +311,7 @@ mod native { } pub fn pause(&self) { if let Ok(handle) = self.handle.lock() { + if unsafe { uu_portal_capture_alive(handle.0.as_ptr()) } != 0 { return; } unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; self.was_locked.store(true, std::sync::atomic::Ordering::Release); } else { @@ -283,7 +319,46 @@ mod native { } } pub fn capture(&self, frame: &mut [u8]) -> Result<(), String> { - self.capture_changed(frame, &mut None).map(|_| ()) + let mut version = None; + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(1); + loop { + self.capture_changed(frame, &mut version)?; + // A one-shot snapshot must wait for SHM renegotiation after DMA initialization. + // Release the handle between attempts so input and recovery remain responsive. + if version != Some(super::FrameVersion(0, 0)) || std::time::Instant::now() >= deadline { return Ok(()); } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + } + pub fn disable_dmabuf(&self) { + if let Ok(handle) = self.handle.lock() { unsafe { uu_portal_dmabuf_disable(handle.0.as_ptr()) }; } + } + pub fn with_dmabuf(&self, version: &mut Option, + encode: impl FnOnce(&super::DmaFrame, bool) -> Result, + ) -> Result, String> { + let handle = self.handle.lock().map_err(|_| "Portal session lock poisoned")?; + unsafe { uu_portal_input_dispatch(handle.0.as_ptr()) }; + // Hold the handle guard through GPU completion, so monitor swaps cannot invalidate a lease. + if unsafe { uu_portal_capture_alive(handle.0.as_ptr()) } == 0 { + return Ok(super::DmaCapture::Cpu); + } + let identity = handle.0.as_ptr() as usize; + let mut revision = version.filter(|v| v.0 == identity).map_or(0, |v| v.1); + let mut frame = super::DmaFrame::default(); + match unsafe { uu_portal_dmabuf_acquire(handle.0.as_ptr(), &mut frame, &mut revision) } { + code @ (0 | 12) => { + struct Lease(*mut c_void); + impl Drop for Lease { + fn drop(&mut self) { unsafe { uu_portal_dmabuf_release(self.0) }; } + } + let _lease = Lease(handle.0.as_ptr()); + let encoded = encode(&frame, code == 0)?; + *version = Some(super::FrameVersion(identity, revision)); + Ok(super::DmaCapture::Frame(encoded)) + }, + 10 => Ok(super::DmaCapture::Pending), + 15 => Ok(super::DmaCapture::Blank), + code => Ok(super::DmaCapture::Unavailable(code)), + } } pub fn capture_changed(&self, frame: &mut [u8], version: &mut Option) -> Result { let (width, height) = self.size(); @@ -293,21 +368,18 @@ mod native { // Drain EIS even on a static desktop so pause/disconnect and sync // events are processed without waiting for remote input. unsafe { uu_portal_input_dispatch(handle.0.as_ptr()) }; - let state = *self.locked.borrow(); - if state != Some(false) { - // Unknown state (watcher not reported yet): show nothing but keep the cached frame. - if state == Some(true) { + if unsafe { uu_portal_capture_alive(handle.0.as_ptr()) } == 0 { + // A healthy source is forwarded regardless of lock state. Only an unavailable + // source is blanked; wait for unlock before reopening a compositor-closed session. + if *self.locked.borrow() != Some(false) { unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; self.was_locked.store(true, std::sync::atomic::Ordering::Release); + return Ok(blank_changed(frame, version)); + } + if self.was_locked.load(std::sync::atomic::Ordering::Acquire) { + if self.take_lock_observation()? { unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; } + return Ok(blank_changed(frame, version)); } - return Ok(blank_changed(frame, version)); - } - if self.was_locked.load(std::sync::atomic::Ordering::Acquire) { - // First unlocked capture after a real lock: drop frames captured while locked. - if self.take_lock_observation()? { unsafe { uu_portal_reset_frame(handle.0.as_ptr()) }; } - return Ok(blank_changed(frame, version)); - } - if unsafe { uu_portal_capture_alive(handle.0.as_ptr()) } == 0 { // Serialize recovery without making another capture worker hold // the input mutex while waiting for this worker's Portal calls. let mut recovery = match self.recovery.try_lock() { @@ -351,6 +423,7 @@ mod native { // Recheck lock state on the next capture before showing the new session. return Ok(blank_changed(frame, version)); } else { + self.was_locked.store(false, std::sync::atomic::Ordering::Release); match self.recovery.try_lock() { Ok(mut recovery) => recovery.retry_at = None, Err(TryLockError::WouldBlock) => (), @@ -739,6 +812,10 @@ impl PortalSession { pub fn pause(&self) {} pub fn capture(&self, _: &mut [u8]) -> Result<(), String> { Err(error(15)) } pub fn capture_changed(&self, _: &mut [u8], _: &mut Option) -> Result { Err(error(15)) } + pub fn disable_dmabuf(&self) {} + pub fn with_dmabuf(&self, _: &mut Option, _: impl FnOnce(&DmaFrame, bool) -> Result) -> Result, String> { + Ok(DmaCapture::Unavailable(-1)) + } pub fn move_pointer(&self, _: f64, _: f64, _: bool) -> Result<(), String> { Err(error(15)) } pub fn button(&self, _: i32, _: bool) -> Result<(), String> { Err(error(15)) } pub fn touch_supported(&self) -> bool { false } diff --git a/src/session_lock.rs b/src/session_lock.rs index 40258a0..1b2b46b 100644 --- a/src/session_lock.rs +++ b/src/session_lock.rs @@ -92,8 +92,7 @@ pub fn watch_locked() -> watch::Receiver> { if locked.is_none() { session = None; } // session ended or changed: look it up again sender.send_if_modified(|current| if *current != locked { log::info!("Desktop lock state: {}", match locked { Some(true) => "locked", Some(false) => "unlocked", None => "unknown" }); - // The viewer only sees black frames while locked and gives no hint (#61). - if locked == Some(true) { log::info!("Desktop is locked: viewers see a black screen until it is unlocked (viewer 自动解锁被控端, or `loginctl unlock-session` on the host)"); } + if locked == Some(true) { log::info!("Desktop is locked; capture continues while the compositor supplies frames"); } *current = locked; true } else { false }); std::thread::sleep(Duration::from_secs(1)); diff --git a/src/uinput.rs b/src/uinput.rs index e538f65..fec1486 100644 --- a/src/uinput.rs +++ b/src/uinput.rs @@ -10,6 +10,29 @@ //! buttons and wheels, like a QEMU tablet). Mixing relative and absolute axes in one device //! confuses libinput. +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct TextKey { pub code: u16, pub shift: u16 } + +/// Resolve an ASCII prefix against the host's current XKB layout and locked modifiers. +/// X11/XWayland is queried read-only; the returned events are emitted through uinput. +pub fn text_plan(text: &str) -> Vec { + #[cfg(target_os = "linux")] + { + extern "C" { + fn uu_uinput_text_plan(text: *const u8, length: usize, plan: *mut TextKey, capacity: usize) -> usize; + } + let capacity = text.bytes().take_while(u8::is_ascii).count(); + if capacity == 0 { return Vec::new(); } + let mut plan = vec![TextKey::default(); capacity]; + let count = unsafe { uu_uinput_text_plan(text.as_ptr(), text.len(), plan.as_mut_ptr(), plan.len()) }; + plan.truncate(count); + plan + } + #[cfg(not(target_os = "linux"))] + { let _ = text; Vec::new() } +} + #[cfg(target_os = "linux")] mod imp { use std::fs::{File, OpenOptions}; @@ -95,6 +118,31 @@ mod imp { pub struct Devices { keyboard: Mutex, pointer: Mutex } impl Devices { + #[cfg(test)] + pub fn recording_for_test() -> (Self, File, File) { + let recording = || { + let path = std::env::temp_dir().join(format!("uu-uinput-test-{}-{}", std::process::id(), rand::random::())); + let file = OpenOptions::new().read(true).write(true).create_new(true).mode(0o600).open(&path).unwrap(); + std::fs::remove_file(path).unwrap(); + let reader = file.try_clone().unwrap(); + (Mutex::new(Device(file)), reader) + }; + let (keyboard, keys) = recording(); + let (pointer, mouse) = recording(); + (Self { keyboard, pointer }, keys, mouse) + } + #[cfg(test)] + pub fn recorded_events_for_test(file: &mut File) -> Vec<(u16, u16, i32)> { + use std::io::{Read, Seek, SeekFrom}; + file.seek(SeekFrom::Start(0)).unwrap(); + let mut data = Vec::new(); + file.read_to_end(&mut data).unwrap(); + assert_eq!(data.len() % std::mem::size_of::(), 0); + data.chunks_exact(std::mem::size_of::()).map(|chunk| { + let event = unsafe { std::ptr::read_unaligned(chunk.as_ptr().cast::()) }; + (event.kind, event.code, event.value) + }).collect() + } pub fn open() -> std::io::Result { let keyboard = Device::create("uu-remote-linux keyboard", 1, |d| { d.ioctl(UI_SET_EVBIT, EV_KEY.into())?; diff --git a/src/video.rs b/src/video.rs index 6e91e78..5c549ec 100644 --- a/src/video.rs +++ b/src/video.rs @@ -374,6 +374,9 @@ pub async fn ffmpeg_encoders() -> Option> { /// Installing FFmpeg clears the window line on the next check; no restart is needed. pub fn check_ffmpeg_usable() { if crate::desktop_backend::headless() { return; } + #[cfg(feature = "inproc-av")] + if native_engine_eligible(crate::video_select::selection().encoder, NativeDisplayKind::Portal) + && native_display_kind() == NativeDisplayKind::Portal { return; } tokio::spawn(async { static LAST_NOTICE: std::sync::Mutex> = std::sync::Mutex::new(None); let summary = crate::i18n::s("无法传输远程画面","Cannot stream the remote desktop"); @@ -515,7 +518,7 @@ pub(crate) async fn probe_encoder_idr(encoder: Encoder) -> crate::video_select:: } /// True when `read_encoder_with` would open the in-process engine for this candidate -/// on the selected desktop backend (X11 only; not Wayland, portal, KMS, or headless). +/// on the selected backend (X11 hardware encoders, or Portal VAAPI). fn native_engine_would_run(encoder: Encoder) -> bool { native_engine_eligible(encoder, native_display_kind()) } @@ -695,7 +698,7 @@ async fn read_encoder( if encoder == Encoder::Cpu { return read_encoder_with(size,display,portal,latest,Encoder::Cpu,true,sequence,settings,idr,budget,selection).await; } #[cfg_attr(not(feature = "inproc-av"), allow(unused_mut))] let mut result = read_encoder_with(size,display.clone(),portal.clone(),latest.clone(),encoder,true,sequence.clone(),settings,idr.clone(),budget.clone(),selection).await; - // A failed native NVENC session (old driver, unsupported mode) still leaves FFmpeg's NVENC process pipeline. + // A failed native session still leaves the matching FFmpeg hardware pipeline. #[cfg(feature = "inproc-av")] if let Err(error) = &result { if !latest.is_closed() && native_inproc(&display, portal.is_some(), encoder) && x11_resized(&display, portal.is_some(), size).is_none() { @@ -720,18 +723,19 @@ fn x11_resized(display: &str, portal: bool, size: (u32, u32)) -> Option<(u32, u3 crate::capture::x11_size().filter(|current| *current != size) } -/// Same predicate `native_inproc` uses: X11 only, and the in-process engine is selected. +/// Same predicate `native_inproc` uses for backend/encoder eligibility. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum NativeDisplayKind { X11, Wayland, Portal, Kms, None } pub(crate) fn native_engine_eligible(encoder: Encoder, kind: NativeDisplayKind) -> bool { #[cfg(feature = "inproc-av")] - { kind == NativeDisplayKind::X11 && crate::inproc_encoder::selected(encoder) } + { (kind == NativeDisplayKind::X11 || (kind == NativeDisplayKind::Portal && encoder == Encoder::Vaapi)) + && crate::inproc_encoder::selected(encoder) } #[cfg(not(feature = "inproc-av"))] { let _ = (encoder, kind); false } } -/// The native in-process NVENC path serves X11 only (not Wayland/KMS capture or a portal session). +/// NVENC serves X11; VAAPI also accepts frames from the existing Portal session. #[cfg(feature = "inproc-av")] fn native_inproc(display: &str, portal: bool, encoder: Encoder) -> bool { native_engine_eligible(encoder, display_kind(display, portal)) @@ -749,7 +753,7 @@ fn display_kind(display: &str, portal: bool) -> NativeDisplayKind { #[cfg(feature = "inproc-av")] async fn read_inprocess( config: crate::inproc_encoder::Config, latest: mpsc::Sender>, - sequence: Arc, idr: Arc, budget: Arc, + sequence: Arc, idr: Arc, budget: Arc, source: (usize, u64), ) -> Result<(), String> { let codec = config.codec; let (units_tx, mut units) = mpsc::channel::(4); @@ -765,7 +769,7 @@ async fn read_inprocess( let mut gate = DependencyGate::default(); while let Some(unit) = units.recv().await { let sequence = sequence.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; - gate.submit(&latest, Arc::new(CompleteFrame { source_generation: 0, source_id: 0, codec, sequence, completed_at: unit.completed_at, data: unit.data, _permit: Some(unit.permit) })); + gate.submit(&latest, Arc::new(CompleteFrame { source_generation: source.1, source_id: source.0, codec, sequence, completed_at: unit.completed_at, data: unit.data, _permit: Some(unit.permit) })); if latest.is_closed() { break; } } // Dropping the receiver also unblocks the encoder if it is waiting to send. @@ -832,6 +836,7 @@ async fn read_encoder_with( if allow_inproc && x11_region.is_none() && native_inproc(&display, portal.is_some(), encoder) { let max_kbps = settings.max_kbps().unwrap_or(30_000) as i64; let config = crate::inproc_encoder::Config { + portal: portal.clone(), display: display.clone(), input: size, output: settings.output, fps: u32::from(settings.fps), gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32, target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, @@ -841,7 +846,7 @@ async fn read_encoder_with( log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} source={} pipeline=in-process", display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name_for(settings.codec), crate::video_select::encoder_source_label(selection.encoder_source)); - return read_inprocess(config, latest, sequence, idr, budget).await; + return read_inprocess(config, latest, sequence, idr, budget, (settings.source_id, settings.source_generation)).await; } let source_size = size; let size = format!("{}x{}", size.0, size.1); @@ -1514,6 +1519,18 @@ mod tests { } } + #[test] + fn portal_native_pipeline_only_accepts_vaapi() { + assert!(!native_engine_eligible(Encoder::Cpu, NativeDisplayKind::Portal)); + assert!(!native_engine_eligible(Encoder::Nvenc, NativeDisplayKind::Portal)); + #[cfg(feature = "inproc-av")] + assert_eq!(native_engine_eligible(Encoder::Vaapi, NativeDisplayKind::Portal), + crate::inproc_encoder::selected(Encoder::Vaapi)); + for kind in [NativeDisplayKind::Wayland, NativeDisplayKind::Kms, NativeDisplayKind::None] { + assert!(!native_engine_eligible(Encoder::Vaapi, kind)); + } + } + #[test] fn timing_probe_failure_keeps_modern_args() { let modern = b" -fps_mode mode\n -enc_time_base ratio\n"; diff --git a/src/webrtc.rs b/src/webrtc.rs index 4d9431d..9b6f44c 100644 --- a/src/webrtc.rs +++ b/src/webrtc.rs @@ -1373,8 +1373,9 @@ impl WebRtcSession { if channel_name.contains("CONTROL") { match crate::control_protocol::decode_control(&msg.data) { Ok(crate::control_protocol::ControlMessage::Input(event)) => { + // A ready compositor source remains controllable on its lock screen. if screen.backend == crate::desktop_backend::DesktopBackend::Portal - && (screen.init_error.is_some() || !desktop_ready || *lock_for_message.borrow() != Some(false)) { return; } + && (screen.init_error.is_some() || !desktop_ready) { return; } if !crate::wayland::input_ready() {return;} use crate::control_protocol::ControlInput; // Separate receipt budgets keep motion floods from hiding clicks. diff --git a/tests/portal_dmabuf.c b/tests/portal_dmabuf.c new file mode 100644 index 0000000..8f4954f --- /dev/null +++ b/tests/portal_dmabuf.c @@ -0,0 +1,189 @@ +#define _GNU_SOURCE +#include +#include +#include + +static struct pw_buffer *incoming[8], *returned[32]; +static unsigned input_count, input_next, return_count, loop_depth, updates; +static int requested_dma; +static struct pw_buffer *test_dequeue(struct pw_stream *stream) { + (void)stream; + assert(loop_depth == 1); + return input_next < input_count ? incoming[input_next++] : NULL; +} +static int test_queue(struct pw_stream *stream, struct pw_buffer *buffer) { + (void)stream; + assert(loop_depth == 1 && return_count < 32); + returned[return_count++] = buffer; + return 0; +} +static void test_loop_lock(struct pw_thread_loop *loop) { (void)loop; assert(loop_depth++ == 0); } +static void test_loop_unlock(struct pw_thread_loop *loop) { (void)loop; assert(loop_depth-- == 1); } +static int test_update(struct pw_stream *stream, const struct spa_pod **params, uint32_t count) { + (void)stream; + assert(loop_depth == 1); + if (count == 1) { updates++; return 0; } /* Negotiated buffer requirements. */ + assert(count == 2 || count == 3); + requested_dma = count == 3; + assert(!!spa_pod_find_prop(params[0], NULL, SPA_FORMAT_VIDEO_modifier) == requested_dma); + if (requested_dma) assert(!spa_pod_find_prop(params[1], NULL, SPA_FORMAT_VIDEO_modifier)); + updates++; + return 0; +} +#define pw_stream_dequeue_buffer test_dequeue +#define pw_stream_queue_buffer test_queue +#define pw_stream_update_params test_update +#define pw_thread_loop_lock test_loop_lock +#define pw_thread_loop_unlock test_loop_unlock +#include "../c_engine/portal_engine.c" + +static struct uu_portal *make_portal(void) { + struct uu_portal *p = calloc(1, sizeof(*p)); + assert(p); + pthread_mutex_init(&p->lock, NULL); + pthread_cond_init(&p->changed, NULL); + p->width=2; p->height=2; p->format=SPA_VIDEO_FORMAT_BGRA; p->format_valid=1; + p->frame_length=16; p->frame=malloc(16); assert(p->frame); memset(p->frame, 99, 16); + p->pw_loop=(struct pw_thread_loop *)1; p->stream=(struct pw_stream *)1; + input_count=input_next=return_count=updates=loop_depth=0; + return p; +} +static void close_portal(struct uu_portal *p) { + assert(!loop_depth); + p->stream=NULL; p->pw_loop=NULL; + uu_portal_close(p); +} +static void process(struct uu_portal *p, struct pw_buffer *buffer) { + input_count=1; input_next=0; incoming[0]=buffer; + test_loop_lock(p->pw_loop); stream_process(p); test_loop_unlock(p->pw_loop); +} +static void descriptor_bounds(void) { + struct uu_portal *p=make_portal(); + struct spa_chunk chunk={.offset=8, .size=24, .stride=16}; + struct spa_data data={.type=SPA_DATA_DmaBuf, .fd=5, .mapoffset=4, .maxsize=64, .chunk=&chunk}; + struct spa_buffer buffer={.n_datas=1, .datas=&data}; + UuDmaFrame frame; + assert(describe_dma_buffer(p,&buffer,&frame)==0); + assert(frame.offset==12 && frame.size==68 && frame.stride==16 && frame.fourcc==0x34325241); + chunk.size=23; assert(describe_dma_buffer(p,&buffer,&frame)==10); chunk.size=24; + data.maxsize=31; assert(describe_dma_buffer(p,&buffer,&frame)==14); data.maxsize=64; + data.mapoffset=UINT32_MAX; assert(describe_dma_buffer(p,&buffer,&frame)==14); data.mapoffset=4; + chunk.stride=7; assert(describe_dma_buffer(p,&buffer,&frame)==14); chunk.stride=16; + data.fd=-1; assert(describe_dma_buffer(p,&buffer,&frame)==14); data.fd=5; + data.type=SPA_DATA_MemFd; assert(describe_dma_buffer(p,&buffer,&frame)==14); data.type=SPA_DATA_DmaBuf; + buffer.n_datas=2; assert(describe_dma_buffer(p,&buffer,&frame)==14); buffer.n_datas=1; + p->modifier=1; assert(describe_dma_buffer(p,&buffer,&frame)==19); p->modifier=0; + chunk.flags=SPA_CHUNK_FLAG_CORRUPTED; assert(describe_dma_buffer(p,&buffer,&frame)==10); + chunk.flags=SPA_CHUNK_FLAG_EMPTY; assert(describe_dma_buffer(p,&buffer,&frame)==15); chunk.flags=0; + struct spa_meta_header header={.flags=SPA_META_HEADER_FLAG_CORRUPTED}; + struct spa_meta meta={.type=SPA_META_Header,.size=sizeof(header),.data=&header}; + buffer.n_metas=1; buffer.metas=&meta; assert(describe_dma_buffer(p,&buffer,&frame)==10); + struct spa_meta_region crop={.region=SPA_REGION(1,0,2,2)}; + meta=(struct spa_meta){.type=SPA_META_VideoCrop,.size=sizeof(crop),.data=&crop}; + assert(describe_dma_buffer(p,&buffer,&frame)==14); +#if PW_CHECK_VERSION(1, 2, 0) + struct spa_meta_sync_timeline sync={0}; + meta=(struct spa_meta){.type=SPA_META_SyncTimeline,.size=sizeof(sync),.data=&sync}; + assert(describe_dma_buffer(p,&buffer,&frame)==18); +#endif + close_portal(p); +} +static void ownership_and_reset(void) { + struct uu_portal *p=make_portal(); + p->dma_requested=p->dma_negotiated=p->dma_format=1; + struct spa_chunk chunk={.size=16,.stride=8}; + /* Invalid CPU address: the DMA path must never dereference or copy pixels. */ + struct spa_data data={.type=SPA_DATA_DmaBuf,.fd=5,.maxsize=16,.data=(void *)1,.chunk=&chunk}; + struct spa_buffer spa={.n_datas=1,.datas=&data}; + struct pw_buffer a={.buffer=&spa}, b={.buffer=&spa}, c={.buffer=&spa}; + UuDmaFrame frame; uint64_t revision=0; + process(p,&a); + assert(p->dma_pending==&a && !return_count && p->frame[0]==99); + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==0 && loop_depth==1); + assert(p->dma_current==&a && !p->dma_pending && !return_count); + uu_portal_dmabuf_release(p); assert(!loop_depth && !return_count); + process(p,&b); process(p,&c); + assert(return_count==1 && returned[0]==&b && p->dma_current==&a && p->dma_pending==&c); + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==0); + assert(return_count==2 && returned[1]==&a && p->dma_current==&c); + uu_portal_dmabuf_release(p); + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==12); + uu_portal_dmabuf_release(p); + assert(return_count==2); + uu_portal_reset_frame(p); + assert(return_count==3 && returned[2]==&c && !p->dma_current && !p->dma_pending && !p->sequence); + for (unsigned i=0;i<16;i++) assert(!p->frame[i]); + process(p,&a); + stream_remove_buffer(p,&a); + assert(!p->dma_pending); + process(p,&b); + uu_portal_dmabuf_disable(p); + assert(p->dma_blocked && !requested_dma && !p->dma_pending && !p->dma_current); + assert(return_count==4 && returned[3]==&b); + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==14 && !loop_depth); + close_portal(p); +} +static void negotiation_and_black(void) { + struct uu_portal *p=make_portal(); + UuDmaFrame frame; uint64_t revision=0; + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==10 && !loop_depth); + assert(requested_dma && updates==1); + p->dma_negotiated=1; p->dma_format=1; + p->dma_requested_at=g_get_monotonic_time()-3*G_TIME_SPAN_SECOND; + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==17 && !loop_depth); + struct spa_chunk chunk={.flags=SPA_CHUNK_FLAG_EMPTY}; + struct spa_data data={.type=SPA_DATA_DmaBuf,.fd=5,.chunk=&chunk}; + struct spa_buffer spa={.n_datas=1,.datas=&data}; + struct pw_buffer buffer={.buffer=&spa}; + process(p,&buffer); + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==15 && !loop_depth); + p->dma_neutral=0; p->dma_format=0; + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==16 && !loop_depth); + uu_portal_dmabuf_disable(p); + assert(!requested_dma && updates==2); + close_portal(p); +} +static void initial_dma_transport(void) { + struct uu_portal *p=make_portal(); + uint8_t storage[2048]; + const struct spa_pod *params[3]; + struct spa_pod_builder builder=SPA_POD_BUILDER_INIT(storage,sizeof(storage)); + p->width=p->height=0; + assert(capture_formats(p,&builder,params,0)==2); + assert(!spa_pod_find_prop(params[0],NULL,SPA_FORMAT_VIDEO_modifier)); + builder=(struct spa_pod_builder)SPA_POD_BUILDER_INIT(storage,sizeof(storage)); + assert(capture_formats(p,&builder,params,1)==3); + assert(spa_pod_find_prop(params[0],NULL,SPA_FORMAT_VIDEO_modifier)); + assert(!spa_pod_find_prop(params[1],NULL,SPA_FORMAT_VIDEO_modifier)); + /* Recovery must request DMA before the first buffer: opening in CPU mode and + * switching later drops a compositor's already-negotiated DMA first frame. */ + p->dma_requested=1; + p->dma_bad=18; + assert(wait_for_first_frame(p)==14); /* Reject unsupported startup buffers without a 10 s stall. */ + p->dma_bad=0; + builder=(struct spa_pod_builder)SPA_POD_BUILDER_INIT(storage,sizeof(storage)); + const struct spa_pod *format=spa_format_video_raw_build(&builder,SPA_PARAM_Format, + &SPA_VIDEO_INFO_RAW_INIT(.format=SPA_VIDEO_FORMAT_BGRA,.size=SPA_RECTANGLE(2,2), + .flags=SPA_VIDEO_FLAG_MODIFIER,.modifier=0,.framerate=SPA_FRACTION(60,1))); + test_loop_lock(p->pw_loop); + stream_format(p,SPA_PARAM_Format,format); + test_loop_unlock(p->pw_loop); + struct spa_chunk chunk={.size=16,.stride=8}; + struct spa_data data={.type=SPA_DATA_DmaBuf,.fd=5,.maxsize=16,.data=(void *)1,.chunk=&chunk}; + struct spa_buffer spa={.n_datas=1,.datas=&data}; + struct pw_buffer buffer={.buffer=&spa}; + process(p,&buffer); + assert(wait_for_first_frame(p)==0 && p->dma_pending==&buffer && p->frame[0]==99); + UuDmaFrame frame; uint64_t revision=0; + assert(uu_portal_dmabuf_acquire(p,&frame,&revision)==0); + uu_portal_dmabuf_release(p); + uu_portal_reset_frame(p); + close_portal(p); +} +int main(void) { + descriptor_bounds(); + ownership_and_reset(); + negotiation_and_black(); + initial_dma_transport(); + return 0; +} diff --git a/tests/uinput_keymap.c b/tests/uinput_keymap.c new file mode 100644 index 0000000..29fb687 --- /dev/null +++ b/tests/uinput_keymap.c @@ -0,0 +1,48 @@ +/* Read-only live-keymap regression: cc tests/uinput_keymap.c -o /tmp/uu-keymap-test -lX11 + * Requires DISPLAY (X11 or XWayland). Never injects keys or changes the server keymap. */ +#include +#include +#include "../c_engine/uinput_keymap.c" + +int main(void) { + Display *display=XOpenDisplay(NULL); + assert(display); + XkbDescPtr map=XkbGetMap(display,XkbAllClientInfoMask,XkbUseCoreKbd); + assert(map && XkbGetNames(display,XkbKeyNamesMask,map)==Success); + XkbStateRec state={0}; + XkbStateRec actual; + assert(XkbGetState(display,XkbUseCoreKbd,&actual)==Success); + state.group=actual.group; + uint8_t characters[97]; + for (unsigned i=0;i<95;i++) characters[i]=0x20+i; + characters[95]='\t'; characters[96]='\n'; + UuTextKey plan[97]; + for (unsigned locks=0;locks<3;locks++) { + state.locked_mods=locks==0 ? 0 : locks==1 ? LockMask : Mod2Mask; + assert(text_plan(map,&state,characters,sizeof(characters),plan,97)==97); + for (unsigned i=0;i<97;i++) { + KeySym symbol=NoSymbol; unsigned consumed=0; + unsigned modifiers=XkbBuildCoreState(state.locked_mods | (plan[i].shift ? ShiftMask : 0),state.group); + assert(XkbTranslateKeyCode(map,plan[i].code+8,modifiers,&consumed,&symbol)); + assert(symbol==(characters[i]=='\n' ? XK_Return : characters[i]=='\t' ? XK_Tab : characters[i])); + assert(plan[i].code>=1 && plan[i].code<=248 && plan[i].shift<=248); + } + } + state.locked_mods=0; + const uint8_t mixed[]={'a','b',0xc3,0xa9}; + assert(text_plan(map,&state,mixed,sizeof(mixed),plan,97)==2); + assert(text_plan(map,&state,characters,sizeof(characters),plan,2)==2); + state.base_mods=ControlMask; + assert(text_plan(map,&state,characters,sizeof(characters),plan,97)==0); + state.base_mods=0; state.latched_mods=ShiftMask; + assert(text_plan(map,&state,characters,sizeof(characters),plan,97)==0); + state.latched_mods=0; + XkbKeyNameRec previous=map->names->keys[38]; + memcpy(map->names->keys[38].name,"NONE",4); + assert(text_plan(map,&state,characters,sizeof(characters),plan,97)==0); + map->names->keys[38]=previous; + XkbFreeKeyboard(map,XkbAllComponentsMask,True); + XCloseDisplay(display); + puts("PASS: ASCII, shift/lock modifiers, partial prefixes and keycode validation"); + return 0; +} diff --git a/tests/vaapi_dmabuf.c b/tests/vaapi_dmabuf.c new file mode 100644 index 0000000..cb3a9cc --- /dev/null +++ b/tests/vaapi_dmabuf.c @@ -0,0 +1,67 @@ +/* Hardware regression: cc tests/vaapi_dmabuf.c -o /tmp/vaapi-dma -lgbm -ldl + * Requires a VAAPI render node. stdout is synthetic H.264: red DMA-BUF, then black host upload. + * No desktop capture or account credentials are used. */ +#include +#include +#include +#include +#include +#include +#include "../c_engine/vaapi_engine.c" + +static void assert_color(struct gbm_bo *bo, int write_red) { + uint32_t stride=gbm_bo_get_stride(bo); + int fd=gbm_bo_get_fd(bo); + assert(fd>=0); + size_t size=(size_t)stride*720; + uint8_t *pixels=mmap(NULL,size,PROT_READ | PROT_WRITE,MAP_SHARED,fd,0); + assert(pixels!=MAP_FAILED); + struct dma_buf_sync sync={.flags=DMA_BUF_SYNC_START | DMA_BUF_SYNC_RW}; + assert(ioctl(fd,DMA_BUF_IOCTL_SYNC,&sync)==0); + for (unsigned y=0; y<720; y++) { + for (unsigned x=0; x<1280; x++) { + uint8_t *pixel=pixels+y*stride+x*4; + if (write_red) { pixel[0]=0; pixel[1]=0; pixel[2]=255; pixel[3]=255; } + else assert(pixel[0]==0 && pixel[1]==0 && pixel[2]==255 && pixel[3]==255); + } + } + sync.flags=DMA_BUF_SYNC_END | DMA_BUF_SYNC_RW; + assert(ioctl(fd,DMA_BUF_IOCTL_SYNC,&sync)==0); + munmap(pixels,size); close(fd); +} +static void encode(VaapiEngine *engine) { + const uint8_t *data=NULL; + uint32_t size=0; + assert(vaapi_encode_frame(engine, 1, &data, &size)==0 && data && size); + assert(fwrite(data, 1, size, stdout)==size); +} +int main(void) { + const char *device=getenv("UU_VAAPI_DEVICE"); + if (!device) device="/dev/dri/renderD128"; + int fd=open(device, O_RDWR | O_CLOEXEC); + assert(fd>=0); + struct gbm_device *gbm=gbm_create_device(fd); + assert(gbm); + const uint64_t modifier=0; + struct gbm_bo *bo=gbm_bo_create_with_modifiers2(gbm,1280,720,GBM_FORMAT_ARGB8888,&modifier,1,GBM_BO_USE_RENDERING); + assert(bo && gbm_bo_get_modifier(bo)==0); + assert_color(bo,1); + int dma=gbm_bo_get_fd(bo); + assert(dma>=0); + UuDmaFrame frame={.fd=dma,.fourcc=GBM_FORMAT_ARGB8888,.width=1280,.height=720, + .stride=gbm_bo_get_stride(bo),.size=gbm_bo_get_stride(bo)*720,.modifier=0}; + VaapiConfig config={.width=1280,.height=720,.cap_width=1280,.cap_height=720,.fps=30,.gop=30, + .target_bps=1000000,.max_bps=2000000,.buffer_bits=500000,.device=device}; + VaapiEngine *engine=vaapi_create(&config); + assert(engine && vaapi_import_frame(engine,&frame)==0); + encode(engine); + uint8_t *black=calloc(1280*720,4); + assert(black && vaapi_upload_host_buffer(engine,black,1280*4)==0); + encode(engine); + /* Host fallback must allocate its own surface, never overwrite the leased compositor buffer. */ + assert_color(bo,0); + free(black); + vaapi_destroy(engine); + close(dma); gbm_bo_destroy(bo); gbm_device_destroy(gbm); close(fd); + return 0; +}