From 21c01d2e18d03331cf33b92157336f9548e576e5 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Thu, 22 Oct 2026 06:25:13 +0800 Subject: [PATCH] inproc-av: native VAAPI H.264/HEVC through DRI3 DMA-BUF, VPP and VCN (#80) With --features inproc-av on X11 and UU_VIDEO_ENCODER=vaapi, VAAPI runs in process: a DRI3 DMA-BUF of the root (XShm upload without DRI3) goes through VPP to NV12 and the hardware encoder. SPS/PPS(/VPS) and slice headers are packed by the host because radeonsi (Mesa 26) does not write them. libva/libva-drm are dlopen'ed; release builds do not enable inproc-av. The NVENC path is unchanged. Accepted by the contributor on AMD HawkPoint1 (Mesa 26.2.3), 1920x1200@60, with the official Mac viewer (HEVC, presets, fps, scaling). Intel iHD and 4K are not verified. Thanks @ferstar. --- .github/workflows/ci.yml | 5 +- AGENTS.md | 14 +- build.rs | 12 + c_engine/dri3_capture.c | 242 ++++++++++++ c_engine/dri3_capture.h | 33 ++ c_engine/vaapi_engine.c | 835 +++++++++++++++++++++++++++++++++++++++ c_engine/vaapi_engine.h | 49 +++ src/inproc_encoder.rs | 272 ++++++++++++- src/video.rs | 1 + 9 files changed, 1456 insertions(+), 7 deletions(-) create mode 100644 c_engine/dri3_capture.c create mode 100644 c_engine/dri3_capture.h create mode 100644 c_engine/vaapi_engine.c create mode 100644 c_engine/vaapi_engine.h diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e1e7a0a..6c35f6d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,11 +94,12 @@ jobs: runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - name: Install build dependencies (X11; inproc-av vendors the NVENC headers) + - name: Install build dependencies (X11; inproc-av vendors the NVENC headers; the VAAPI/DRI3 engines need libva/gbm/xcb-dri3 headers, libva itself is dlopen'ed at runtime) run: | sudo apt-get update sudo apt-get install -y build-essential pkg-config \ - libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev + libx11-dev libxext-dev libxtst-dev libxfixes-dev libgtk-3-dev \ + libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage0-dev - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - run: cargo test --locked --features inproc-av diff --git a/AGENTS.md b/AGENTS.md index 5cbcfce..65e055c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,8 +38,11 @@ Dependencies (Ubuntu/Debian): ```sh sudo apt install build-essential pkg-config \ libx11-dev libxext-dev libxtst-dev libxfixes-dev libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev libgtk-3-dev ffmpeg -# --features inproc-av: nothing extra (X11 in-process capture + native NVENC; vendored NVENC headers, -# libcuda/libnvidia-encode are dlopen'ed at runtime; NVENC API 12.1, driver >= 530) +# --features inproc-av: X11 in-process capture + native NVENC/VAAPI. NVENC headers are vendored and +# libcuda/libnvidia-encode dlopen'ed at runtime (NVENC API 12.1, driver >= 530). +# The VAAPI/DRI3 engines compile against +# libva-dev libgbm-dev libx11-xcb-dev libxcb-dri3-dev libxcb-damage-dev; +# libva/libva-drm themselves are dlopen'ed at runtime (no hard libva dependency) # --features viewer-ffmpeg: libavcodec-dev libavformat-dev libavdevice-dev libswscale-dev libclang-dev # (FFmpeg 6.x only. FFmpeg >= 7 fails in ffmpeg-sys-next with "avfft.h file not found") # --features wayland: libwayland-dev libxkbcommon-dev wl-clipboard @@ -276,8 +279,11 @@ started a second full host server instead, so check that a subcommand exists in `UU_VIDEO_ENCODER=vaapi` selects Intel/AMD H.264 VAAPI; `UU_VAAPI_DEVICE` defaults to `/dev/dri/renderD128` (render-node permissions required). No automatic VAAPI selection. -The FFmpeg process pipeline uploads NV12 frames to a VAAPI frame pool (`--features inproc-av` is NVENC only; -VAAPI and CPU encoders always use the process pipeline). +The FFmpeg process pipeline uploads NV12 frames to a VAAPI frame pool. With `--features inproc-av` on X11, VAAPI +H.264/HEVC Main runs in process: a DRI3 DMA-BUF of the root (XShm upload without DRI3) goes through VPP to NV12 +and VCN, with SPS/PPS(/VPS) and slice headers packed by the host as radeonsi (Mesa 26) needs; parameters follow +FFmpeg's hevc_vaapi/h264_vaapi on that driver. Accepted on AMD HawkPoint1 (Mesa 26.2.3) with the official +Mac viewer (presets, fps, scaling); Intel iHD is not verified. VBR uses the viewer bitrate preset and CQ-derived minimum QP, no B-frames, async depth 1, AUD and the existing GOP/VFR timing. Unsupported device/driver/initialization falls back to libx264 with a warning. Accepted on Intel UHD 770 (iHD) with the default FFmpeg process pipeline and the official Mac diff --git a/build.rs b/build.rs index 545ec89..ff75a45 100644 --- a/build.rs +++ b/build.rs @@ -31,7 +31,19 @@ fn main() { .file("c_engine/pam_check.c"); if std::env::var_os("CARGO_FEATURE_INPROC_AV").is_some() { x11_build.file("c_engine/nvenc_engine.c"); + x11_build.file("c_engine/vaapi_engine.c"); + x11_build.file("c_engine/dri3_capture.c"); + for f in ["c_engine/vaapi_engine.c", "c_engine/vaapi_engine.h", "c_engine/dri3_capture.c", "c_engine/dri3_capture.h"] { + println!("cargo:rerun-if-changed={f}"); + } x11_build.include("vendor/ffnvcodec"); + // libva/libva-drm are dlopen'ed at runtime (like NVENC's libcuda); these are + // link-time-only deps of the DRI3/VAAPI engines and must stay out of default builds. + println!("cargo:rustc-link-lib=gbm"); + println!("cargo:rustc-link-lib=X11-xcb"); + println!("cargo:rustc-link-lib=xcb"); + println!("cargo:rustc-link-lib=xcb-dri3"); + println!("cargo:rustc-link-lib=xcb-damage"); } x11_build.compile("x11_engine"); diff --git a/c_engine/dri3_capture.c b/c_engine/dri3_capture.c new file mode 100644 index 0000000..5efbd2b --- /dev/null +++ b/c_engine/dri3_capture.c @@ -0,0 +1,242 @@ +#include "dri3_capture.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct Dri3Capture { + Display *dpy; + xcb_connection_t *conn; + xcb_screen_t *screen; + uint32_t width; + uint32_t height; + + int drm_fd; + struct gbm_device *gbm; + struct gbm_bo *bo; + int bo_fd; + uint32_t stride; + + xcb_pixmap_t pix; + xcb_gcontext_t gc; + xcb_damage_damage_t damage; + uint8_t damage_event_code; +}; + +Dri3Capture* dri3_capture_create(const char* display_name, uint32_t width, uint32_t height) { + Display *dpy = XOpenDisplay(display_name); + if (!dpy) { + fprintf(stderr, "[dri3_capture] failed to open display %s\n", display_name ? display_name : "default"); + return NULL; + } + + // Events are read with xcb_poll_for_event below, so XCB, not Xlib, must own the queue + XSetEventQueueOwner(dpy, XCBOwnsEventQueue); + xcb_connection_t *conn = XGetXCBConnection(dpy); + const xcb_setup_t *setup = xcb_get_setup(conn); + xcb_screen_t *screen = xcb_setup_roots_iterator(setup).data; + if (!screen) { + XCloseDisplay(dpy); + return NULL; + } + + // Check the extension first: a DRI3 request on a server without it breaks the connection, + // and the Xlib cleanup below then exits the process. + const xcb_query_extension_reply_t *dri3_ext = xcb_get_extension_data(conn, &xcb_dri3_id); + if (!dri3_ext || !dri3_ext->present) { + fprintf(stderr, "[dri3_capture] DRI3 extension not supported on display\n"); + XCloseDisplay(dpy); + return NULL; + } + // Query DRI3 + xcb_dri3_query_version_cookie_t qvc = xcb_dri3_query_version(conn, 1, 2); + xcb_dri3_query_version_reply_t *qvr = xcb_dri3_query_version_reply(conn, qvc, NULL); + if (!qvr) { + fprintf(stderr, "[dri3_capture] DRI3 extension not supported on display\n"); + XCloseDisplay(dpy); + return NULL; + } + free(qvr); + + // Open DRM device via DRI3 + xcb_dri3_open_cookie_t oc = xcb_dri3_open(conn, screen->root, 0); + xcb_dri3_open_reply_t *or = xcb_dri3_open_reply(conn, oc, NULL); + if (!or) { + fprintf(stderr, "[dri3_capture] xcb_dri3_open failed\n"); + XCloseDisplay(dpy); + return NULL; + } + int drm_fd = xcb_dri3_open_reply_fds(conn, or)[0]; + free(or); + + uint32_t w = width > 0 ? width : screen->width_in_pixels; + uint32_t h = height > 0 ? height : screen->height_in_pixels; + + struct gbm_device *gbm = gbm_create_device(drm_fd); + if (!gbm) { + fprintf(stderr, "[dri3_capture] gbm_create_device failed\n"); + close(drm_fd); + XCloseDisplay(dpy); + return NULL; + } + + struct gbm_bo *bo = gbm_bo_create(gbm, w, h, GBM_FORMAT_XRGB8888, GBM_BO_USE_RENDERING | GBM_BO_USE_LINEAR); + if (!bo) { + fprintf(stderr, "[dri3_capture] gbm_bo_create failed\n"); + gbm_device_destroy(gbm); + close(drm_fd); + XCloseDisplay(dpy); + return NULL; + } + + int bo_fd = gbm_bo_get_fd(bo); + uint32_t stride = gbm_bo_get_stride(bo); + + xcb_pixmap_t pix = xcb_generate_id(conn); + xcb_generic_error_t *err = NULL; + // XCB takes ownership of the fd it sends and closes it after transmission; only a + // duplicate goes to X11, bo_fd stays ours for the VAAPI import and teardown. + int send_fd = fcntl(bo_fd, F_DUPFD_CLOEXEC, 3); + if (send_fd < 0) { + close(bo_fd); + gbm_bo_destroy(bo); + gbm_device_destroy(gbm); + close(drm_fd); + XCloseDisplay(dpy); + return NULL; + } + xcb_void_cookie_t pc = xcb_dri3_pixmap_from_buffer_checked( + conn, pix, screen->root, stride * h, w, h, stride, screen->root_depth, 32, send_fd + ); + err = xcb_request_check(conn, pc); + if (err) { + fprintf(stderr, "[dri3_capture] xcb_dri3_pixmap_from_buffer failed: %d\n", err->error_code); + free(err); + close(bo_fd); + gbm_bo_destroy(bo); + gbm_device_destroy(gbm); + close(drm_fd); + XCloseDisplay(dpy); + return NULL; + } + + xcb_gcontext_t gc = xcb_generate_id(conn); + // The default ClipByChildren copies only the root background (solid black); XShmGetImage on the root + // includes child windows, so the blit must too. + uint32_t gc_mode = XCB_SUBWINDOW_MODE_INCLUDE_INFERIORS; + xcb_create_gc(conn, gc, pix, XCB_GC_SUBWINDOW_MODE, &gc_mode); + + // Setup XDamage on root window + const xcb_query_extension_reply_t *ext = xcb_get_extension_data(conn, &xcb_damage_id); + uint8_t damage_event_code = 0; + xcb_damage_damage_t damage = 0; + if (ext && ext->present) { + damage_event_code = ext->first_event; + xcb_damage_query_version(conn, 1, 1); + damage = xcb_generate_id(conn); + xcb_damage_create(conn, damage, screen->root, XCB_DAMAGE_REPORT_LEVEL_NON_EMPTY); + } + + Dri3Capture* cap = (Dri3Capture*)calloc(1, sizeof(Dri3Capture)); + cap->dpy = dpy; + cap->conn = conn; + cap->screen = screen; + cap->width = w; + cap->height = h; + cap->drm_fd = drm_fd; + cap->gbm = gbm; + cap->bo = bo; + cap->bo_fd = bo_fd; + cap->stride = stride; + cap->pix = pix; + cap->gc = gc; + cap->damage = damage; + cap->damage_event_code = damage_event_code; + + fprintf(stderr, "[dri3_capture] Initialized: %ux%u (stride: %u, dma_fd: %d) Pure GPU Blit ready!\n", + w, h, stride, bo_fd); + return cap; +} + +int dri3_capture_blit(Dri3Capture* cap) { + if (!cap) return -1; + xcb_copy_area(cap->conn, cap->screen->root, cap->pix, cap->gc, 0, 0, 0, 0, cap->width, cap->height); + // Round trip: the server has executed (and flushed to the GPU) the copy before VAAPI reads the BO; + // xcb_flush alone only sends the request. + xcb_get_input_focus_reply_t *r = xcb_get_input_focus_reply(cap->conn, xcb_get_input_focus(cap->conn), NULL); + if (!r) return -1; + free(r); + return 0; +} + +int dri3_capture_wait_damage(Dri3Capture* cap, int timeout_ms) { + if (!cap) return -1; + + // Check queued events + xcb_generic_event_t *ev = NULL; + int damaged = 0; + while ((ev = xcb_poll_for_event(cap->conn)) != NULL) { + if ((ev->response_type & 0x7f) == (cap->damage_event_code + XCB_DAMAGE_NOTIFY)) { + damaged = 1; + } + free(ev); + } + if (damaged) { + xcb_damage_subtract(cap->conn, cap->damage, XCB_NONE, XCB_NONE); + xcb_flush(cap->conn); + return 1; + } + + int fd = xcb_get_file_descriptor(cap->conn); + struct pollfd pfd = { .fd = fd, .events = POLLIN, .revents = 0 }; + int ret = poll(&pfd, 1, timeout_ms); + if (ret > 0 && (pfd.revents & POLLIN)) { + // Readable but not yet a whole event: polling again would return at once and spin a core + ev = xcb_poll_for_event(cap->conn); + if (!ev) ev = xcb_wait_for_event(cap->conn); + if (!ev) return -1; + do { + if ((ev->response_type & 0x7f) == (cap->damage_event_code + XCB_DAMAGE_NOTIFY)) { + damaged = 1; + } + free(ev); + } while ((ev = xcb_poll_for_event(cap->conn)) != NULL); + if (damaged) { + xcb_damage_subtract(cap->conn, cap->damage, XCB_NONE, XCB_NONE); + xcb_flush(cap->conn); + return 1; + } + } + return ret < 0 ? -1 : 0; +} + +int dri3_capture_get_dma_fd(Dri3Capture* cap) { + return cap ? cap->bo_fd : -1; +} + +uint32_t dri3_capture_get_stride(Dri3Capture* cap) { + return cap ? cap->stride : 0; +} + +void dri3_capture_destroy(Dri3Capture* cap) { + if (!cap) return; + + if (cap->damage) xcb_damage_destroy(cap->conn, cap->damage); + if (cap->gc) xcb_free_gc(cap->conn, cap->gc); + if (cap->pix) xcb_free_pixmap(cap->conn, cap->pix); + if (cap->bo_fd >= 0) close(cap->bo_fd); + if (cap->bo) gbm_bo_destroy(cap->bo); + if (cap->gbm) gbm_device_destroy(cap->gbm); + if (cap->drm_fd >= 0) close(cap->drm_fd); + if (cap->dpy) XCloseDisplay(cap->dpy); + + free(cap); +} diff --git a/c_engine/dri3_capture.h b/c_engine/dri3_capture.h new file mode 100644 index 0000000..0b7a7c5 --- /dev/null +++ b/c_engine/dri3_capture.h @@ -0,0 +1,33 @@ +#ifndef UU_DRI3_CAPTURE_H +#define UU_DRI3_CAPTURE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct Dri3Capture Dri3Capture; + +Dri3Capture* dri3_capture_create(const char* display_name, uint32_t width, uint32_t height); + +// Trigger GPU blit from Root Window to the DRI3 DMA-BUF Pixmap. +// Returns 0 on success, negative on error. +int dri3_capture_blit(Dri3Capture* cap); + +// Wait for next XDamage event or timeout (ms). +// Returns 1 if damaged, 0 if timeout, negative on error. +int dri3_capture_wait_damage(Dri3Capture* cap, int timeout_ms); + +// Get the exported DMA-BUF file descriptor and stride +int dri3_capture_get_dma_fd(Dri3Capture* cap); +uint32_t dri3_capture_get_stride(Dri3Capture* cap); + +void dri3_capture_destroy(Dri3Capture* cap); + +#ifdef __cplusplus +} +#endif + +#endif // UU_DRI3_CAPTURE_H diff --git a/c_engine/vaapi_engine.c b/c_engine/vaapi_engine.c new file mode 100644 index 0000000..c06ac09 --- /dev/null +++ b/c_engine/vaapi_engine.c @@ -0,0 +1,835 @@ +#include "vaapi_engine.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Runtime-loaded libva, like the NVENC engine's libcuda/libnvidia-encode: an inproc-av build +// must not add a hard DT_NEEDED on libva for users who never touch VAAPI. +// Signatures come from the installed headers via __typeof__ so they cannot drift. +static struct { + void *libva, *libva_drm; + __typeof__(vaGetDisplayDRM) *vaGetDisplayDRM; + __typeof__(vaInitialize) *vaInitialize; + __typeof__(vaTerminate) *vaTerminate; + __typeof__(vaErrorStr) *vaErrorStr; + __typeof__(vaCreateSurfaces) *vaCreateSurfaces; + __typeof__(vaDestroySurfaces) *vaDestroySurfaces; + __typeof__(vaCreateConfig) *vaCreateConfig; + __typeof__(vaDestroyConfig) *vaDestroyConfig; + __typeof__(vaCreateContext) *vaCreateContext; + __typeof__(vaDestroyContext) *vaDestroyContext; + __typeof__(vaCreateBuffer) *vaCreateBuffer; + __typeof__(vaDestroyBuffer) *vaDestroyBuffer; + __typeof__(vaMapBuffer) *vaMapBuffer; + __typeof__(vaUnmapBuffer) *vaUnmapBuffer; + __typeof__(vaCreateImage) *vaCreateImage; + __typeof__(vaDestroyImage) *vaDestroyImage; + __typeof__(vaPutImage) *vaPutImage; + __typeof__(vaBeginPicture) *vaBeginPicture; + __typeof__(vaRenderPicture) *vaRenderPicture; + __typeof__(vaEndPicture) *vaEndPicture; + __typeof__(vaSyncSurface) *vaSyncSurface; +} RT; + +static int vaapi_load_runtime(void) { + if (RT.libva) return 0; + RT.libva = dlopen("libva.so.2", RTLD_NOW | RTLD_GLOBAL); + RT.libva_drm = dlopen("libva-drm.so.2", RTLD_NOW); + if (!RT.libva || !RT.libva_drm) { + fprintf(stderr, "[vaapi] dlopen libva/libva-drm failed\n"); + return -1; + } + // libva-drm depends on libva, so one lookup covers both libraries + #define VA_LOAD(sym) do { \ + *(void**)&RT.sym = dlsym(RT.libva_drm, #sym) ? dlsym(RT.libva_drm, #sym) : dlsym(RT.libva, #sym); \ + if (!RT.sym) { dlclose(RT.libva_drm); dlclose(RT.libva); RT.libva = RT.libva_drm = NULL; return -1; } \ + } while (0) + VA_LOAD(vaGetDisplayDRM); + VA_LOAD(vaInitialize); + VA_LOAD(vaTerminate); + VA_LOAD(vaErrorStr); + VA_LOAD(vaCreateSurfaces); + VA_LOAD(vaDestroySurfaces); + VA_LOAD(vaCreateConfig); + VA_LOAD(vaDestroyConfig); + VA_LOAD(vaCreateContext); + VA_LOAD(vaDestroyContext); + VA_LOAD(vaCreateBuffer); + VA_LOAD(vaDestroyBuffer); + VA_LOAD(vaMapBuffer); + VA_LOAD(vaUnmapBuffer); + VA_LOAD(vaCreateImage); + VA_LOAD(vaDestroyImage); + VA_LOAD(vaPutImage); + VA_LOAD(vaBeginPicture); + VA_LOAD(vaRenderPicture); + VA_LOAD(vaEndPicture); + VA_LOAD(vaSyncSurface); + #undef VA_LOAD + return 0; +} + +struct VaapiEngine { + int drm_fd; + VADisplay va_dpy; + uint32_t cap_width; // capture (RGB input) size + uint32_t cap_height; + uint32_t width; // encode (output) size + uint32_t height; + uint32_t fps; + uint32_t gop; + int64_t target_bps; + int64_t max_bps; + int64_t buffer_bits; + int is_hevc; + uint32_t frame_count; + // frames since the last IDR (periodic GOP), the alternating idr_pic_id, and an IDR owed after a lost frame + uint32_t since_idr; + uint32_t idr_pic_id; + int idr_pending; + // frame_num / POC bookkeeping since the last IDR; H264 state the driver does not derive itself + uint32_t frame_num; + uint32_t rc_mode; + + // DMA-BUF backing + int dma_fd; + uint32_t stride; + + // Surfaces & Contexts + VASurfaceID in_rgb_surf; + VASurfaceID nv12_surf; + VASurfaceID recon_surfs[2]; + + VAConfigID vpp_cfg; + VAContextID vpp_ctx; + VABufferID vpp_buf; + VARectangle src_rect, dst_rect; + + VAConfigID enc_cfg; + VAContextID enc_ctx; + VABufferID coded_buf; + + uint8_t *output_staging; + uint32_t output_staging_cap; + // Host upload path: a VAImage the CPU writes, then vaPutImage into in_rgb_surf + VAImage upload_img; + int upload_ready; +}; + +// The VPP pipeline buffer points at the engine-owned rects, so VPP scales cap -> output +static int vaapi_make_vpp_buffer(VaapiEngine* engine) { + VAProcPipelineParameterBuffer vpp_param; + memset(&vpp_param, 0, sizeof(vpp_param)); + vpp_param.surface = engine->in_rgb_surf; + vpp_param.surface_region = &engine->src_rect; + vpp_param.output_region = &engine->dst_rect; + return RT.vaCreateBuffer(engine->va_dpy, engine->vpp_ctx, VAProcPipelineParameterBufferType, + sizeof(vpp_param), 1, &vpp_param, &engine->vpp_buf) == VA_STATUS_SUCCESS ? 0 : -1; +} + +VaapiEngine* vaapi_create(const VaapiConfig* config) { + if (!config || config->width == 0 || config->height == 0 || config->cap_width == 0 || config->cap_height == 0) return NULL; + + if (vaapi_load_runtime() != 0) return NULL; + + VaapiEngine* engine = (VaapiEngine*)calloc(1, sizeof(VaapiEngine)); + if (!engine) return NULL; + + engine->cap_width = config->cap_width; + engine->cap_height = config->cap_height; + engine->width = config->width; + engine->height = config->height; + engine->fps = config->fps > 0 ? config->fps : 60; + engine->gop = config->gop > 0 ? config->gop : 30; + engine->target_bps = config->target_bps > 0 ? config->target_bps : 6000000; + engine->max_bps = config->max_bps > engine->target_bps ? config->max_bps : engine->target_bps; + engine->buffer_bits = config->buffer_bits > 0 ? config->buffer_bits : engine->max_bps; + engine->is_hevc = config->is_hevc; + engine->dma_fd = -1; + // calloc leaves surface/context IDs at 0, which is a valid VA ID; mark them unset. + engine->in_rgb_surf = VA_INVALID_ID; + engine->nv12_surf = VA_INVALID_ID; + engine->recon_surfs[0] = VA_INVALID_ID; + engine->recon_surfs[1] = VA_INVALID_ID; + engine->vpp_cfg = VA_INVALID_ID; + engine->vpp_ctx = VA_INVALID_ID; + engine->vpp_buf = VA_INVALID_ID; + engine->enc_cfg = VA_INVALID_ID; + engine->enc_ctx = VA_INVALID_ID; + engine->coded_buf = VA_INVALID_ID; + engine->src_rect.width = config->cap_width; + engine->src_rect.height = config->cap_height; + engine->dst_rect.width = config->width; + engine->dst_rect.height = config->height; + + engine->drm_fd = open(config->device ? config->device : "/dev/dri/renderD128", O_RDWR); + if (engine->drm_fd < 0) { + fprintf(stderr, "[vaapi] failed to open %s\n", config->device ? config->device : "/dev/dri/renderD128"); + free(engine); + return NULL; + } + + engine->va_dpy = RT.vaGetDisplayDRM(engine->drm_fd); + if (!engine->va_dpy) { + close(engine->drm_fd); + free(engine); + return NULL; + } + + int maj, min; + VAStatus st = RT.vaInitialize(engine->va_dpy, &maj, &min); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[vaapi] vaInitialize failed: %s\n", RT.vaErrorStr(st)); + close(engine->drm_fd); + free(engine); + return NULL; + } + + // 1. NV12 intermediate + two Recon surfaces at the encode size (ping-pong: a P frame must + // not reference the surface its own reconstruction is being written into) + st = RT.vaCreateSurfaces(engine->va_dpy, VA_RT_FORMAT_YUV420, engine->width, engine->height, &engine->nv12_surf, 1, NULL, 0); + if (st != VA_STATUS_SUCCESS) { + vaapi_destroy(engine); + return NULL; + } + st = RT.vaCreateSurfaces(engine->va_dpy, VA_RT_FORMAT_YUV420, engine->width, engine->height, engine->recon_surfs, 2, NULL, 0); + if (st != VA_STATUS_SUCCESS) { + vaapi_destroy(engine); + return NULL; + } + + // 2. Setup VPP (Video Processing Pipeline for Hardware RGB->NV12 with scaling) + uint32_t vpp_w = engine->cap_width > engine->width ? engine->cap_width : engine->width; + uint32_t vpp_h = engine->cap_height > engine->height ? engine->cap_height : engine->height; + st = RT.vaCreateConfig(engine->va_dpy, VAProfileNone, VAEntrypointVideoProc, NULL, 0, &engine->vpp_cfg); + if (st != VA_STATUS_SUCCESS) { + vaapi_destroy(engine); + return NULL; + } + st = RT.vaCreateContext(engine->va_dpy, engine->vpp_cfg, vpp_w, vpp_h, 0, NULL, 0, &engine->vpp_ctx); + if (st != VA_STATUS_SUCCESS) { + vaapi_destroy(engine); + return NULL; + } + + // 3. Setup H.264 / HEVC EncSlice + VAProfile profile = engine->is_hevc ? VAProfileHEVCMain : VAProfileH264Main; + VAConfigAttrib enc_attrib[3]; + enc_attrib[0].type = VAConfigAttribRTFormat; + enc_attrib[0].value = VA_RT_FORMAT_YUV420; + enc_attrib[1].type = VAConfigAttribRateControl; + // The engine writes SPS, PPS and slice headers itself (see BitWriter) + enc_attrib[2].type = VAConfigAttribEncPackedHeaders; + enc_attrib[2].value = VA_ENC_PACKED_HEADER_SEQUENCE | VA_ENC_PACKED_HEADER_SLICE; + // VBR like the FFmpeg VAAPI pipeline (target + maxrate), then CBR, then CQP + enc_attrib[1].value = VA_RC_VBR; + engine->rc_mode = VA_RC_VBR; + st = RT.vaCreateConfig(engine->va_dpy, profile, VAEntrypointEncSlice, enc_attrib, 3, &engine->enc_cfg); + if (st != VA_STATUS_SUCCESS) { + enc_attrib[1].value = VA_RC_CBR; + engine->rc_mode = VA_RC_CBR; + st = RT.vaCreateConfig(engine->va_dpy, profile, VAEntrypointEncSlice, enc_attrib, 3, &engine->enc_cfg); + } + if (st != VA_STATUS_SUCCESS) { + // Fallback to CQP if CBR unsupported + enc_attrib[1].value = VA_RC_CQP; + st = RT.vaCreateConfig(engine->va_dpy, profile, VAEntrypointEncSlice, enc_attrib, 3, &engine->enc_cfg); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[vaapi] vaCreateConfig EncSlice failed: %s\n", RT.vaErrorStr(st)); + vaapi_destroy(engine); + return NULL; + } + engine->rc_mode = VA_RC_CQP; + } + + st = RT.vaCreateContext(engine->va_dpy, engine->enc_cfg, engine->width, engine->height, VA_PROGRESSIVE, engine->recon_surfs, 2, &engine->enc_ctx); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[vaapi] vaCreateContext EncSlice failed: %s\n", RT.vaErrorStr(st)); + vaapi_destroy(engine); + return NULL; + } + + // Coded bitstream buffer: generous, a 20M-preset intra frame can burst past 4MB + st = RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncCodedBufferType, 16 * 1024 * 1024, 1, NULL, &engine->coded_buf); + if (st != VA_STATUS_SUCCESS) { + vaapi_destroy(engine); + return NULL; + } + + return engine; +} + +int vaapi_import_dma_buffer(VaapiEngine* engine, int dma_fd, uint32_t stride) { + if (!engine || dma_fd < 0) return -1; + + if (engine->in_rgb_surf != VA_INVALID_ID) { + RT.vaDestroySurfaces(engine->va_dpy, &engine->in_rgb_surf, 1); + engine->in_rgb_surf = VA_INVALID_ID; + } + if (engine->vpp_buf != VA_INVALID_ID) { + RT.vaDestroyBuffer(engine->va_dpy, engine->vpp_buf); + engine->vpp_buf = VA_INVALID_ID; + } + + engine->dma_fd = dma_fd; + engine->stride = stride; + + VADRMPRIMESurfaceDescriptor prime_desc; + memset(&prime_desc, 0, sizeof(prime_desc)); + prime_desc.fourcc = VA_FOURCC_BGRX; + prime_desc.width = engine->cap_width; + prime_desc.height = engine->cap_height; + prime_desc.num_objects = 1; + prime_desc.objects[0].fd = dma_fd; + prime_desc.objects[0].size = stride * engine->cap_height; + prime_desc.objects[0].drm_format_modifier = 0; + prime_desc.num_layers = 1; + prime_desc.layers[0].drm_format = 0x34325258; // DRM_FORMAT_XRGB8888 + prime_desc.layers[0].num_planes = 1; + prime_desc.layers[0].pitch[0] = stride; + + VASurfaceAttrib attribs[2]; + attribs[0].type = VASurfaceAttribMemoryType; + attribs[0].flags = VA_SURFACE_ATTRIB_SETTABLE; + attribs[0].value.type = VAGenericValueTypeInteger; + attribs[0].value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2; + attribs[1].type = VASurfaceAttribExternalBufferDescriptor; + attribs[1].flags = VA_SURFACE_ATTRIB_SETTABLE; + attribs[1].value.type = VAGenericValueTypePointer; + attribs[1].value.value.p = &prime_desc; + + VAStatus st = RT.vaCreateSurfaces(engine->va_dpy, VA_RT_FORMAT_RGB32, engine->cap_width, engine->cap_height, &engine->in_rgb_surf, 1, attribs, 2); + if (st != VA_STATUS_SUCCESS) { + fprintf(stderr, "[vaapi] vaCreateSurfaces DRM_PRIME_2 failed: %s\n", RT.vaErrorStr(st)); + return -1; + } + + if (vaapi_make_vpp_buffer(engine) != 0) { + fprintf(stderr, "[vaapi] vaCreateBuffer VPP failed\n"); + return -2; + } + + return 0; +} + +int vaapi_upload_host_buffer(VaapiEngine* engine, const uint8_t* host_ptr, uint32_t pitch) { + if (!engine || !host_ptr) return -1; + + if (engine->in_rgb_surf == VA_INVALID_ID) { + VAStatus st = RT.vaCreateSurfaces(engine->va_dpy, VA_RT_FORMAT_RGB32, engine->cap_width, engine->cap_height, &engine->in_rgb_surf, 1, NULL, 0); + if (st != VA_STATUS_SUCCESS) return -2; + if (vaapi_make_vpp_buffer(engine) != 0) return -3; + } + + if (!engine->upload_ready) { + VAImageFormat fmt; + memset(&fmt, 0, sizeof(fmt)); + fmt.fourcc = VA_FOURCC_BGRX; + fmt.byte_order = VA_LSB_FIRST; + fmt.bits_per_pixel = 32; + fmt.depth = 24; + fmt.red_mask = 0x00ff0000; + fmt.green_mask = 0x0000ff00; + fmt.blue_mask = 0x000000ff; + // vaDeriveImage on the RGB32 surface is refused by some drivers + VAStatus st = RT.vaCreateImage(engine->va_dpy, &fmt, engine->cap_width, engine->cap_height, &engine->upload_img); + if (st != VA_STATUS_SUCCESS) return -4; + engine->upload_ready = 1; + } + + void* p = NULL; + VAStatus st = RT.vaMapBuffer(engine->va_dpy, engine->upload_img.buf, &p); + if (st != VA_STATUS_SUCCESS || !p) return -5; + uint32_t copy_bytes = engine->cap_width * 4; + for (uint32_t y = 0; y < engine->cap_height; y++) { + memcpy((uint8_t*)p + y * engine->upload_img.pitches[0], host_ptr + y * pitch, copy_bytes); + } + RT.vaUnmapBuffer(engine->va_dpy, engine->upload_img.buf); + + st = RT.vaPutImage(engine->va_dpy, engine->in_rgb_surf, engine->upload_img.image_id, + 0, 0, engine->cap_width, engine->cap_height, 0, 0, engine->cap_width, engine->cap_height); + if (st != VA_STATUS_SUCCESS) return -6; + return 0; +} + +// Sequence-level misc parameter. AMD's Mesa driver feeds the VCN ring message from these; +// a CBR config that never gets its rate-control/HRD parameters submits garbage and the +// encode job wedges the ring until a GPU reset. +// Packed H.264 / HEVC headers. Current radeonsi (Mesa 26) writes neither SPS/PPS nor the NAL/slice header from the +// parameter buffers alone (output began 00 00 00 01 00 ..., no SPS), so the host writes them like FFmpeg does. +// Every field here must match the sequence/picture/slice parameters submitted with it. +typedef struct { uint8_t buf[128]; uint32_t bits; } BitWriter; + +static void bw_put(BitWriter* w, uint32_t value, int n) { + for (int i = n - 1; i >= 0; i--, w->bits++) { + if (w->bits / 8 >= sizeof(w->buf)) return; + if ((value >> i) & 1) w->buf[w->bits / 8] |= 0x80 >> (w->bits % 8); + } +} +static void bw_ue(BitWriter* w, uint32_t value) { + int len = 0; + for (uint32_t v = value + 1; v > 1; v >>= 1) len++; + bw_put(w, 0, len); + bw_put(w, value + 1, len + 1); +} +static void bw_se(BitWriter* w, int32_t value) { bw_ue(w, value > 0 ? 2 * (uint32_t)value - 1 : 2 * (uint32_t)(-value)); } +static void bw_trailing(BitWriter* w) { bw_put(w, 1, 1); if (w->bits % 8) bw_put(w, 0, 8 - w->bits % 8); } +// Start code + NAL header; the payload follows unescaped, emulation prevention is applied by bw_escape. +static void bw_nal(BitWriter* w, int ref_idc, int type) { bw_put(w, 1, 32); bw_put(w, (ref_idc << 5) | type, 8); } + +// Insert emulation prevention bytes after the start code + NAL header (byte 5 on). Returns the byte length. +static uint32_t bw_escape(const BitWriter* w, uint8_t* out, uint32_t cap) { + uint32_t len = (w->bits + 7) / 8, n = 0, zeros = 0; + for (uint32_t i = 0; i < len && n < cap; i++) { + if (i >= 5 && zeros >= 2 && w->buf[i] <= 3) { out[n++] = 3; zeros = 0; if (n >= cap) break; } + zeros = w->buf[i] == 0 && i >= 5 ? zeros + 1 : 0; + out[n++] = w->buf[i]; + } + return n; +} + +// Each NAL is escaped on its own, so the next NAL's start code stays intact. +static int vaapi_add_packed(VaapiEngine* engine, VAEncPackedHeaderType type, const BitWriter* nals, int count, + VABufferID* ids, int* n) { + uint8_t data[320]; + uint32_t bytes = 0; + for (int i = 0; i < count; i++) bytes += bw_escape(&nals[i], data + bytes, sizeof(data) - bytes); + VAEncPackedHeaderParameterBuffer param = { .type = type, .bit_length = bytes * 8, .has_emulation_bytes = 1 }; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncPackedHeaderParameterBufferType, sizeof(param), 1, ¶m, &ids[*n]) != VA_STATUS_SUCCESS) + return -1; + (*n)++; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncPackedHeaderDataBufferType, bytes, 1, data, &ids[*n]) != VA_STATUS_SUCCESS) + return -1; + (*n)++; + return 0; +} + +static VABufferID vaapi_create_misc_buffer(VaapiEngine* engine, VAEncMiscParameterType type, const void* param, uint32_t param_size) { + uint8_t buf[sizeof(VAEncMiscParameterBuffer) + 64]; + VAEncMiscParameterBuffer* misc = (VAEncMiscParameterBuffer*)buf; + misc->type = type; + memcpy(misc->data, param, param_size); + VABufferID id = VA_INVALID_ID; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncMiscParameterBufferType, + sizeof(VAEncMiscParameterBuffer) + param_size, 1, buf, &id) != VA_STATUS_SUCCESS) { + return VA_INVALID_ID; + } + return id; +} + +// Buffers for one encoded picture; every created ID is destroyed after the picture, success or not. +typedef struct { + VABufferID seq, misc[3], pic, slice, packed[6]; + int misc_n, packed_n; +} EncBuffers; + +static int vaapi_h264_params(VaapiEngine* engine, int is_idr, VASurfaceID curr_recon, VASurfaceID prev_recon, EncBuffers* b) { + uint32_t mb_w = (engine->width + 15) / 16; + uint32_t mb_h = (engine->height + 15) / 16; + uint32_t frame_num = is_idr ? 0 : engine->frame_num; + uint32_t crop_right = (mb_w * 16 - engine->width) / 2, crop_bottom = (mb_h * 16 - engine->height) / 2; + if (is_idr) { + VAEncSequenceParameterBufferH264 seq_p; + memset(&seq_p, 0, sizeof(seq_p)); + seq_p.seq_parameter_set_id = 0; + seq_p.level_idc = 51; + seq_p.intra_period = engine->gop; + seq_p.intra_idr_period = engine->gop; + seq_p.ip_period = 1; + seq_p.bits_per_second = (uint32_t)engine->max_bps; + seq_p.max_num_ref_frames = 1; + seq_p.picture_width_in_mbs = mb_w; + seq_p.picture_height_in_mbs = mb_h; + seq_p.seq_fields.bits.direct_8x8_inference_flag = 1; + seq_p.seq_fields.bits.frame_mbs_only_flag = 1; + seq_p.seq_fields.bits.pic_order_cnt_type = 0; + seq_p.frame_cropping_flag = crop_right || crop_bottom; + seq_p.frame_crop_right_offset = crop_right; + seq_p.frame_crop_bottom_offset = crop_bottom; + // An IDR without its SPS is undecodable: fail rather than submit pic + slice alone + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncSequenceParameterBufferType, sizeof(seq_p), 1, &seq_p, &b->seq) != VA_STATUS_SUCCESS) + return -1; + + // SPS: Main, level 5.1, frame_num and POC lsb 4 bits (both wrap & 15 above), one reference + BitWriter sps = {0}; + bw_nal(&sps, 3, 7); + bw_put(&sps, 77, 8); bw_put(&sps, 0x40, 8); bw_put(&sps, 51, 8); // constraint_set1: Main + bw_ue(&sps, 0); bw_ue(&sps, 0); bw_ue(&sps, 0); bw_ue(&sps, 0); // sps id, log2_max_frame_num-4, poc type 0, log2_max_poc_lsb-4 + bw_ue(&sps, 1); bw_put(&sps, 0, 1); // max_num_ref_frames, gaps + bw_ue(&sps, mb_w - 1); bw_ue(&sps, mb_h - 1); + bw_put(&sps, 1, 1); bw_put(&sps, 1, 1); // frame_mbs_only, direct_8x8_inference + bw_put(&sps, seq_p.frame_cropping_flag, 1); + if (seq_p.frame_cropping_flag) { bw_ue(&sps, 0); bw_ue(&sps, crop_right); bw_ue(&sps, 0); bw_ue(&sps, crop_bottom); } + bw_put(&sps, 0, 1); // no VUI + bw_trailing(&sps); + // PPS: CABAC, one L0 reference, QP 26, default deblocking + BitWriter pps = {0}; + bw_nal(&pps, 3, 8); + bw_ue(&pps, 0); bw_ue(&pps, 0); bw_put(&pps, 1, 1); bw_put(&pps, 0, 1); + bw_ue(&pps, 0); bw_ue(&pps, 0); bw_ue(&pps, 0); // slice groups, num_ref_idx l0/l1 - 1 + bw_put(&pps, 0, 1); bw_put(&pps, 0, 2); // weighted pred / bipred + bw_se(&pps, 0); bw_se(&pps, 0); bw_se(&pps, 0); // pic_init_qp-26, qs, chroma offset + bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // deblocking control, constrained intra, redundant + bw_trailing(&pps); + // SPS and PPS go out as one Sequence header, like FFmpeg + BitWriter headers[2] = { sps, pps }; + if (vaapi_add_packed(engine, VAEncPackedHeaderSequence, headers, 2, b->packed, &b->packed_n) != 0) return -1; + } + + VAEncPictureParameterBufferH264 pic_p; + memset(&pic_p, 0, sizeof(pic_p)); + pic_p.CurrPic.picture_id = curr_recon; + pic_p.CurrPic.flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + pic_p.CurrPic.frame_idx = frame_num; + pic_p.CurrPic.TopFieldOrderCnt = pic_p.CurrPic.BottomFieldOrderCnt = (2 * frame_num) & 15; + for (int i = 0; i < 16; i++) pic_p.ReferenceFrames[i].picture_id = VA_INVALID_ID; + if (!is_idr) { + pic_p.ReferenceFrames[0].picture_id = prev_recon; + pic_p.ReferenceFrames[0].flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + pic_p.ReferenceFrames[0].frame_idx = (frame_num - 1) & 15; + pic_p.ReferenceFrames[0].TopFieldOrderCnt = pic_p.ReferenceFrames[0].BottomFieldOrderCnt = (2 * (frame_num - 1)) & 15; + } + pic_p.coded_buf = engine->coded_buf; + pic_p.frame_num = frame_num; + pic_p.pic_init_qp = 26; + pic_p.seq_parameter_set_id = 0; + pic_p.pic_parameter_set_id = 0; + pic_p.pic_fields.bits.idr_pic_flag = is_idr ? 1 : 0; + pic_p.pic_fields.bits.reference_pic_flag = 1; + pic_p.pic_fields.bits.entropy_coding_mode_flag = 1; + pic_p.pic_fields.bits.deblocking_filter_control_present_flag = 0; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncPictureParameterBufferType, sizeof(pic_p), 1, &pic_p, &b->pic) != VA_STATUS_SUCCESS) + return -1; + + VAEncSliceParameterBufferH264 slice_p; + memset(&slice_p, 0, sizeof(slice_p)); + slice_p.macroblock_address = 0; + slice_p.num_macroblocks = mb_w * mb_h; + slice_p.macroblock_info = VA_INVALID_ID; + slice_p.slice_type = is_idr ? 2 : 0; // 2 = I-Slice, 0 = P-Slice + slice_p.direct_spatial_mv_pred_flag = 1; + // Consecutive IDRs must differ in idr_pic_id (7.4.3); flip it on every IDR + if (is_idr) engine->idr_pic_id ^= 1; + slice_p.idr_pic_id = engine->idr_pic_id; + slice_p.pic_order_cnt_lsb = pic_p.CurrPic.TopFieldOrderCnt; + for (int i = 0; i < 32; i++) slice_p.RefPicList0[i].picture_id = VA_INVALID_ID; + if (!is_idr) { + slice_p.RefPicList0[0].picture_id = prev_recon; + slice_p.RefPicList0[0].flags = VA_PICTURE_H264_SHORT_TERM_REFERENCE; + slice_p.RefPicList0[0].frame_idx = (frame_num - 1) & 15; + } + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncSliceParameterBufferType, sizeof(slice_p), 1, &slice_p, &b->slice) != VA_STATUS_SUCCESS) + return -1; + + // Slice header including cabac_alignment_one_bit, byte aligned and escaped as FFmpeg submits it + BitWriter sh = {0}; + bw_nal(&sh, 3, is_idr ? 5 : 1); + bw_ue(&sh, 0); bw_ue(&sh, is_idr ? 7 : 5); bw_ue(&sh, 0); // first_mb, slice_type I/P (all slices), pps id + bw_put(&sh, frame_num & 15, 4); + if (is_idr) bw_ue(&sh, slice_p.idr_pic_id); + bw_put(&sh, slice_p.pic_order_cnt_lsb & 15, 4); + if (!is_idr) { bw_put(&sh, 0, 1); bw_put(&sh, 0, 1); } // num_ref_idx override, ref list modification + if (is_idr) { bw_put(&sh, 0, 1); bw_put(&sh, 0, 1); } // no_output_of_prior_pics, long_term_reference + else bw_put(&sh, 0, 1); // adaptive_ref_pic_marking + if (!is_idr) bw_ue(&sh, 0); // cabac_init_idc + bw_se(&sh, 0); // slice_qp_delta (no deblocking fields) + while (sh.bits % 8) bw_put(&sh, 1, 1); + return vaapi_add_packed(engine, VAEncPackedHeaderSlice, &sh, 1, b->packed, &b->packed_n); +} + +// HEVC NAL header: forbidden 0, nal_unit_type, nuh_layer_id 0, nuh_temporal_id_plus1 1. +static void bw_hevc_nal(BitWriter* w, int type) { bw_put(w, 1, 32); bw_put(w, (type << 9) | 1, 16); } + +// profile_tier_level(1, 0): Main, Main 10 compatible like FFmpeg, progressive frames only. +static void bw_hevc_ptl(BitWriter* w, uint32_t level) { + bw_put(w, 0, 2); bw_put(w, 0, 1); bw_put(w, 1, 5); // profile space, tier, general_profile_idc Main + bw_put(w, 0x60000000, 32); // compatibility flags 1 and 2 + bw_put(w, 1, 1); bw_put(w, 0, 1); bw_put(w, 1, 1); bw_put(w, 1, 1); // progressive, interlaced, non_packed, frame_only + bw_put(w, 0, 32); bw_put(w, 0, 12); // reserved_zero_43bits + inbld_flag + bw_put(w, level, 8); +} + +// HEVC Main with FFmpeg's hevc_vaapi parameters on radeonsi (LIBVA_TRACE + trace_headers of its output): +// 64x64 CTB, 8x8 min CB, 4x4..32x32 TB, AMP and SAO on, one short-term reference, POC lsb 12 bits. +// FFmpeg also sends VA max_transform_hierarchy_depth 0 / diff_cu_qp_delta_depth 3 while its SPS/PPS +// say 4 / 0; both copies are kept as FFmpeg has them, since that is what decodes on this driver. +static int vaapi_hevc_params(VaapiEngine* engine, int is_idr, VASurfaceID curr_recon, VASurfaceID prev_recon, EncBuffers* b) { + // pic size must be a multiple of the 8x8 min CB; the conformance window (in chroma units) crops the rest + uint32_t w = (engine->width + 7) & ~7u, h = (engine->height + 7) & ~7u; + uint32_t crop_right = (w - engine->width) / 2, crop_bottom = (h - engine->height) / 2; + uint32_t poc = is_idr ? 0 : engine->since_idr; + // Level from luma picture size and sample rate (Table A.8): 4.1, 5.0, 5.1, 5.2 + uint64_t size = (uint64_t)w * h, rate = size * engine->fps; + uint32_t level = rate > 534773760 ? 156 : rate > 267386880 ? 153 : (size > 2228224 || rate > 133693440) ? 150 : 123; + if (is_idr) { + VAEncSequenceParameterBufferHEVC seq_p; + memset(&seq_p, 0, sizeof(seq_p)); + seq_p.general_profile_idc = 1; + seq_p.general_level_idc = level; + seq_p.intra_period = engine->gop; + seq_p.intra_idr_period = engine->gop; + seq_p.ip_period = 1; + seq_p.bits_per_second = (uint32_t)engine->max_bps; + seq_p.pic_width_in_luma_samples = w; + seq_p.pic_height_in_luma_samples = h; + seq_p.seq_fields.bits.chroma_format_idc = 1; + seq_p.seq_fields.bits.amp_enabled_flag = 1; + seq_p.seq_fields.bits.sample_adaptive_offset_enabled_flag = 1; + seq_p.log2_diff_max_min_luma_coding_block_size = 3; + seq_p.log2_diff_max_min_transform_block_size = 3; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncSequenceParameterBufferType, sizeof(seq_p), 1, &seq_p, &b->seq) != VA_STATUS_SUCCESS) + return -1; + + BitWriter vps = {0}; + bw_hevc_nal(&vps, 32); + bw_put(&vps, 0, 4); bw_put(&vps, 1, 1); bw_put(&vps, 1, 1); // vps id, base layer internal / available + bw_put(&vps, 0, 6); bw_put(&vps, 0, 3); bw_put(&vps, 1, 1); // max layers-1, max sub layers-1, temporal id nesting + bw_put(&vps, 0xffff, 16); + bw_hevc_ptl(&vps, level); + bw_put(&vps, 0, 1); bw_ue(&vps, 1); bw_ue(&vps, 0); bw_ue(&vps, 0); // ordering info: dec buffering-1, reorder, latency + bw_put(&vps, 0, 6); bw_ue(&vps, 0); // max layer id, num layer sets-1 + bw_put(&vps, 0, 1); bw_put(&vps, 0, 1); // no timing info, no extension + bw_trailing(&vps); + + BitWriter sps = {0}; + bw_hevc_nal(&sps, 33); + bw_put(&sps, 0, 4); bw_put(&sps, 0, 3); bw_put(&sps, 1, 1); // vps id, max sub layers-1, temporal id nesting + bw_hevc_ptl(&sps, level); + bw_ue(&sps, 0); bw_ue(&sps, 1); bw_ue(&sps, w); bw_ue(&sps, h); // sps id, 4:2:0, size + bw_put(&sps, crop_right || crop_bottom, 1); + if (crop_right || crop_bottom) { bw_ue(&sps, 0); bw_ue(&sps, crop_right); bw_ue(&sps, 0); bw_ue(&sps, crop_bottom); } + bw_ue(&sps, 0); bw_ue(&sps, 0); bw_ue(&sps, 8); // 8-bit luma/chroma, log2_max_poc_lsb-4 + bw_put(&sps, 0, 1); bw_ue(&sps, 1); bw_ue(&sps, 0); bw_ue(&sps, 0); + bw_ue(&sps, 0); bw_ue(&sps, 3); bw_ue(&sps, 0); bw_ue(&sps, 3); // CB 8..64, TB 4..32 + bw_ue(&sps, 4); bw_ue(&sps, 4); // max transform hierarchy depth inter/intra + bw_put(&sps, 0, 1); bw_put(&sps, 1, 1); bw_put(&sps, 1, 1); bw_put(&sps, 0, 1); // scaling list, AMP, SAO, PCM + bw_ue(&sps, 0); bw_put(&sps, 0, 1); // RPS in slice headers, no long-term refs + bw_put(&sps, 0, 1); bw_put(&sps, 0, 1); // temporal MVP, strong intra smoothing + bw_put(&sps, 0, 1); bw_put(&sps, 0, 1); // no VUI, no extension + bw_trailing(&sps); + + BitWriter pps = {0}; + bw_hevc_nal(&pps, 34); + bw_ue(&pps, 0); bw_ue(&pps, 0); + bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); bw_put(&pps, 0, 3); // dependent slices, output flag, extra bits + bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // sign data hiding, cabac init present + bw_ue(&pps, 0); bw_ue(&pps, 0); bw_se(&pps, 0); // num_ref_idx l0/l1 default-1, init_qp-26 + bw_put(&pps, 0, 1); bw_put(&pps, 1, 1); bw_put(&pps, 1, 1); // constrained intra, transform skip, cu_qp_delta + bw_ue(&pps, 0); bw_se(&pps, 0); bw_se(&pps, 0); // diff_cu_qp_delta_depth, cb/cr offsets + bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // slice chroma offsets, weighted pred/bipred + bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // transquant bypass, tiles, entropy sync + bw_put(&pps, 1, 1); bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // loop filter across slices, deblocking ctl, scaling + bw_put(&pps, 0, 1); bw_ue(&pps, 0); bw_put(&pps, 0, 1); bw_put(&pps, 0, 1); // list mod, merge level-2, ext + bw_trailing(&pps); + + BitWriter headers[3] = { vps, sps, pps }; + if (vaapi_add_packed(engine, VAEncPackedHeaderSequence, headers, 3, b->packed, &b->packed_n) != 0) return -1; + } + + VAEncPictureParameterBufferHEVC pic_p; + memset(&pic_p, 0, sizeof(pic_p)); + pic_p.decoded_curr_pic.picture_id = curr_recon; + pic_p.decoded_curr_pic.pic_order_cnt = poc; + for (int i = 0; i < 15; i++) { + pic_p.reference_frames[i].picture_id = VA_INVALID_ID; + pic_p.reference_frames[i].flags = VA_PICTURE_HEVC_INVALID; + } + if (!is_idr) { + pic_p.reference_frames[0].picture_id = prev_recon; + pic_p.reference_frames[0].pic_order_cnt = poc - 1; + pic_p.reference_frames[0].flags = VA_PICTURE_HEVC_RPS_ST_CURR_BEFORE; + } + pic_p.coded_buf = engine->coded_buf; + pic_p.collocated_ref_pic_index = 0xff; + pic_p.pic_init_qp = 30; + pic_p.diff_cu_qp_delta_depth = 3; + pic_p.nal_unit_type = is_idr ? 19 : 1; // IDR_W_RADL / TRAIL_R + pic_p.pic_fields.bits.idr_pic_flag = is_idr; + pic_p.pic_fields.bits.coding_type = is_idr ? 1 : 2; + pic_p.pic_fields.bits.reference_pic_flag = 1; + pic_p.pic_fields.bits.transform_skip_enabled_flag = 1; + pic_p.pic_fields.bits.cu_qp_delta_enabled_flag = 1; + pic_p.pic_fields.bits.pps_loop_filter_across_slices_enabled_flag = 1; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncPictureParameterBufferType, sizeof(pic_p), 1, &pic_p, &b->pic) != VA_STATUS_SUCCESS) + return -1; + + VAEncSliceParameterBufferHEVC slice_p; + memset(&slice_p, 0, sizeof(slice_p)); + slice_p.num_ctu_in_slice = ((w + 63) / 64) * ((h + 63) / 64); + slice_p.slice_type = is_idr ? 2 : 1; // I / P + for (int i = 0; i < 15; i++) { + slice_p.ref_pic_list0[i].picture_id = slice_p.ref_pic_list1[i].picture_id = VA_INVALID_ID; + slice_p.ref_pic_list0[i].flags = slice_p.ref_pic_list1[i].flags = VA_PICTURE_HEVC_INVALID; + } + if (!is_idr) slice_p.ref_pic_list0[0] = pic_p.reference_frames[0]; + slice_p.max_num_merge_cand = 5; + slice_p.slice_fields.bits.last_slice_of_pic_flag = 1; + slice_p.slice_fields.bits.slice_sao_luma_flag = 1; + slice_p.slice_fields.bits.slice_sao_chroma_flag = 1; + slice_p.slice_fields.bits.collocated_from_l0_flag = !is_idr; + if (RT.vaCreateBuffer(engine->va_dpy, engine->enc_ctx, VAEncSliceParameterBufferType, sizeof(slice_p), 1, &slice_p, &b->slice) != VA_STATUS_SUCCESS) + return -1; + + BitWriter sh = {0}; + bw_hevc_nal(&sh, pic_p.nal_unit_type); + bw_put(&sh, 1, 1); // first slice segment + if (is_idr) bw_put(&sh, 0, 1); // no_output_of_prior_pics + bw_ue(&sh, 0); bw_ue(&sh, slice_p.slice_type); // pps id, slice type + if (!is_idr) { + bw_put(&sh, poc & 0xfff, 12); + bw_put(&sh, 0, 1); // RPS coded here: one previous picture + bw_ue(&sh, 1); bw_ue(&sh, 0); bw_ue(&sh, 0); bw_put(&sh, 1, 1); // negative 1, positive 0, delta-1 0, used + } + bw_put(&sh, 1, 1); bw_put(&sh, 1, 1); // SAO luma, chroma + if (!is_idr) { bw_put(&sh, 0, 1); bw_ue(&sh, 0); } // num_ref_idx override, five_minus_max_merge + bw_se(&sh, 0); // slice_qp_delta (the driver writes the real one) + bw_put(&sh, 0, 1); // loop filter across slices + bw_put(&sh, 1, 1); // byte_alignment() + while (sh.bits % 8) bw_put(&sh, 0, 1); + return vaapi_add_packed(engine, VAEncPackedHeaderSlice, &sh, 1, b->packed, &b->packed_n); +} + +int vaapi_encode_frame( + VaapiEngine* engine, + int force_idr, + const uint8_t** out_data, + uint32_t* out_len +) { + if (out_data) *out_data = NULL; + if (out_len) *out_len = 0; + if (!engine || engine->in_rgb_surf == VA_INVALID_ID) return -1; + + int is_idr = force_idr || engine->idr_pending || engine->frame_count == 0 || engine->since_idr >= engine->gop; + // Recon ping-pong: a P frame must reference the previous recon surface, not its own + VASurfaceID curr_recon = engine->recon_surfs[engine->frame_count & 1]; + VASurfaceID prev_recon = engine->recon_surfs[(engine->frame_count + 1) & 1]; + + // 1. Hardware VPP Color Conversion + scaling (RGB -> NV12 entirely inside VRAM) + VAStatus st = RT.vaBeginPicture(engine->va_dpy, engine->vpp_ctx, engine->nv12_surf); + if (st != VA_STATUS_SUCCESS) return -10; + st = RT.vaRenderPicture(engine->va_dpy, engine->vpp_ctx, &engine->vpp_buf, 1); + if (st != VA_STATUS_SUCCESS) return -10; + st = RT.vaEndPicture(engine->va_dpy, engine->vpp_ctx); + if (st != VA_STATUS_SUCCESS) return -10; + + // 2. Rate control (every IDR) and the codec's sequence / picture / slice parameters + packed headers + EncBuffers b = { .seq = VA_INVALID_ID, .pic = VA_INVALID_ID, .slice = VA_INVALID_ID }; + int ret = -11; + if (is_idr) { + VAEncMiscParameterRateControl rc; + memset(&rc, 0, sizeof(rc)); + // VBR: bits_per_second is the ceiling (maxrate) and target_percentage the average (#33) + rc.bits_per_second = (uint32_t)engine->max_bps; + rc.target_percentage = engine->rc_mode == VA_RC_VBR ? (uint32_t)(engine->target_bps * 100 / engine->max_bps) : 100; + rc.window_size = 1000; + rc.initial_qp = 28; + rc.min_qp = 16; + rc.max_qp = 44; + rc.rc_flags.bits.disable_frame_skip = 1; // a remote desktop must not drop frames silently + b.misc[b.misc_n++] = vaapi_create_misc_buffer(engine, VAEncMiscParameterTypeRateControl, &rc, sizeof(rc)); + + VAEncMiscParameterHRD hrd; + memset(&hrd, 0, sizeof(hrd)); + hrd.buffer_size = (uint32_t)engine->buffer_bits; // the viewer preset's bufsize + hrd.initial_buffer_fullness = hrd.buffer_size / 2; + b.misc[b.misc_n++] = vaapi_create_misc_buffer(engine, VAEncMiscParameterTypeHRD, &hrd, sizeof(hrd)); + + // numerator in the low 16 bits, denominator in the high 16 bits + VAEncMiscParameterFrameRate fr = { .framerate = (1u << 16) | engine->fps }; + b.misc[b.misc_n++] = vaapi_create_misc_buffer(engine, VAEncMiscParameterTypeFrameRate, &fr, sizeof(fr)); + } + if ((engine->is_hevc ? vaapi_hevc_params : vaapi_h264_params)(engine, is_idr, curr_recon, prev_recon, &b) != 0) goto done; + + // 3. Render and Encode on AMD VCN hardware + VABufferID render_b[12]; + int render_n = 0; + // FFmpeg's order: sequence, rate control misc, picture, packed headers (parameter sets, slice header), slice. + // With misc first, radeonsi (Mesa 26) ignored the VBR target: the 1M preset ran at 1.6 Mbps, 0.96 after. + if (b.seq != VA_INVALID_ID) render_b[render_n++] = b.seq; + for (int i = 0; i < b.misc_n; i++) { + if (b.misc[i] != VA_INVALID_ID) render_b[render_n++] = b.misc[i]; + } + render_b[render_n++] = b.pic; + for (int i = 0; i < b.packed_n; i++) render_b[render_n++] = b.packed[i]; + render_b[render_n++] = b.slice; + + st = RT.vaBeginPicture(engine->va_dpy, engine->enc_ctx, engine->nv12_surf); + if (st != VA_STATUS_SUCCESS) goto done; + st = RT.vaRenderPicture(engine->va_dpy, engine->enc_ctx, render_b, render_n); + if (st != VA_STATUS_SUCCESS) goto done; + st = RT.vaEndPicture(engine->va_dpy, engine->enc_ctx); + if (st != VA_STATUS_SUCCESS) goto done; + st = RT.vaSyncSurface(engine->va_dpy, curr_recon); + if (st != VA_STATUS_SUCCESS) goto done; + + // 4. Map Bitstream Output; an access unit may arrive as several linked segments + VACodedBufferSegment *segment = NULL; + st = RT.vaMapBuffer(engine->va_dpy, engine->coded_buf, (void**)&segment); + if (st != VA_STATUS_SUCCESS || !segment) goto done; + uint32_t total = 0; + for (VACodedBufferSegment *seg = segment; seg; seg = seg->next) total += seg->size; + if (total > 0 && !(segment->status & VA_CODED_BUF_STATUS_SLICE_OVERFLOW_MASK)) { + if (total > engine->output_staging_cap) { + engine->output_staging_cap = total + 64 * 1024; + engine->output_staging = (uint8_t*)realloc(engine->output_staging, engine->output_staging_cap); + } + uint32_t off = 0; + for (VACodedBufferSegment *seg = segment; seg; seg = seg->next) { + memcpy(engine->output_staging + off, seg->buf, seg->size); + off += seg->size; + } + *out_data = engine->output_staging; + *out_len = total; + } + RT.vaUnmapBuffer(engine->va_dpy, engine->coded_buf); + + engine->frame_num = is_idr ? 1 : ((engine->frame_num + 1) & 15); + engine->since_idr = is_idr ? 1 : engine->since_idr + 1; + engine->frame_count++; + ret = 0; + +done: + // No bitstream (failure, empty or slice overflow): the next frame_num / POC would skip, so restart with an IDR + engine->idr_pending = ret != 0 || *out_len == 0; + for (int i = 0; i < b.misc_n; i++) { + if (b.misc[i] != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, b.misc[i]); + } + if (b.seq != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, b.seq); + if (b.pic != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, b.pic); + if (b.slice != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, b.slice); + for (int i = 0; i < b.packed_n; i++) RT.vaDestroyBuffer(engine->va_dpy, b.packed[i]); + return ret; +} + +void vaapi_destroy(VaapiEngine* engine) { + if (!engine) return; + + if (engine->coded_buf != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, engine->coded_buf); + if (engine->enc_ctx != VA_INVALID_ID) RT.vaDestroyContext(engine->va_dpy, engine->enc_ctx); + if (engine->enc_cfg != VA_INVALID_ID) RT.vaDestroyConfig(engine->va_dpy, engine->enc_cfg); + + if (engine->vpp_buf != VA_INVALID_ID) RT.vaDestroyBuffer(engine->va_dpy, engine->vpp_buf); + if (engine->vpp_ctx != VA_INVALID_ID) RT.vaDestroyContext(engine->va_dpy, engine->vpp_ctx); + if (engine->vpp_cfg != VA_INVALID_ID) RT.vaDestroyConfig(engine->va_dpy, engine->vpp_cfg); + + if (engine->upload_ready) RT.vaDestroyImage(engine->va_dpy, engine->upload_img.image_id); + if (engine->in_rgb_surf != VA_INVALID_ID) RT.vaDestroySurfaces(engine->va_dpy, &engine->in_rgb_surf, 1); + if (engine->nv12_surf != VA_INVALID_ID) RT.vaDestroySurfaces(engine->va_dpy, &engine->nv12_surf, 1); + if (engine->recon_surfs[0] != VA_INVALID_ID || engine->recon_surfs[1] != VA_INVALID_ID) { + RT.vaDestroySurfaces(engine->va_dpy, engine->recon_surfs, 2); + } + + if (engine->va_dpy) RT.vaTerminate(engine->va_dpy); + if (engine->drm_fd >= 0) close(engine->drm_fd); + if (engine->output_staging) free(engine->output_staging); + + free(engine); +} diff --git a/c_engine/vaapi_engine.h b/c_engine/vaapi_engine.h new file mode 100644 index 0000000..567b9fc --- /dev/null +++ b/c_engine/vaapi_engine.h @@ -0,0 +1,49 @@ +#ifndef UU_VAAPI_ENGINE_H +#define UU_VAAPI_ENGINE_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct VaapiEngine VaapiEngine; + +typedef struct { + uint32_t width; // encode (output) size + uint32_t height; + uint32_t cap_width; // capture (input) size; VPP scales cap -> width + uint32_t cap_height; + uint32_t fps; + uint32_t gop; + int64_t target_bps; + int64_t max_bps; // VBR ceiling (viewer preset maxrate) + int64_t buffer_bits; // HRD buffer size + int is_hevc; + const char *device; // DRM render node (UU_VAAPI_DEVICE); NULL = /dev/dri/renderD128 +} VaapiConfig; + +VaapiEngine* vaapi_create(const VaapiConfig* config); + +// Import a DRI3 / GBM DMA-BUF fd as the zero-copy input surface +int vaapi_import_dma_buffer(VaapiEngine* engine, int dma_fd, uint32_t stride); + +// Host fallback: upload ARGB host memory if DMA-BUF is unavailable +int vaapi_upload_host_buffer(VaapiEngine* engine, const uint8_t* host_ptr, uint32_t pitch); + +// Encode one frame +int vaapi_encode_frame( + VaapiEngine* engine, + int force_idr, + const uint8_t** out_data, + uint32_t* out_len +); + +void vaapi_destroy(VaapiEngine* engine); + +#ifdef __cplusplus +} +#endif + +#endif // UU_VAAPI_ENGINE_H diff --git a/src/inproc_encoder.rs b/src/inproc_encoder.rs index 8c48837..a4a6d77 100644 --- a/src/inproc_encoder.rs +++ b/src/inproc_encoder.rs @@ -1,8 +1,12 @@ //! In-process X11 capture (XDamage + XShm) and NVENC encoding through the NVIDIA driver. +//! With UU_VIDEO_ENCODER=vaapi, H.264 / HEVC go through VAAPI instead: a DRI3 DMA-BUF of the root window is +//! imported as a VA surface and converted/scaled by VPP, so the CPU never touches the pixels (XShm upload +//! when DRI3 is unavailable). //! When capture and encoder sizes match, the XShm buffer is registered as CUDA pinned memory and //! NVENC reads it by DMA, so there is no host-to-device copy (the X server still writes XShm). use std::collections::VecDeque; +use std::ffi::CString; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -29,6 +33,31 @@ struct NvencConfigFFI { chroma_444: libc::c_int, } +#[repr(C)] +struct VaapiEngineOpaque { + _private: [u8; 0], +} + +#[repr(C)] +struct VaapiConfigFFI { + width: u32, + height: u32, + cap_width: u32, + cap_height: u32, + fps: u32, + gop: u32, + target_bps: i64, + max_bps: i64, + buffer_bits: i64, + is_hevc: libc::c_int, + device: *const libc::c_char, +} + +#[repr(C)] +struct Dri3CaptureOpaque { + _private: [u8; 0], +} + extern "C" { fn nvenc_create(config: *const NvencConfigFFI) -> *mut NvencEngineOpaque; fn nvenc_register_host_buffer( @@ -45,6 +74,24 @@ extern "C" { out_len: *mut u32, ) -> libc::c_int; fn nvenc_destroy(engine: *mut NvencEngineOpaque); + + fn vaapi_create(config: *const VaapiConfigFFI) -> *mut VaapiEngineOpaque; + fn vaapi_import_dma_buffer(engine: *mut VaapiEngineOpaque, dma_fd: libc::c_int, stride: u32) -> libc::c_int; + fn vaapi_upload_host_buffer(engine: *mut VaapiEngineOpaque, host_ptr: *const u8, pitch: u32) -> libc::c_int; + fn vaapi_encode_frame( + engine: *mut VaapiEngineOpaque, + force_idr: libc::c_int, + out_data: *mut *const u8, + out_len: *mut u32, + ) -> libc::c_int; + fn vaapi_destroy(engine: *mut VaapiEngineOpaque); + + fn dri3_capture_create(display_name: *const libc::c_char, width: u32, height: u32) -> *mut Dri3CaptureOpaque; + fn dri3_capture_blit(cap: *mut Dri3CaptureOpaque) -> libc::c_int; + fn dri3_capture_wait_damage(cap: *mut Dri3CaptureOpaque, timeout_ms: libc::c_int) -> libc::c_int; + fn dri3_capture_get_dma_fd(cap: *mut Dri3CaptureOpaque) -> libc::c_int; + fn dri3_capture_get_stride(cap: *mut Dri3CaptureOpaque) -> u32; + fn dri3_capture_destroy(cap: *mut Dri3CaptureOpaque); } struct SafeNvencEngine(*mut NvencEngineOpaque); @@ -60,6 +107,28 @@ impl Drop for SafeNvencEngine { } } +struct SafeVaapiEngine(*mut VaapiEngineOpaque); +unsafe impl Send for SafeVaapiEngine {} +impl Drop for SafeVaapiEngine { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { vaapi_destroy(self.0) }; + self.0 = std::ptr::null_mut(); + } + } +} + +struct SafeDri3Capture(*mut Dri3CaptureOpaque); +unsafe impl Send for SafeDri3Capture {} +impl Drop for SafeDri3Capture { + fn drop(&mut self) { + if !self.0.is_null() { + unsafe { dri3_capture_destroy(self.0) }; + self.0 = std::ptr::null_mut(); + } + } +} + pub struct Config { pub display: String, pub input: (u32, u32), @@ -73,6 +142,8 @@ pub struct Config { pub codec: crate::capture_setting::VideoCodec, /// ChromaFormat (capture_setting::CHROMA_444 = 4:4:4). pub chroma: u64, + /// UU_VIDEO_ENCODER=vaapi: the VAAPI engine (H.264 / HEVC Main) instead of NVENC. + pub vaapi: bool, } pub fn disabled() -> bool { @@ -84,7 +155,7 @@ pub fn damage_enabled() -> bool { } pub fn selected(encoder: crate::video::Encoder) -> bool { - encoder == crate::video::Encoder::Nvenc && damage_enabled() && !disabled() && crate::config::ffmpeg_root().is_none() + matches!(encoder, crate::video::Encoder::Nvenc | crate::video::Encoder::Vaapi) && damage_enabled() && !disabled() && crate::config::ffmpeg_root().is_none() } /// One encoder thread per stream. Aborting the reader only asks the blocking thread to stop at its @@ -139,6 +210,205 @@ pub fn run( stop: Arc, idr: Arc, budget: Arc, +) -> Result<(), String> { + if config.vaapi { + run_vaapi(config, units, stop, idr, budget) + } else { + run_nvenc(config, units, stop, idr, budget) + } +} + +fn run_vaapi( + config: Config, + units: mpsc::Sender, + stop: Arc, + idr: Arc, + budget: Arc, +) -> Result<(), String> { + let (width, height) = config.output.unwrap_or(config.input); + let (width, height) = (width + width % 2, height + height % 2); + let (cap_width, cap_height) = config.input; + log::info!( + "Initializing Native In-Process VAAPI: {}x{} -> {}x{}, {} fps, GOP {}, HEVC={}", + cap_width, + cap_height, + width, + height, + config.fps, + config.gop, + config.codec == crate::capture_setting::VideoCodec::H265 + ); + let device = CString::new(crate::video::vaapi_device()).map_err(|_| "UU_VAAPI_DEVICE contains NUL".to_string())?; + let ffi_cfg = VaapiConfigFFI { + width, + height, + cap_width, + cap_height, + fps: config.fps, + gop: config.gop, + target_bps: config.target_bps, + max_bps: config.max_bps, + buffer_bits: config.buffer_bits, + is_hevc: libc::c_int::from(config.codec == crate::capture_setting::VideoCodec::H265), + device: device.as_ptr(), + }; + let raw_engine = unsafe { vaapi_create(&ffi_cfg) }; + if raw_engine.is_null() { + return Err("Failed to create native VAAPI engine".into()); + } + let engine = SafeVaapiEngine(raw_engine); + + // Capture stays at the full input size; VPP scales (or pads) to the encoder size. + let display = CString::new(config.display.as_str()).map_err(|_| "DISPLAY contains NUL".to_string())?; + let dri3 = Some(SafeDri3Capture(unsafe { dri3_capture_create(display.as_ptr(), cap_width, cap_height) })) + .filter(|c| !c.0.is_null()) + .filter(|c| unsafe { + let (fd, stride) = (dri3_capture_get_dma_fd(c.0), dri3_capture_get_stride(c.0)); + fd >= 0 && stride > 0 && vaapi_import_dma_buffer(engine.0, fd, stride) == 0 + }); + let mut capture = None; + if dri3.is_some() { + log::info!("Native VAAPI imports a DRI3 DMA-BUF of the root window (GPU blit, VPP to NV12)"); + } else { + log::warn!("DRI3 DMA-BUF unavailable on {}, uploading XShm frames to VAAPI", config.display); + capture = Some( + DamageCapture::open(&config.display) + .map_err(|e| format!("DamageCapture::open failed on {}: {e}", config.display))?, + ); + } + + let frame_interval = Duration::from_nanos(1_000_000_000 / u64::from(config.fps)); + let mut seed = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(1, |d| d.subsec_nanos()) + | 1; + let mut keepalive = random_keepalive(&mut seed); + let mut dirty = true; + let mut next_tick = Instant::now(); + let mut last_send = Instant::now(); + let mut permits = VecDeque::with_capacity(4); + + let locked = crate::session_lock::watch_locked(); + let mut last_locked = *locked.borrow(); + + loop { + if stop.load(Ordering::Relaxed) { + return Ok(()); + } + + let deadline = if dirty { + next_tick + } else { + last_send + keepalive + }; + let timeout = deadline.saturating_duration_since(Instant::now()); + + if vaapi_wait(&dri3, &mut capture, timeout.max(Duration::from_millis(1)))? { + dirty = true; + } + + let now = Instant::now(); + if dirty && now < next_tick { + continue; + } + if !dirty && now < last_send + keepalive { + continue; + } + + let Some(permit) = acquire(&budget, &units) else { + return Ok(()); + }; + if stop.load(Ordering::Relaxed) { + return Ok(()); + } + + dirty |= vaapi_wait(&dri3, &mut capture, Duration::ZERO)?; + let state = *locked.borrow(); + if state != last_locked { + dirty = true; + last_locked = state; + } + + let force_idr = if idr.swap(false, Ordering::AcqRel) { + 1 + } else { + 0 + }; + + // Keepalive frames re-encode the last surface; only damage needs a new copy of the screen. + if dirty { + match (dri3.as_ref(), capture.as_mut()) { + (Some(c), _) => { + if unsafe { dri3_capture_blit(c.0) } != 0 { + return Err("DRI3 blit of the root window failed".into()); + } + } + (None, Some(c)) => { + let pitch = c.size().0 * 4; + let res = unsafe { vaapi_upload_host_buffer(engine.0, c.capture()?.as_ptr(), pitch) }; + if res != 0 { + return Err(format!("vaapi_upload_host_buffer failed with error {res}")); + } + } + (None, None) => unreachable!(), + } + } + + let mut out_data: *const u8 = std::ptr::null(); + let mut out_len: u32 = 0; + let res = unsafe { vaapi_encode_frame(engine.0, force_idr, &mut out_data, &mut out_len) }; + if res != 0 { + return Err(format!("vaapi_encode_frame failed with error {res}")); + } + + if out_len > 0 && !out_data.is_null() { + let data = unsafe { std::slice::from_raw_parts(out_data, out_len as usize) }.to_vec(); + permits.push_back(permit); + let send_permit = permits.pop_front().unwrap(); + + if units + .blocking_send(EncodedUnit { + data, + permit: send_permit, + completed_at: Instant::now(), + }) + .is_err() + { + return Ok(()); + } + } + + let now = Instant::now(); + if dirty { + let next = next_tick + frame_interval; + next_tick = if next > now { + next + } else { + now + frame_interval + }; + dirty = false; + } + last_send = now; + keepalive = random_keepalive(&mut seed); + } +} +fn vaapi_wait(dri3: &Option, capture: &mut Option, timeout: Duration) -> Result { + match (dri3, capture) { + (Some(c), _) => match unsafe { dri3_capture_wait_damage(c.0, timeout.as_millis().min(1000) as i32) } { + r if r < 0 => Err("XDamage wait on the DRI3 connection failed".into()), + r => Ok(r > 0), + }, + (None, Some(c)) => c.wait(timeout), + (None, None) => unreachable!(), + } +} + +fn run_nvenc( + config: Config, + units: mpsc::Sender, + stop: Arc, + idr: Arc, + budget: Arc, ) -> Result<(), String> { let (width, height) = config.output.unwrap_or(config.input); let (width, height) = (width + width % 2, height + height % 2); diff --git a/src/video.rs b/src/video.rs index 8ebe7eb..9f921bc 100644 --- a/src/video.rs +++ b/src/video.rs @@ -617,6 +617,7 @@ async fn read_encoder_with( gop: gop_frames(settings.fps, true) as u32, cq: settings.cq() as u32, target_bps: settings.max_kbps().map_or(16_000, |k| i64::from(k) * 3 / 4) * 1000, max_bps: max_kbps * 1000, buffer_bits: i64::from(buffer_for(&settings, encoder)) * 1000, codec: settings.codec, chroma: settings.chroma, + vaapi: encoder == Encoder::Vaapi, }; log::info!("Desktop encoder: display={} source={}x{} output={:?} fps={} cq={} maxrate={}k encoder={} pipeline=in-process", display,size.0,size.1,settings.output,settings.fps,config.cq,max_kbps,encoder.name_for(settings.codec));