Implement VAAPI rendering for X11

app/app.pro

@@ -35,6 +35,7 @@ macx {
 unix:!macx {
     CONFIG += link_pkgconfig
     PKGCONFIG += openssl sdl2 libavcodec libavdevice libavformat libavutil
+    LIBS += -ldl
 }
 win32 {
     LIBS += -llibssl -llibcrypto -lSDL2 -lavcodec -lavdevice -lavformat -lavutil
@@ -66,6 +67,9 @@ win32 {
 macx {
     SOURCES += streaming/video/ffmpeg-renderers/vt.mm
 }
+unix {
+    SOURCES += streaming/video/ffmpeg-renderers/vaapi.cpp
+}
 
 HEADERS += \
     utils.h \
@@ -89,6 +93,9 @@ win32 {
 macx {
     HEADERS += streaming/video/ffmpeg-renderers/vt.h
 }
+unix {
+    HEADERS += streaming/video/ffmpeg-renderers/vaapi.h
+}
 
 RESOURCES += \
     resources.qrc \

app/streaming/session.cpp

@@ -620,8 +620,10 @@ void Session::exec()
         }
 
         case SDL_WINDOWEVENT:
-            // We want to recreate the decoder for resizes (full-screen toggles) and the initial shown event
-            if (event.window.event != SDL_WINDOWEVENT_RESIZED && event.window.event != SDL_WINDOWEVENT_SHOWN) {
+            // We want to recreate the decoder for resizes (full-screen toggles) and the initial shown event.
+            // We use SDL_WINDOWEVENT_SIZE_CHANGED rather than SDL_WINDOWEVENT_RESIZED because the latter doesn't
+            // seem to fire when switching from windowed to full-screen on X11.
+            if (event.window.event != SDL_WINDOWEVENT_SIZE_CHANGED && event.window.event != SDL_WINDOWEVENT_SHOWN) {
                 break;
             }
 

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -0,0 +1,132 @@
+#include "vaapi.h"
+
+#include <dlfcn.h>
+
+#include <SDL_syswm.h>
+
+VAAPIRenderer::VAAPIRenderer()
+    : m_HwContext(nullptr),
+      m_X11VaLibHandle(nullptr),
+      m_vaPutSurface(nullptr)
+{
+
+}
+
+VAAPIRenderer::~VAAPIRenderer()
+{
+    if (m_HwContext != nullptr) {
+        av_buffer_unref(&m_HwContext);
+    }
+
+    if (m_X11VaLibHandle != nullptr) {
+        dlclose(m_X11VaLibHandle);
+    }
+}
+
+bool
+VAAPIRenderer::initialize(SDL_Window* window, int, int width, int height)
+{
+    int err;
+    SDL_SysWMinfo info;
+
+    m_VideoWidth = width;
+    m_VideoHeight = height;
+
+    SDL_GetWindowSize(window, &m_DisplayWidth, &m_DisplayHeight);
+
+    SDL_VERSION(&info.version);
+
+    if (!SDL_GetWindowWMInfo(window, &info)) {
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "SDL_GetWindowWMInfo() failed: %s",
+                    SDL_GetError());
+        return false;
+    }
+
+    SDL_assert(info.subsystem == SDL_SYSWM_X11);
+
+    if (info.subsystem == SDL_SYSWM_X11) {
+        m_XWindow = info.info.x11.window;
+
+        m_X11VaLibHandle = dlopen("libva-x11.so", RTLD_LAZY);
+        if (!m_X11VaLibHandle) {
+            SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                        "dlopen(libva.so) failed: %s",
+                        dlerror());
+            return false;
+        }
+
+        m_vaPutSurface = (vaPutSurface_t)dlsym(m_X11VaLibHandle, "vaPutSurface");
+    }
+    else if (info.subsystem == SDL_SYSWM_WAYLAND) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VAAPI backend does not currently support Wayland");
+        return false;
+    }
+    else {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Unsupported VAAPI rendering subsystem: %d",
+                     info.subsystem);
+        return false;
+    }
+
+
+    err = av_hwdevice_ctx_create(&m_HwContext,
+                                 AV_HWDEVICE_TYPE_VAAPI,
+                                 nullptr, nullptr, 0);
+    if (err < 0) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "Failed to create VAAPI context: %d",
+                     err);
+        return false;
+    }
+
+    return true;
+}
+
+bool
+VAAPIRenderer::prepareDecoderContext(AVCodecContext* context)
+{
+    context->hw_device_ctx = av_buffer_ref(m_HwContext);
+    return true;
+}
+
+void
+VAAPIRenderer::renderFrame(AVFrame* frame)
+{
+    VASurfaceID surface = (VASurfaceID)(uintptr_t)frame->data[3];
+    AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
+    AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
+
+    // Center in frame and preserve aspect ratio
+    int x, y, width, height;
+    double srcAspectRatio = (double)m_VideoWidth / (double)m_VideoHeight;
+    double dstAspectRatio = (double)m_DisplayWidth / (double)m_DisplayHeight;
+    if (dstAspectRatio < srcAspectRatio) {
+        // Greater height per width
+        int drawHeight = (int)(m_DisplayWidth / srcAspectRatio);
+        y = (m_DisplayHeight - drawHeight) / 2;
+        height = drawHeight;
+        x = 0;
+        width = m_DisplayWidth;
+    }
+    else {
+        // Greater width per height
+        int drawWidth = (int)(m_DisplayHeight * srcAspectRatio);
+        y = 0;
+        height = m_DisplayHeight;
+        x = (m_DisplayWidth - drawWidth) / 2;
+        width = drawWidth;
+    }
+
+    m_vaPutSurface(vaDeviceContext->display,
+                   surface,
+                   m_XWindow,
+                   0, 0,
+                   m_VideoWidth, m_VideoHeight,
+                   x, y,
+                   width, height,
+                   NULL, 0, 0);
+
+    av_frame_free(&frame);
+}

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include "renderer.h"
+
+extern "C" {
+#include <va/va.h>
+#include <va/va_x11.h>
+#include <libavutil/hwcontext_vaapi.h>
+}
+
+class VAAPIRenderer : public IFFmpegRenderer
+{
+    typedef VAStatus (*vaPutSurface_t)(
+        VADisplay dpy,
+        VASurfaceID surface,
+        Drawable draw, /* X Drawable */
+        short srcx,
+        short srcy,
+        unsigned short srcw,
+        unsigned short srch,
+        short destx,
+        short desty,
+        unsigned short destw,
+        unsigned short desth,
+        VARectangle *cliprects,
+        unsigned int number_cliprects,
+        unsigned int flags
+    );
+
+public:
+    VAAPIRenderer();
+    virtual ~VAAPIRenderer();
+    virtual bool initialize(SDL_Window* window,
+                            int videoFormat,
+                            int width,
+                            int height);
+    virtual bool prepareDecoderContext(AVCodecContext* context);
+    virtual void renderFrame(AVFrame* frame);
+
+private:
+    Window m_XWindow;
+    AVBufferRef* m_HwContext;
+    void* m_X11VaLibHandle;
+    vaPutSurface_t m_vaPutSurface;
+    int m_VideoWidth;
+    int m_VideoHeight;
+    int m_DisplayWidth;
+    int m_DisplayHeight;
+};
+

app/streaming/video/ffmpeg.cpp

@@ -9,6 +9,10 @@
 #include "ffmpeg-renderers/vt.h"
 #endif
 
+#ifdef Q_OS_UNIX
+#include "ffmpeg-renderers/vaapi.h"
+#endif
+
 bool FFmpegVideoDecoder::chooseDecoder(
         StreamingPreferences::VideoDecoderSelection vds,
         SDL_Window* window,
@@ -69,6 +73,11 @@ bool FFmpegVideoDecoder::chooseDecoder(
         case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
             newRenderer = VTRendererFactory::createRenderer();
             break;
+#endif
+#ifdef Q_OS_UNIX
+        case AV_HWDEVICE_TYPE_VAAPI:
+            newRenderer = new VAAPIRenderer();
+            break;
 #endif
         default:
             continue;