Use SDL to render on Wayland for VAAPI

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -41,6 +41,11 @@ public:
         return true;
     }
 
+    virtual bool isDirectRenderingSupported() {
+        // The renderer can render directly to the display
+        return true;
+    }
+
     // IOverlayRenderer
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override {
         // Nothing

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -76,8 +76,6 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
     }
     else if (info.subsystem == SDL_SYSWM_WAYLAND) {
 #ifdef HAVE_LIBVA_WAYLAND
-        m_WaylandSurface = info.info.wl.surface;
-        m_WaylandDisplay = info.info.wl.display;
         vaDeviceContext->display = vaGetDisplayWl(info.info.wl.display);
         if (!vaDeviceContext->display) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@@ -171,6 +169,14 @@ VAAPIRenderer::needsTestFrame()
     return true;
 }
 
+bool
+VAAPIRenderer::isDirectRenderingSupported()
+{
+    // Many Wayland renderers don't support YUV surfaces, so use
+    // another frontend renderer to draw our frames.
+    return m_WindowSystem == SDL_SYSWM_X11;
+}
+
 void
 VAAPIRenderer::renderFrame(AVFrame* frame)
 {
@@ -201,27 +207,10 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
 #endif
     }
     else if (m_WindowSystem == SDL_SYSWM_WAYLAND) {
-#ifdef HAVE_LIBVA_WAYLAND
+        // We don't support direct rendering on Wayland, so we should
-        struct wl_buffer* buffer;
+        // never get called there. Many common Wayland compositors don't
-        VAStatus status;
+        // support YUV surfaces, so direct rendering would fail.
-
+        SDL_assert(false);
-        status = vaGetSurfaceBufferWl(vaDeviceContext->display,
-                                      surface,
-                                      VA_FRAME_PICTURE,
-                                      &buffer);
-        if (status == VA_STATUS_SUCCESS) {
-            wl_surface_attach(m_WaylandSurface, buffer, 0, 0);
-            wl_surface_damage(m_WaylandSurface, dst.x, dst.y, dst.w, dst.h);
-
-            wl_display_flush(m_WaylandDisplay);
-            wl_surface_commit(m_WaylandSurface);
-        }
-        else {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "vaGetSurfaceBufferWl failed(): %d",
-                         status);
-        }
-#endif
     }
     else {
         // We don't accept anything else in initialize().

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -34,6 +34,7 @@ public:
     virtual bool prepareDecoderContext(AVCodecContext* context) override;
     virtual void renderFrame(AVFrame* frame) override;
     virtual bool needsTestFrame() override;
+    virtual bool isDirectRenderingSupported() override;
 
 private:
     int m_WindowSystem;
@@ -43,11 +44,6 @@ private:
     Window m_XWindow;
 #endif
 
-#ifdef HAVE_LIBVA_WAYLAND
-    struct wl_surface* m_WaylandSurface;
-    struct wl_display* m_WaylandDisplay;
-#endif
-
     int m_VideoWidth;
     int m_VideoHeight;
     int m_DisplayWidth;

app/streaming/video/ffmpeg.cpp

@@ -137,7 +137,18 @@ void FFmpegVideoDecoder::reset()
 
 bool FFmpegVideoDecoder::createFrontendRenderer(PDECODER_PARAMETERS params)
 {
+    if (m_BackendRenderer->isDirectRenderingSupported()) {
+        // The backend renderer can render to the display
         m_FrontendRenderer = m_BackendRenderer;
+    }
+    else {
+        // The backend renderer cannot directly render to the display, so
+        // we will create an SDL renderer to draw the frames.
+        m_FrontendRenderer = new SdlRenderer();
+        if (!m_FrontendRenderer->initialize(params)) {
+            return false;
+        }
+    }
 
     // Determine whether the frontend renderer prefers frame pacing
     auto vsyncConstraint = m_FrontendRenderer->getFramePacingConstraint();