Deprioritize VAAPI for Gallium and Nvidia drivers and work around RFI latency bug

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -12,8 +12,9 @@
 #include <unistd.h>
 #include <fcntl.h>
 
-VAAPIRenderer::VAAPIRenderer()
+VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass)
-    : m_HwContext(nullptr),
+    : m_DecoderSelectionPass(decoderSelectionPass),
+      m_HwContext(nullptr),
       m_BlacklistedForDirectRendering(false),
       m_OverlayMutex(nullptr)
 {
@@ -193,11 +194,7 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
                 status = vaInitialize(vaDeviceContext->display, &major, &minor);
             }
 
-            if (status != VA_STATUS_SUCCESS
+            if (status != VA_STATUS_SUCCESS && (m_WindowSystem != SDL_SYSWM_X11 || m_DecoderSelectionPass > 0)) {
-        #if defined(HAVE_CUDA) || defined(HAVE_LIBVDPAU)
-                    && m_WindowSystem != SDL_SYSWM_X11
-        #endif
-                    ) {
                 // The unofficial nvidia VAAPI driver over NVDEC/CUDA works well on Wayland,
                 // but we'd rather use CUDA for XWayland and VDPAU for regular X11.
                 qputenv("LIBVA_DRIVER_NAME", "nvidia");
@@ -265,29 +262,32 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
                 "Driver: %s",
                 vendorString ? vendorString : "<unknown>");
 
-    // Older versions of the Gallium VAAPI driver have a nasty memory leak that
+    // The Snap (core22) and Focal/Jammy Mesa drivers have a bug that causes
-    // causes memory to be leaked for each submitted frame. I believe this is
+    // a large amount of video latency when using more than one reference frame
-    // resolved in the libva2 drivers (VAAPI 1.x).
+    // and severe rendering glitches on my Ryzen 3300U system.
-    // If we're using Wayland, we have no choice but to use VAAPI because VDPAU
+    m_HasRfiLatencyBug = vendorStr.contains("Gallium", Qt::CaseInsensitive) && qgetenv("IGNORE_RFI_LATENCY_BUG") != "1";
-    // is only supported under X11 or XWayland.
+    if (m_HasRfiLatencyBug) {
-    if (major == 0 && qgetenv("FORCE_VAAPI") != "1" && !WMUtils::isRunningWayland()) {
-        if (vendorStr.contains("AMD", Qt::CaseInsensitive) ||
-                vendorStr.contains("Radeon", Qt::CaseInsensitive)) {
-            // Fail and let VDPAU pick this up
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
-                        "Avoiding VAAPI on AMD driver. Set FORCE_VAAPI=1 to override.");
+                    "VAAPI driver is affected by RFI latency bug");
-            return false;
-        }
     }
 
-#if defined(HAVE_CUDA) || defined(HAVE_LIBVDPAU)
+    // Older versions of the Gallium VAAPI driver have a nasty memory leak that
-    if (m_WindowSystem == SDL_SYSWM_X11 && qgetenv("FORCE_VAAPI") != "1" && vendorStr.contains("VA-API NVDEC", Qt::CaseInsensitive)) {
+    // causes memory to be leaked for each submitted frame. I believe this is
-        // Prefer CUDA for XWayland and VDPAU for regular X11.
+    // resolved in the libva2 drivers (VAAPI 1.x). We will try to use VDPAU
+    // instead for old VAAPI versions or drivers affected by the RFI latency bug.
+    if (m_DecoderSelectionPass == 0 && (major == 0 || m_HasRfiLatencyBug) && qgetenv("FORCE_VAAPI") != "1" && vendorStr.contains("Gallium", Qt::CaseInsensitive)) {
+        // Fail and let VDPAU pick this up
         SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
-                    "Avoiding VAAPI for NVIDIA driver on X11/XWayland. Set FORCE_VAAPI=1 to override.");
+                    "Deprioritizing VAAPI on Gallium driver. Set FORCE_VAAPI=1 to override.");
+        return false;
+    }
+
+    if (m_DecoderSelectionPass == 0 && qgetenv("FORCE_VAAPI") != "1" && vendorStr.contains("VA-API NVDEC", Qt::CaseInsensitive)) {
+        // Prefer CUDA for XWayland and VDPAU for regular X11.
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Deprioritizing VAAPI for NVIDIA driver on X11/XWayland. Set FORCE_VAAPI=1 to override.");
         return false;
     }
-#endif
 
     if (WMUtils::isRunningWayland()) {
         // The iHD VAAPI driver can initialize on XWayland but it crashes in
@@ -479,7 +479,13 @@ int VAAPIRenderer::getDecoderColorspace()
 
 int VAAPIRenderer::getDecoderCapabilities()
 {
-    return CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC;
+    int caps = 0;
+
+    if (!m_HasRfiLatencyBug) {
+        caps |= CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC;
+    }
+
+    return caps;
 }
 
 void VAAPIRenderer::notifyOverlayUpdated(Overlay::OverlayType type)

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -34,7 +34,7 @@ extern "C" {
 class VAAPIRenderer : public IFFmpegRenderer
 {
 public:
-    VAAPIRenderer();
+    VAAPIRenderer(int decoderSelectionPass);
     virtual ~VAAPIRenderer() override;
     virtual bool initialize(PDECODER_PARAMETERS params) override;
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
@@ -67,9 +67,11 @@ private:
     bool canExportSurfaceHandle(int layerTypeFlag);
 #endif
 
+    int m_DecoderSelectionPass;
     int m_WindowSystem;
     AVBufferRef* m_HwContext;
     bool m_BlacklistedForDirectRendering;
+    bool m_HasRfiLatencyBug;
 
     SDL_mutex* m_OverlayMutex;
     VAImageFormat m_OverlayFormat;

app/streaming/video/ffmpeg.cpp

@@ -670,7 +670,7 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
 #endif
 #ifdef HAVE_LIBVA
         case AV_HWDEVICE_TYPE_VAAPI:
-            return new VAAPIRenderer();
+            return new VAAPIRenderer(pass);
 #endif
 #ifdef HAVE_LIBVDPAU
         case AV_HWDEVICE_TYPE_VDPAU:
@@ -700,6 +700,14 @@ IFFmpegRenderer* FFmpegVideoDecoder::createHwAccelRenderer(const AVCodecHWConfig
             return new DXVA2Renderer(pass);
         case AV_HWDEVICE_TYPE_D3D11VA:
             return new D3D11VARenderer(pass);
+#endif
+#ifdef HAVE_LIBVA
+        case AV_HWDEVICE_TYPE_VAAPI:
+            return new VAAPIRenderer(pass);
+#endif
+#ifdef HAVE_LIBVDPAU
+        case AV_HWDEVICE_TYPE_VDPAU:
+            return new VDPAURenderer();
 #endif
         default:
             return nullptr;