Don't use VAAPI drivers that lack VPP support

intel-media-va-driver (free) is one example of such a driver which
is apparently missing the shaders to perform colorspace conversion.

app/streaming/video/ffmpeg-renderers/vaapi.cpp

@@ -38,68 +38,87 @@ VAAPIRenderer::~VAAPIRenderer()
 }
 
 bool
-VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
+VAAPIRenderer::validateDriver(VADisplay display)
+{
+    const char* vendorString = vaQueryVendorString(display);
+    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                "Validating VAAPI driver: %s",
+                vendorString ? vendorString : "<unknown>");
+
+    if (isDirectRenderingSupported()) {
+        int entrypointCount;
+        VAEntrypoint entrypoints[vaMaxNumEntrypoints(display)];
+        VAStatus status = vaQueryConfigEntrypoints(display, VAProfileNone, entrypoints, &entrypointCount);
+        if (status == VA_STATUS_SUCCESS) {
+            for (int i = 0; i < entrypointCount; i++) {
+                // Without VAEntrypointVideoProc support, the driver will crash inside vaPutSurface()
+                if (entrypoints[i] == VAEntrypointVideoProc) {
+                    SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                                "VAEntrypointVideoProc is supported");
+                    return true;
+                }
+            }
+        }
+
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                     "VAAPI driver doesn't support VAEntrypointVideoProc!");
+        return false;
+    }
+    else {
+        // Indirect rendering can use any driver
+        return true;
+    }
+}
+
+VADisplay
+VAAPIRenderer::openDisplay(SDL_Window* window)
 {
-    int err;
     SDL_SysWMinfo info;
-
+    VADisplay display;
-    m_VideoWidth = params->width;
-    m_VideoHeight = params->height;
-
-    SDL_GetWindowSize(params->window, &m_DisplayWidth, &m_DisplayHeight);
 
     SDL_VERSION(&info.version);
 
-    if (!SDL_GetWindowWMInfo(params->window, &info)) {
+    if (!SDL_GetWindowWMInfo(window, &info)) {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "SDL_GetWindowWMInfo() failed: %s",
                      SDL_GetError());
-        return false;
+        return nullptr;
     }
 
-    m_HwContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
-    if (!m_HwContext) {
-        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                    "Failed to allocate VAAPI context");
-        return false;
-    }
-
-    AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
-    AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
-
     m_WindowSystem = info.subsystem;
     if (info.subsystem == SDL_SYSWM_X11) {
-#ifdef HAVE_LIBVA_X11
         m_XWindow = info.info.x11.window;
-        vaDeviceContext->display = vaGetDisplay(info.info.x11.display);
+#ifdef HAVE_LIBVA_X11
-        if (!vaDeviceContext->display) {
+        display = vaGetDisplay(info.info.x11.display);
+        if (display == nullptr) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                          "Unable to open X11 display for VAAPI");
-            return false;
+            return nullptr;
         }
 #else
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Moonlight not compiled with VAAPI X11 support!");
-        return false;
+        return nullptr;
 #endif
     }
     else if (info.subsystem == SDL_SYSWM_WAYLAND) {
 #ifdef HAVE_LIBVA_WAYLAND
-        vaDeviceContext->display = vaGetDisplayWl(info.info.wl.display);
+        display = vaGetDisplayWl(info.info.wl.display);
-        if (!vaDeviceContext->display) {
+        if (display == nullptr) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                          "Unable to open Wayland display for VAAPI");
-            return false;
+            return nullptr;
         }
 #else
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Moonlight not compiled with VAAPI Wayland support!");
-        return false;
+        return nullptr;
 #endif
     }
     // TODO: Upstream a better solution for SDL_GetWindowWMInfo on KMSDRM
     else if (strcmp(SDL_GetCurrentVideoDriver(), "KMSDRM") == 0) {
 #ifdef HAVE_LIBVA_DRM
+        if (m_DrmFd < 0) {
             const char* device = SDL_getenv("DRM_DEV");
 
             if (device == nullptr) {
@@ -115,25 +134,53 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
                 SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                              "Failed to open DRM device: %d",
                              errno);
-            return false;
+                return nullptr;
+            }
         }
 
-        vaDeviceContext->display = vaGetDisplayDRM(m_DrmFd);
+        display = vaGetDisplayDRM(m_DrmFd);
-        if (!vaDeviceContext->display) {
+        if (display == nullptr) {
             SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                          "Unable to open DRM display for VAAPI");
-            return false;
+            return nullptr;
         }
 #else
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Moonlight not compiled with VAAPI DRM support!");
-        return false;
+        return nullptr;
 #endif
     }
     else {
         SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
                      "Unsupported VAAPI rendering subsystem: %d",
                      info.subsystem);
+        return nullptr;
+    }
+}
+
+bool
+VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
+{
+    int err;
+
+    m_VideoWidth = params->width;
+    m_VideoHeight = params->height;
+
+    SDL_GetWindowSize(params->window, &m_DisplayWidth, &m_DisplayHeight);
+
+    m_HwContext = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VAAPI);
+    if (!m_HwContext) {
+        SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                    "Failed to allocate VAAPI context");
+        return false;
+    }
+
+    AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
+    AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
+
+    vaDeviceContext->display = openDisplay(params->window);
+    if (vaDeviceContext->display == nullptr) {
+        // openDisplay() logs the error
         return false;
     }
 
@@ -143,6 +190,11 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
 
     for (;;) {
         status = vaInitialize(vaDeviceContext->display, &major, &minor);
+        if (status == VA_STATUS_SUCCESS && !validateDriver(vaDeviceContext->display)) {
+            vaTerminate(vaDeviceContext->display);
+            vaDeviceContext->display = openDisplay(params->window);
+            status = VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+        }
         if (status != VA_STATUS_SUCCESS && qEnvironmentVariableIsEmpty("LIBVA_DRIVER_NAME")) {
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                         "Trying fallback VAAPI driver names");
@@ -157,6 +209,11 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
                 // always seem to be the case for some reason.
                 qputenv("LIBVA_DRIVER_NAME", "iHD");
                 status = vaInitialize(vaDeviceContext->display, &major, &minor);
+                if (status == VA_STATUS_SUCCESS && !validateDriver(vaDeviceContext->display)) {
+                    vaTerminate(vaDeviceContext->display);
+                    vaDeviceContext->display = openDisplay(params->window);
+                    status = VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+                }
             }
 
             if (status != VA_STATUS_SUCCESS) {
@@ -165,6 +222,11 @@ VAAPIRenderer::initialize(PDECODER_PARAMETERS params)
                 // explicitly try i965 to handle this case.
                 qputenv("LIBVA_DRIVER_NAME", "i965");
                 status = vaInitialize(vaDeviceContext->display, &major, &minor);
+                if (status == VA_STATUS_SUCCESS && !validateDriver(vaDeviceContext->display)) {
+                    vaTerminate(vaDeviceContext->display);
+                    vaDeviceContext->display = openDisplay(params->window);
+                    status = VA_STATUS_ERROR_UNSUPPORTED_PROFILE;
+                }
             }
 
             if (status != VA_STATUS_SUCCESS) {

app/streaming/video/ffmpeg-renderers/vaapi.h

@@ -41,6 +41,9 @@ public:
     virtual int getDecoderColorspace() override;
 
 private:
+    bool validateDriver(VADisplay display);
+    VADisplay openDisplay(SDL_Window* window);
+
     int m_WindowSystem;
     AVBufferRef* m_HwContext;
     int m_DrmFd;