diff --git a/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp b/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp index fad875dd..a0f63a38 100644 --- a/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp +++ b/app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp @@ -1,9 +1,21 @@ #include "eglimagefactory.h" -// Don't take a dependency on libdrm just for this constant +// Don't take a dependency on libdrm just for these constants #ifndef DRM_FORMAT_MOD_INVALID #define DRM_FORMAT_MOD_INVALID ((1ULL << 56) - 1) #endif +#ifndef DRM_FORMAT_MOD_LINEAR +#define DRM_FORMAT_MOD_LINEAR 0 +#endif +#ifndef fourcc_code +#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24)) +#endif +#ifndef DRM_FORMAT_R8 +#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ') +#endif +#ifndef DRM_FORMAT_GR88 +#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8') +#endif EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) : m_Renderer(renderer), @@ -11,7 +23,9 @@ EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) : m_eglCreateImage(nullptr), m_eglDestroyImage(nullptr), m_eglCreateImageKHR(nullptr), - m_eglDestroyImageKHR(nullptr) + m_eglDestroyImageKHR(nullptr), + m_eglQueryDmaBufFormatsEXT(nullptr), + m_eglQueryDmaBufModifiersEXT(nullptr) { } @@ -25,6 +39,10 @@ bool EglImageFactory::initializeEGL(EGLDisplay, } m_EGLExtDmaBuf = ext.isSupported("EGL_EXT_image_dma_buf_import_modifiers"); + if (m_EGLExtDmaBuf) { + m_eglQueryDmaBufFormatsEXT = (typeof(m_eglQueryDmaBufFormatsEXT))eglGetProcAddress("eglQueryDmaBufFormatsEXT"); + m_eglQueryDmaBufModifiersEXT = (typeof(m_eglQueryDmaBufModifiersEXT))eglGetProcAddress("eglQueryDmaBufModifiersEXT"); + } // NB: eglCreateImage() and eglCreateImageKHR() have slightly different definitions m_eglCreateImage = (typeof(m_eglCreateImage))eglGetProcAddress("eglCreateImage"); @@ -42,6 +60,8 @@ bool EglImageFactory::initializeEGL(EGLDisplay, return true; } +#ifdef HAVE_DRM + ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES); @@ -216,9 +236,11 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d return 1; } +#endif + #ifdef HAVE_LIBVA -ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[]) +ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { ssize_t count = 0; @@ -229,8 +251,8 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip for (size_t i = 0; i < vaFrame->num_layers; ++i) { const auto &layer = vaFrame->layers[i]; - // Max 31 attributes (1 key + 1 value for each) - const int EGL_ATTRIB_COUNT = 31 * 2; + // Max 33 attributes (1 key + 1 value for each) + const int EGL_ATTRIB_COUNT = 33 * 2; EGLAttrib attribs[EGL_ATTRIB_COUNT] = { EGL_LINUX_DRM_FOURCC_EXT, layer.drm_format, EGL_WIDTH, i == 0 ? frame->width : frame->width / 2, @@ -308,6 +330,61 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip } } + // For composed exports, add the YUV metadata + if (vaFrame->num_layers == 1) { + // Add colorspace metadata + switch (m_Renderer->getFrameColorspace(frame)) { + case COLORSPACE_REC_601: + attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT; + attribs[attribIndex++] = EGL_ITU_REC601_EXT; + break; + case COLORSPACE_REC_709: + attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT; + attribs[attribIndex++] = EGL_ITU_REC709_EXT; + break; + case COLORSPACE_REC_2020: + attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT; + attribs[attribIndex++] = EGL_ITU_REC2020_EXT; + break; + } + + // Add color range metadata + attribs[attribIndex++] = EGL_SAMPLE_RANGE_HINT_EXT; + attribs[attribIndex++] = m_Renderer->isFrameFullRange(frame) ? EGL_YUV_FULL_RANGE_EXT : EGL_YUV_NARROW_RANGE_EXT; + + // Add chroma siting metadata + switch (frame->chroma_location) { + case AVCHROMA_LOC_LEFT: + case AVCHROMA_LOC_TOPLEFT: + attribs[attribIndex++] = EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT; + attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_EXT; + break; + + case AVCHROMA_LOC_CENTER: + case AVCHROMA_LOC_TOP: + attribs[attribIndex++] = EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT; + attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_5_EXT; + break; + default: + break; + } + switch (frame->chroma_location) { + case AVCHROMA_LOC_TOPLEFT: + case AVCHROMA_LOC_TOP: + attribs[attribIndex++] = EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT; + attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_EXT; + break; + + case AVCHROMA_LOC_LEFT: + case AVCHROMA_LOC_CENTER: + attribs[attribIndex++] = EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT; + attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_5_EXT; + break; + default: + break; + } + } + // Terminate the attribute list attribs[attribIndex++] = EGL_NONE; SDL_assert(attribIndex <= EGL_ATTRIB_COUNT); @@ -341,6 +418,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip ++count; } + return count; fail: @@ -348,6 +426,77 @@ fail: return -1; } +bool EglImageFactory::supportsImportingFormat(EGLDisplay dpy, EGLint format) +{ + if (!m_eglQueryDmaBufFormatsEXT) { + // These are the standard formats used for importing separate layers of NV12. + // We will assume all EGL implementations can handle these. + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Assuming R8 and GR88 format support because eglQueryDmaBufFormatsEXT() is not supported"); + return format == DRM_FORMAT_R8 || format == DRM_FORMAT_GR88; + } + + // Get the number of formats + EGLint numFormats; + if (!m_eglQueryDmaBufFormatsEXT(dpy, 0, nullptr, &numFormats)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "eglQueryDmaBufFormatsEXT() #1 failed: %d", eglGetError()); + return false; + } + + EGLint formats[numFormats]; + if (!m_eglQueryDmaBufFormatsEXT(dpy, numFormats, formats, &numFormats)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "eglQueryDmaBufFormatsEXT() #2 failed: %d", eglGetError()); + return false; + } + + for (EGLint i = 0; i < numFormats; i++) { + if (format == formats[i]) { + return true; + } + } + + return false; +} + +bool EglImageFactory::supportsImportingModifier(EGLDisplay dpy, EGLint format, EGLuint64KHR modifier) +{ + // We assume linear and no modifiers are always supported + if (modifier == DRM_FORMAT_MOD_LINEAR || modifier == DRM_FORMAT_MOD_INVALID) { + return true; + } + + if (!m_eglQueryDmaBufModifiersEXT) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Assuming linear modifier support because eglQueryDmaBufModifiersEXT() is not supported"); + return false; + } + + // Get the number of modifiers + EGLint numModifiers; + if (!m_eglQueryDmaBufModifiersEXT(dpy, format, 0, nullptr, nullptr, &numModifiers)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "eglQueryDmaBufModifiersEXT() #1 failed: %d", eglGetError()); + return false; + } + + EGLuint64KHR modifiers[numModifiers]; + if (!m_eglQueryDmaBufModifiersEXT(dpy, format, numModifiers, modifiers, nullptr, &numModifiers)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "eglQueryDmaBufModifiersEXT() #2 failed: %d", eglGetError()); + return false; + } + + for (EGLint i = 0; i < numModifiers; i++) { + if (modifier == modifiers[i]) { + return true; + } + } + + return false; +} + #endif void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { @@ -361,4 +510,5 @@ void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLAN } } } + memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES); } diff --git a/app/streaming/video/ffmpeg-renderers/eglimagefactory.h b/app/streaming/video/ffmpeg-renderers/eglimagefactory.h index 7a4dc4d3..3cd90e4a 100644 --- a/app/streaming/video/ffmpeg-renderers/eglimagefactory.h +++ b/app/streaming/video/ffmpeg-renderers/eglimagefactory.h @@ -14,12 +14,18 @@ class EglImageFactory public: EglImageFactory(IFFmpegRenderer* renderer); bool initializeEGL(EGLDisplay, const EGLExtensions &ext); + +#ifdef HAVE_DRM ssize_t exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); +#endif #ifdef HAVE_LIBVA ssize_t exportVAImages(AVFrame* frame, VADRMPRIMESurfaceDescriptor* vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); #endif + bool supportsImportingFormat(EGLDisplay dpy, EGLint format); + bool supportsImportingModifier(EGLDisplay dpy, EGLint format, EGLuint64KHR modifier); + void freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]); private: @@ -29,4 +35,6 @@ private: PFNEGLDESTROYIMAGEPROC m_eglDestroyImage; PFNEGLCREATEIMAGEKHRPROC m_eglCreateImageKHR; PFNEGLDESTROYIMAGEKHRPROC m_eglDestroyImageKHR; + PFNEGLQUERYDMABUFFORMATSEXTPROC m_eglQueryDmaBufFormatsEXT; + PFNEGLQUERYDMABUFMODIFIERSEXTPROC m_eglQueryDmaBufModifiersEXT; }; diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp index 34359a98..8ff4173b 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp +++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp @@ -18,12 +18,12 @@ VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass) m_BlacklistedForDirectRendering(false), m_OverlayMutex(nullptr) #ifdef HAVE_EGL - , m_EglImageFactory(this) + , m_EglExportType(EglExportType::Unknown), + m_EglImageFactory(this) #endif { #ifdef HAVE_EGL - m_PrimeDescriptor.num_layers = 0; - m_PrimeDescriptor.num_objects = 0; + SDL_zero(m_PrimeDescriptor); #endif SDL_zero(m_OverlayImage); @@ -795,12 +795,11 @@ VAAPIRenderer::renderFrame(AVFrame* frame) // Ensure that vaExportSurfaceHandle() is supported by the VA-API driver bool -VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) { +VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag, VADRMPRIMESurfaceDescriptor* descriptor) { AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data; AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx; VASurfaceID surfaceId; VAStatus st; - VADRMPRIMESurfaceDescriptor descriptor; VASurfaceAttrib attrs[2]; int attributeCount = 0; @@ -848,7 +847,7 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) { surfaceId, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, VA_EXPORT_SURFACE_READ_ONLY | layerTypeFlag, - &descriptor); + descriptor); vaDestroySurfaces(vaDeviceContext->display, &surfaceId, 1); @@ -858,8 +857,9 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) { return false; } - for (size_t i = 0; i < descriptor.num_objects; ++i) { - close(descriptor.objects[i].fd); + for (size_t i = 0; i < descriptor->num_objects; ++i) { + close(descriptor->objects[i].fd); + descriptor->objects[i].fd = -1; } SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, @@ -874,33 +874,122 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) { bool VAAPIRenderer::canExportEGL() { - // Our EGL export logic requires exporting separate layers - return canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS); + VADRMPRIMESurfaceDescriptor descriptor; + + return (qgetenv("VAAPI_EGL_SEPARATE_LAYERS") != "1" && canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor)) || + canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS, &descriptor); } AVPixelFormat VAAPIRenderer::getEGLImagePixelFormat() { - return (m_VideoFormat & VIDEO_FORMAT_MASK_10BIT) ? - AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; + switch (m_EglExportType) { + case EglExportType::Separate: + return (m_VideoFormat & VIDEO_FORMAT_MASK_10BIT) ? + AV_PIX_FMT_P010 : AV_PIX_FMT_NV12; + + case EglExportType::Composed: + // This tells EGLRenderer to treat the EGLImage as a single opaque texture + return AV_PIX_FMT_DRM_PRIME; + + case EglExportType::Unknown: + SDL_assert(m_EglExportType != EglExportType::Unknown); + break; + } + + return AV_PIX_FMT_NONE; } bool VAAPIRenderer::initializeEGL(EGLDisplay dpy, const EGLExtensions &ext) { - return m_EglImageFactory.initializeEGL(dpy, ext); + VADRMPRIMESurfaceDescriptor descriptor; + + if (!m_EglImageFactory.initializeEGL(dpy, ext)) { + return false; + } + + // Prefer exporting composed images absent a user override or lack of support for exporting or importing + if (qgetenv("VAAPI_EGL_SEPARATE_LAYERS") == "1") { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Exporting separate layers due to environment variable override"); + m_EglExportType = EglExportType::Separate; + } + else if (!canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor)) { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Exporting separate layers due to lack of support for VA_EXPORT_SURFACE_COMPOSED_LAYERS"); + m_EglExportType = EglExportType::Separate; + } + else if (!m_EglImageFactory.supportsImportingFormat(dpy, descriptor.layers[0].drm_format)) { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Exporting separate layers due to lack of support for importing format: %08x", descriptor.layers[0].drm_format); + m_EglExportType = EglExportType::Separate; + } + else if (!m_EglImageFactory.supportsImportingModifier(dpy, descriptor.layers[0].drm_format, descriptor.objects[0].drm_format_modifier)) { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Exporting separate layers due to lack of support for importing format and modifier: %08x %016lx", + descriptor.layers[0].drm_format, + descriptor.objects[0].drm_format_modifier); + m_EglExportType = EglExportType::Separate; + } + else { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Exporting composed layers with format and modifier: %08x %016lx", + descriptor.layers[0].drm_format, + descriptor.objects[0].drm_format_modifier); + m_EglExportType = EglExportType::Composed; + } + + // Let's probe for EGL import support on separate layers too, but only warn if it's not supported + if (m_EglExportType == EglExportType::Separate) { + if (!canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS, &descriptor)) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Exporting separate layers is not supported by the VAAPI driver"); + return false; + } + + for (int i = 0; i < descriptor.num_layers; i++) { + if (!m_EglImageFactory.supportsImportingFormat(dpy, descriptor.layers[i].drm_format)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGL implementation lacks support for importing format: %08x", descriptor.layers[0].drm_format); + } + else if (!m_EglImageFactory.supportsImportingModifier(dpy, descriptor.layers[i].drm_format, + descriptor.objects[descriptor.layers[i].object_index[0]].drm_format_modifier)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "EGL implementation lacks support for importing format and modifier: %08x %016lx", + descriptor.layers[i].drm_format, + descriptor.objects[descriptor.layers[i].object_index[0]].drm_format_modifier); + } + } + } + + return true; } ssize_t VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { ssize_t count; + uint32_t exportFlags = VA_EXPORT_SURFACE_READ_ONLY; + + switch (m_EglExportType) { + case EglExportType::Separate: + exportFlags |= VA_EXPORT_SURFACE_SEPARATE_LAYERS; + break; + case EglExportType::Composed: + exportFlags |= VA_EXPORT_SURFACE_COMPOSED_LAYERS; + break; + case EglExportType::Unknown: + SDL_assert(m_EglExportType != EglExportType::Unknown); + return -1; + } + auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data; AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx; - VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3]; + VAStatus st = vaExportSurfaceHandle(vaDeviceContext->display, surface_id, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, - VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS, + exportFlags, &m_PrimeDescriptor); if (st != VA_STATUS_SUCCESS) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, @@ -948,7 +1037,8 @@ VAAPIRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) { bool VAAPIRenderer::canExportDrmPrime() { // Our DRM renderer requires composed layers - return canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS); + VADRMPRIMESurfaceDescriptor descriptor; + return canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor); } bool VAAPIRenderer::mapDrmPrimeFrame(AVFrame* frame, AVDRMFrameDescriptor* drmDescriptor) diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h index b7fce359..3fabdb11 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.h +++ b/app/streaming/video/ffmpeg-renderers/vaapi.h @@ -69,7 +69,7 @@ private: void renderOverlay(VADisplay display, VASurfaceID surface, Overlay::OverlayType type); #if defined(HAVE_EGL) || defined(HAVE_DRM) - bool canExportSurfaceHandle(int layerTypeFlag); + bool canExportSurfaceHandle(int layerTypeFlag, VADRMPRIMESurfaceDescriptor* descriptor); #endif int m_DecoderSelectionPass; @@ -96,6 +96,11 @@ private: int m_DisplayHeight; #ifdef HAVE_EGL + enum class EglExportType { + Unknown, + Separate, + Composed + } m_EglExportType; VADRMPRIMESurfaceDescriptor m_PrimeDescriptor; EglImageFactory m_EglImageFactory; #endif