From c3895f06c015086dd3313a1a084c07dc9380772a Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Thu, 4 Feb 2021 19:39:18 -0600 Subject: [PATCH] Fix handling of preferred/compatible pixel formats with EGLRenderer and VAAPI/DRM backends --- app/streaming/video/ffmpeg-renderers/drm.cpp | 4 ++ app/streaming/video/ffmpeg-renderers/drm.h | 1 + .../video/ffmpeg-renderers/eglvid.cpp | 68 ++++++++----------- app/streaming/video/ffmpeg-renderers/eglvid.h | 3 +- .../video/ffmpeg-renderers/renderer.h | 8 ++- .../video/ffmpeg-renderers/vaapi.cpp | 4 ++ app/streaming/video/ffmpeg-renderers/vaapi.h | 1 + 7 files changed, 47 insertions(+), 42 deletions(-) diff --git a/app/streaming/video/ffmpeg-renderers/drm.cpp b/app/streaming/video/ffmpeg-renderers/drm.cpp index 7608b8d0..70aab00c 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.cpp +++ b/app/streaming/video/ffmpeg-renderers/drm.cpp @@ -386,6 +386,10 @@ bool DrmRenderer::canExportEGL() { return true; } +AVPixelFormat DrmRenderer::getEGLImagePixelFormat() { + return AV_PIX_FMT_NV12; +} + bool DrmRenderer::initializeEGL(EGLDisplay, const EGLExtensions &ext) { if (!ext.isSupported("EGL_EXT_image_dma_buf_import")) { diff --git a/app/streaming/video/ffmpeg-renderers/drm.h b/app/streaming/video/ffmpeg-renderers/drm.h index 1adbdfa3..5762a769 100644 --- a/app/streaming/video/ffmpeg-renderers/drm.h +++ b/app/streaming/video/ffmpeg-renderers/drm.h @@ -18,6 +18,7 @@ public: virtual bool isDirectRenderingSupported() override; #ifdef HAVE_EGL virtual bool canExportEGL() override; + virtual AVPixelFormat getEGLImagePixelFormat() override; virtual bool initializeEGL(EGLDisplay dpy, const EGLExtensions &ext) override; virtual ssize_t exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) override; virtual void freeEGLImages(EGLDisplay dpy, EGLImage[EGL_MAX_PLANES]) override; diff --git a/app/streaming/video/ffmpeg-renderers/eglvid.cpp b/app/streaming/video/ffmpeg-renderers/eglvid.cpp index 4b687abd..c6cccea0 100644 --- a/app/streaming/video/ffmpeg-renderers/eglvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/eglvid.cpp @@ -59,7 +59,7 @@ SDL_Window* EGLRenderer::s_LastFailedWindow = nullptr; EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer) : - m_SwPixelFormat(AV_PIX_FMT_NONE), + m_EGLImagePixelFormat(AV_PIX_FMT_NONE), m_EGLDisplay(EGL_NO_DISPLAY), m_Textures{0}, m_OverlayTextures{0}, @@ -153,16 +153,16 @@ void EGLRenderer::notifyOverlayUpdated(Overlay::OverlayType type) } } -bool EGLRenderer::isPixelFormatSupported(int, AVPixelFormat pixelFormat) +bool EGLRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat) { - // Remember to keep this in sync with EGLRenderer::renderFrame()! - switch (pixelFormat) - { - case AV_PIX_FMT_NV12: - return true; - default: - return false; - } + // Pixel format support should be determined by the backend renderer + return m_Backend->isPixelFormatSupported(videoFormat, pixelFormat); +} + +AVPixelFormat EGLRenderer::getPreferredPixelFormat(int videoFormat) +{ + // Pixel format preference should be determined by the backend renderer + return m_Backend->getPreferredPixelFormat(videoFormat); } void EGLRenderer::renderOverlay(Overlay::OverlayType type) @@ -367,10 +367,10 @@ bool EGLRenderer::compileShaders() { SDL_assert(!m_ShaderProgram); SDL_assert(!m_OverlayShaderProgram); - SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE); + SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE); // XXX: TODO: other formats - SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12); + SDL_assert(m_EGLImagePixelFormat == AV_PIX_FMT_NV12); m_ShaderProgram = compileShader("egl.vert", "egl.frag"); if (!m_ShaderProgram) { @@ -721,39 +721,29 @@ void EGLRenderer::renderFrame(AVFrame* frame) return; } - if (frame->hw_frames_ctx != nullptr) { - // Find the native read-back format and load the shader - if (m_SwPixelFormat == AV_PIX_FMT_NONE) { - auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data; + // Find the native read-back format and load the shaders + if (m_EGLImagePixelFormat == AV_PIX_FMT_NONE) { + m_EGLImagePixelFormat = m_Backend->getEGLImagePixelFormat(); + EGL_LOG(Info, "EGLImage pixel format: %d", m_EGLImagePixelFormat); - m_SwPixelFormat = hwFrameCtx->sw_format; - SDL_assert(m_SwPixelFormat != AV_PIX_FMT_NONE); + SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE); - EGL_LOG(Info, "Selected read-back format: %d", m_SwPixelFormat); + m_ColorSpace = frame->colorspace; + m_ColorFull = frame->color_range == AVCOL_RANGE_JPEG; - // XXX: TODO: Handle other pixel formats - SDL_assert(m_SwPixelFormat == AV_PIX_FMT_NV12); - m_ColorSpace = frame->colorspace; - m_ColorFull = frame->color_range == AVCOL_RANGE_JPEG; - - if (!specialize()) { - m_SwPixelFormat = AV_PIX_FMT_NONE; - return; - } - } - - ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs); - if (plane_count < 0) + if (!specialize()) { + m_EGLImagePixelFormat = AV_PIX_FMT_NONE; return; - for (ssize_t i = 0; i < plane_count; ++i) { - glActiveTexture(GL_TEXTURE0 + i); - glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]); - m_glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]); } - } else { - // TODO: load texture for SW decoding ? - EGL_LOG(Error, "EGL rendering only supports hw frames"); + } + + ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs); + if (plane_count < 0) return; + for (ssize_t i = 0; i < plane_count; ++i) { + glActiveTexture(GL_TEXTURE0 + i); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]); + m_glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]); } glClear(GL_COLOR_BUFFER_BIT); diff --git a/app/streaming/video/ffmpeg-renderers/eglvid.h b/app/streaming/video/ffmpeg-renderers/eglvid.h index e6244e55..70e16507 100644 --- a/app/streaming/video/ffmpeg-renderers/eglvid.h +++ b/app/streaming/video/ffmpeg-renderers/eglvid.h @@ -14,6 +14,7 @@ public: virtual void renderFrame(AVFrame* frame) override; virtual void notifyOverlayUpdated(Overlay::OverlayType) override; virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) override; + virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) override; private: @@ -29,7 +30,7 @@ private: int m_ViewportWidth; int m_ViewportHeight; - int m_SwPixelFormat; + AVPixelFormat m_EGLImagePixelFormat; void *m_EGLDisplay; unsigned m_Textures[EGL_MAX_PLANES]; unsigned m_OverlayTextures[Overlay::OverlayMax]; diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h index 1fb048e2..9a11a2af 100644 --- a/app/streaming/video/ffmpeg-renderers/renderer.h +++ b/app/streaming/video/ffmpeg-renderers/renderer.h @@ -101,7 +101,7 @@ public: return true; } - virtual enum AVPixelFormat getPreferredPixelFormat(int videoFormat) { + virtual AVPixelFormat getPreferredPixelFormat(int videoFormat) { if (videoFormat == VIDEO_FORMAT_H265_MAIN10) { // 10-bit YUV 4:2:0 return AV_PIX_FMT_P010; @@ -112,7 +112,7 @@ public: } } - virtual bool isPixelFormatSupported(int videoFormat, enum AVPixelFormat pixelFormat) { + virtual bool isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat) { // By default, we only support the preferred pixel format return getPreferredPixelFormat(videoFormat) == pixelFormat; } @@ -128,6 +128,10 @@ public: return false; } + virtual AVPixelFormat getEGLImagePixelFormat() { + return AV_PIX_FMT_NONE; + } + virtual bool initializeEGL(EGLDisplay, const EGLExtensions &) { return false; diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp index 55c42eb3..a065359b 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp +++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp @@ -483,6 +483,10 @@ VAAPIRenderer::canExportEGL() { return true; } +AVPixelFormat VAAPIRenderer::getEGLImagePixelFormat() { + return AV_PIX_FMT_NV12; +} + bool VAAPIRenderer::initializeEGL(EGLDisplay, const EGLExtensions &ext) { diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h index 032fd4e9..08107398 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.h +++ b/app/streaming/video/ffmpeg-renderers/vaapi.h @@ -44,6 +44,7 @@ public: virtual int getDecoderColorspace() override; #ifdef HAVE_EGL virtual bool canExportEGL() override; + virtual AVPixelFormat getEGLImagePixelFormat() override; virtual bool initializeEGL(EGLDisplay dpy, const EGLExtensions &ext) override; virtual ssize_t exportEGLImages(AVFrame *frame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) override; virtual void freeEGLImages(EGLDisplay dpy, EGLImage[EGL_MAX_PLANES]) override;