Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2025-03-05 07:07:18 +00:00
Code Issues Projects Releases Packages Wiki Activity
moonlight-qt/app/streaming/video/ffmpeg-renderers/eglvid.cpp
Cameron Gutman e64b15d2a7 Rework how we deal with window re-creation for EGLRenderer 
Previously we opted to just lie to SDL and tell it that the current
GL context is actually GLES 3.0 when it wasn't. This meant that
we avoided our window being recreated for GLES usage on Wayland
and KMSDRM, but that meant our 10-bit color change didn't apply
either. I suspect this hackery is what led EGLRenderer to get
stuck in a state where SDL_CreateRenderer() always failed.

Now SDL will recreate our window once to configure a compatible GL
API for our renderer. If that fails, we'll move on to a different
renderer (which may recreate the window again).
2022-01-27 21:49:03 -06:00

933 lines
33 KiB
C++
Raw Blame History

// vim: noai:ts=4:sw=4:softtabstop=4:expandtab
#include "eglvid.h"
#include "path.h"
#include "streaming/session.h"
#include "streaming/streamutils.h"
#include <QDir>
#include <Limelight.h>
#include <unistd.h>
#include <SDL_egl.h>
#include <SDL_opengles2.h>
#include <SDL_render.h>
#include <SDL_syswm.h>
// These are extensions, so some platform headers may not provide them
#ifndef EGL_PLATFORM_WAYLAND_KHR
#define EGL_PLATFORM_WAYLAND_KHR 0x31D8
#endif
#ifndef EGL_PLATFORM_X11_KHR
#define EGL_PLATFORM_X11_KHR 0x31D5
#endif
#ifndef EGL_PLATFORM_GBM_KHR
#define EGL_PLATFORM_GBM_KHR 0x31D7
#endif
#ifndef GL_UNPACK_ROW_LENGTH_EXT
#define GL_UNPACK_ROW_LENGTH_EXT 0x0CF2
#endif
typedef struct _OVERLAY_VERTEX
{
float x, y;
float u, v;
} OVERLAY_VERTEX, *POVERLAY_VERTEX;
/* TODO:
* - handle more pixel formats
* - handle software decoding
*/
/* DOC/misc:
* - https://kernel-recipes.org/en/2016/talks/video-and-colorspaces/
* - http://www.brucelindbloom.com/
* - https://learnopengl.com/Getting-started/Shaders
* - https://github.com/stunpix/yuvit
* - https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
* - https://www.renesas.com/eu/en/www/doc/application-note/an9717.pdf
* - https://www.xilinx.com/support/documentation/application_notes/xapp283.pdf
* - https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.709-6-201506-I!!PDF-E.pdf
* - https://www.khronos.org/registry/OpenGL/extensions/OES/OES_EGL_image_external.txt
* - https://gist.github.com/rexguo/6696123
* - https://wiki.libsdl.org/CategoryVideo
*/
#define EGL_LOG(Category, ...) SDL_Log ## Category(\
SDL_LOG_CATEGORY_APPLICATION, \
"EGLRenderer: " __VA_ARGS__)
SDL_Window* EGLRenderer::s_LastFailedWindow = nullptr;
EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
:
m_EGLImagePixelFormat(AV_PIX_FMT_NONE),
m_EGLDisplay(EGL_NO_DISPLAY),
m_Textures{0},
m_OverlayTextures{0},
m_OverlayVbos{0},
m_OverlayHasValidData{},
m_ShaderProgram(0),
m_OverlayShaderProgram(0),
m_Context(0),
m_Window(nullptr),
m_Backend(backendRenderer),
m_VAO(0),
m_BlockingSwapBuffers(false),
m_LastFrame(av_frame_alloc()),
m_glEGLImageTargetTexture2DOES(nullptr),
m_glGenVertexArraysOES(nullptr),
m_glBindVertexArrayOES(nullptr),
m_glDeleteVertexArraysOES(nullptr),
m_GlesMajorVersion(0),
m_GlesMinorVersion(0),
m_HasExtUnpackSubimage(false),
m_DummyRenderer(nullptr)
{
SDL_assert(backendRenderer);
SDL_assert(backendRenderer->canExportEGL());
}
EGLRenderer::~EGLRenderer()
{
if (m_Context) {
// Reattach the GL context to the main thread for destruction
SDL_GL_MakeCurrent(m_Window, m_Context);
if (m_ShaderProgram) {
glDeleteProgram(m_ShaderProgram);
}
if (m_OverlayShaderProgram) {
glDeleteProgram(m_OverlayShaderProgram);
}
if (m_VAO) {
SDL_assert(m_glDeleteVertexArraysOES != nullptr);
m_glDeleteVertexArraysOES(1, &m_VAO);
}
for (int i = 0; i < EGL_MAX_PLANES; i++) {
if (m_Textures[i] != 0) {
glDeleteTextures(1, &m_Textures[i]);
}
}
for (int i = 0; i < Overlay::OverlayMax; i++) {
if (m_OverlayTextures[i] != 0) {
glDeleteTextures(1, &m_OverlayTextures[i]);
}
if (m_OverlayVbos[i] != 0) {
glDeleteBuffers(1, &m_OverlayVbos[i]);
}
}
SDL_GL_DeleteContext(m_Context);
}
if (m_DummyRenderer) {
SDL_DestroyRenderer(m_DummyRenderer);
}
av_frame_free(&m_LastFrame);
// Reset the global properties back to what they were before
SDL_SetHint(SDL_HINT_OPENGL_ES_DRIVER, "0");
// We avoid restoring GL profile and version on purpose. SDL
// renderers use that internally to determine what GL version
// the window is using. If we reset it, it may not properly
// reset back to desktop GL if we have to fall back to the
// SDL renderer.
SDL_GL_SetAttribute(SDL_GL_RED_SIZE, 8);
SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 8);
SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 8);
}
bool EGLRenderer::prepareDecoderContext(AVCodecContext*, AVDictionary**)
{
/* Nothing to do */
EGL_LOG(Info, "Using EGL renderer");
return true;
}
void EGLRenderer::notifyOverlayUpdated(Overlay::OverlayType type)
{
// We handle uploading the updated overlay texture in renderOverlay().
// notifyOverlayUpdated() is called on an arbitrary thread, which may
// not be have the OpenGL context current on it.
if (!Session::get()->getOverlayManager().isOverlayEnabled(type)) {
// If the overlay has been disabled, mark the data as invalid/stale.
SDL_AtomicSet(&m_OverlayHasValidData[type], 0);
return;
}
}
bool EGLRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
{
// Pixel format support should be determined by the backend renderer
return m_Backend->isPixelFormatSupported(videoFormat, pixelFormat);
}
AVPixelFormat EGLRenderer::getPreferredPixelFormat(int videoFormat)
{
// Pixel format preference should be determined by the backend renderer
return m_Backend->getPreferredPixelFormat(videoFormat);
}
void EGLRenderer::renderOverlay(Overlay::OverlayType type)
{
// Do nothing if this overlay is disabled
if (!Session::get()->getOverlayManager().isOverlayEnabled(type)) {
return;
}
// Upload a new overlay texture if needed
SDL_Surface* newSurface = Session::get()->getOverlayManager().getUpdatedOverlaySurface(type);
if (newSurface != nullptr) {
SDL_assert(!SDL_MUSTLOCK(newSurface));
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[type]);
void* packedPixelData = nullptr;
if (m_GlesMajorVersion >= 3 || m_HasExtUnpackSubimage) {
// If we are GLES 3.0+ or have GL_EXT_unpack_subimage, GL can handle any pitch
SDL_assert(newSurface->pitch % newSurface->format->BytesPerPixel == 0);
glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, newSurface->pitch / newSurface->format->BytesPerPixel);
}
else if (newSurface->pitch != newSurface->w * newSurface->format->BytesPerPixel) {
// If we can't use GL_UNPACK_ROW_LENGTH and the surface isn't tightly packed,
// we must allocate a tightly packed buffer and copy our pixels there.
packedPixelData = malloc(newSurface->w * newSurface->h * newSurface->format->BytesPerPixel);
if (!packedPixelData) {
SDL_FreeSurface(newSurface);
return;
}
SDL_ConvertPixels(newSurface->w, newSurface->h,
newSurface->format->format, newSurface->pixels, newSurface->pitch,
newSurface->format->format, packedPixelData, newSurface->w * newSurface->format->BytesPerPixel);
}
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, newSurface->w, newSurface->h, 0, GL_RGBA, GL_UNSIGNED_BYTE,
packedPixelData ? packedPixelData : newSurface->pixels);
if (packedPixelData) {
free(packedPixelData);
}
// SDL_FRect wasn't added until 2.0.10
struct {
float x;
float y;
float w;
float h;
} overlayRect = {};
// These overlay positions differ from the other renderers because OpenGL
// places the origin in the lower-left corner instead of the upper-left.
if (type == Overlay::OverlayStatusUpdate) {
// Bottom Left
overlayRect.x = 0;
overlayRect.y = 0;
}
else if (type == Overlay::OverlayDebug) {
// Top left
overlayRect.x = 0;
overlayRect.y = m_ViewportHeight - newSurface->h;
} else {
SDL_assert(false);
}
overlayRect.w = newSurface->w;
overlayRect.h = newSurface->h;
SDL_FreeSurface(newSurface);
// Convert screen space to normalized device coordinates
overlayRect.x /= m_ViewportWidth / 2;
overlayRect.w /= m_ViewportWidth / 2;
overlayRect.y /= m_ViewportHeight / 2;
overlayRect.h /= m_ViewportHeight / 2;
overlayRect.x -= 1.0f;
overlayRect.y -= 1.0f;
OVERLAY_VERTEX verts[] =
{
{overlayRect.x + overlayRect.w, overlayRect.y + overlayRect.h, 1.0f, 0.0f},
{overlayRect.x, overlayRect.y + overlayRect.h, 0.0f, 0.0f},
{overlayRect.x, overlayRect.y, 0.0f, 1.0f},
{overlayRect.x, overlayRect.y, 0.0f, 1.0f},
{overlayRect.x + overlayRect.w, overlayRect.y, 1.0f, 1.0f},
{overlayRect.x + overlayRect.w, overlayRect.y + overlayRect.h, 1.0f, 0.0f}
};
glBindBuffer(GL_ARRAY_BUFFER, m_OverlayVbos[type]);
glBufferData(GL_ARRAY_BUFFER, sizeof(verts), verts, GL_STATIC_DRAW);
SDL_AtomicSet(&m_OverlayHasValidData[type], 1);
}
if (!SDL_AtomicGet(&m_OverlayHasValidData[type])) {
// If the overlay is not populated yet or is stale, don't render it.
return;
}
glUseProgram(m_OverlayShaderProgram);
glBindBuffer(GL_ARRAY_BUFFER, m_OverlayVbos[type]);
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(OVERLAY_VERTEX), (void*)offsetof(OVERLAY_VERTEX, x));
glEnableVertexAttribArray(0);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, sizeof(OVERLAY_VERTEX), (void*)offsetof(OVERLAY_VERTEX, u));
glEnableVertexAttribArray(1);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[type]);
glUniform1i(m_OverlayShaderProgramParams[OVERLAY_PARAM_TEXTURE], 0);
glDrawArrays(GL_TRIANGLES, 0, 6);
}
int EGLRenderer::loadAndBuildShader(int shaderType,
const char *file) {
GLuint shader = glCreateShader(shaderType);
if (!shader || shader == GL_INVALID_ENUM) {
EGL_LOG(Error, "Can't create shader: %d", glGetError());
return 0;
}
auto sourceData = Path::readDataFile(file);
GLint len = sourceData.size();
const char *buf = sourceData.data();
glShaderSource(shader, 1, &buf, &len);
glCompileShader(shader);
GLint status;
glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
if (!status) {
char shaderLog[512];
glGetShaderInfoLog(shader, sizeof (shaderLog), nullptr, shaderLog);
EGL_LOG(Error, "Cannot load shader \"%s\": %s", file, shaderLog);
return 0;
}
return shader;
}
bool EGLRenderer::openDisplay(unsigned int platform, void* nativeDisplay)
{
PFNEGLGETPLATFORMDISPLAYPROC eglGetPlatformDisplayProc;
PFNEGLGETPLATFORMDISPLAYEXTPROC eglGetPlatformDisplayEXTProc;
m_EGLDisplay = EGL_NO_DISPLAY;
// NB: eglGetPlatformDisplay() and eglGetPlatformDisplayEXT() have slightly different definitions
eglGetPlatformDisplayProc = (typeof(eglGetPlatformDisplayProc))eglGetProcAddress("eglGetPlatformDisplay");
eglGetPlatformDisplayEXTProc = (typeof(eglGetPlatformDisplayEXTProc))eglGetProcAddress("eglGetPlatformDisplayEXT");
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// eglGetPlatformDisplay() is part of the EGL 1.5 core specification
if (eglGetPlatformDisplayProc != nullptr) {
m_EGLDisplay = eglGetPlatformDisplayProc(platform, nativeDisplay, nullptr);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Warn, "eglGetPlatformDisplay() failed: %d", eglGetError());
}
}
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// eglGetPlatformDisplayEXT() is an extension for EGL 1.4
const EGLExtensions eglExtensions(EGL_NO_DISPLAY);
if (eglExtensions.isSupported("EGL_EXT_platform_base")) {
if (eglGetPlatformDisplayEXTProc != nullptr) {
m_EGLDisplay = eglGetPlatformDisplayEXTProc(platform, nativeDisplay, nullptr);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Warn, "eglGetPlatformDisplayEXT() failed: %d", eglGetError());
}
}
else {
EGL_LOG(Warn, "EGL_EXT_platform_base supported but no eglGetPlatformDisplayEXT() export!");
}
}
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// Finally, if all else fails, use eglGetDisplay()
m_EGLDisplay = eglGetDisplay((EGLNativeDisplayType)nativeDisplay);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Error, "eglGetDisplay() failed: %d", eglGetError());
}
}
return m_EGLDisplay != EGL_NO_DISPLAY;
}
unsigned EGLRenderer::compileShader(const char* vertexShaderSrc, const char* fragmentShaderSrc) {
unsigned shader = 0;
GLuint vertexShader = loadAndBuildShader(GL_VERTEX_SHADER, vertexShaderSrc);
if (!vertexShader)
return false;
GLuint fragmentShader = loadAndBuildShader(GL_FRAGMENT_SHADER, fragmentShaderSrc);
if (!fragmentShader)
goto fragError;
shader = glCreateProgram();
if (!shader) {
EGL_LOG(Error, "Cannot create shader program");
goto progFailCreate;
}
glAttachShader(shader, vertexShader);
glAttachShader(shader, fragmentShader);
glLinkProgram(shader);
int status;
glGetProgramiv(shader, GL_LINK_STATUS, &status);
if (!status) {
char shader_log[512];
glGetProgramInfoLog(shader, sizeof (shader_log), nullptr, shader_log);
EGL_LOG(Error, "Cannot link shader program: %s", shader_log);
glDeleteProgram(shader);
shader = 0;
}
progFailCreate:
glDeleteShader(fragmentShader);
fragError:
glDeleteShader(vertexShader);
return shader;
}
bool EGLRenderer::compileShaders() {
SDL_assert(!m_ShaderProgram);
SDL_assert(!m_OverlayShaderProgram);
SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
// XXX: TODO: other formats
if (m_EGLImagePixelFormat == AV_PIX_FMT_NV12 || m_EGLImagePixelFormat == AV_PIX_FMT_P010) {
m_ShaderProgram = compileShader("egl_nv12.vert", "egl_nv12.frag");
if (!m_ShaderProgram) {
return false;
}
m_ShaderProgramParams[NV12_PARAM_YUVMAT] = glGetUniformLocation(m_ShaderProgram, "yuvmat");
m_ShaderProgramParams[NV12_PARAM_OFFSET] = glGetUniformLocation(m_ShaderProgram, "offset");
m_ShaderProgramParams[NV12_PARAM_PLANE1] = glGetUniformLocation(m_ShaderProgram, "plane1");
m_ShaderProgramParams[NV12_PARAM_PLANE2] = glGetUniformLocation(m_ShaderProgram, "plane2");
}
else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
m_ShaderProgram = compileShader("egl_opaque.vert", "egl_opaque.frag");
if (!m_ShaderProgram) {
return false;
}
m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE] = glGetUniformLocation(m_ShaderProgram, "uTexture");
}
else {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unsupported EGL pixel format: %d",
m_EGLImagePixelFormat);
SDL_assert(false);
return false;
}
m_OverlayShaderProgram = compileShader("egl_overlay.vert", "egl_overlay.frag");
if (!m_OverlayShaderProgram) {
return false;
}
m_OverlayShaderProgramParams[OVERLAY_PARAM_TEXTURE] = glGetUniformLocation(m_OverlayShaderProgram, "uTexture");
return true;
}
bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
{
m_Window = params->window;
// It's not safe to attempt to opportunistically create a GLES2
// renderer prior to 2.0.10. If GLES2 isn't available, SDL will
// attempt to dereference a null pointer and crash Moonlight.
// https://bugzilla.libsdl.org/show_bug.cgi?id=4350
// https://hg.libsdl.org/SDL/rev/84618d571795
if (!SDL_VERSION_ATLEAST(2, 0, 10)) {
EGL_LOG(Error, "Not supported until SDL 2.0.10");
return false;
}
// HACK: Work around bug where renderer will repeatedly fail with:
// SDL_CreateRenderer() failed: Could not create GLES window surface
if (m_Window == s_LastFailedWindow) {
EGL_LOG(Error, "SDL_CreateRenderer() already failed on this window!");
return false;
}
// This hint will ensure we use EGL to retreive our GL context,
// even on X11 where that is not the default. EGL is required
// to avoid a crash in Mesa.
// https://gitlab.freedesktop.org/mesa/mesa/issues/1011
SDL_SetHint(SDL_HINT_OPENGL_ES_DRIVER, "1");
// Change our GL context to use 10-bit colors for 10-bit content
if (params->videoFormat == VIDEO_FORMAT_H265_MAIN10) {
// FIXME: We should try to do this only once per window.
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Changing GL context to 10-bit color");
// Initialize our GL attributes to defaults (desktop GL).
// This ensures we will always hit the SDL_RecreateWindow()
// path which will ensure our color buffer changes below
// take effect.
SDL_GL_ResetAttributes();
// Request 10-bit color for Main10
SDL_GL_SetAttribute(SDL_GL_RED_SIZE, 10);
SDL_GL_SetAttribute(SDL_GL_GREEN_SIZE, 10);
SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 10);
}
int renderIndex;
int maxRenderers = SDL_GetNumRenderDrivers();
SDL_assert(maxRenderers >= 0);
SDL_RendererInfo renderInfo;
for (renderIndex = 0; renderIndex < maxRenderers; ++renderIndex) {
if (SDL_GetRenderDriverInfo(renderIndex, &renderInfo))
continue;
if (!strcmp(renderInfo.name, "opengles2")) {
SDL_assert(renderInfo.flags & SDL_RENDERER_ACCELERATED);
break;
}
}
if (renderIndex == maxRenderers) {
EGL_LOG(Error, "Could not find a suitable SDL_Renderer");
return false;
}
if (!(m_DummyRenderer = SDL_CreateRenderer(m_Window, renderIndex, SDL_RENDERER_ACCELERATED))) {
EGL_LOG(Error, "SDL_CreateRenderer() failed: %s", SDL_GetError());
s_LastFailedWindow = m_Window;
return false;
}
// SDL_CreateRenderer() can end up having to recreate our window (SDL_RecreateWindow())
// to ensure it's compatible with the renderer's OpenGL context. If that happens, we
// can get spurious SDL_WINDOWEVENT events that will cause us to (again) recreate our
// renderer. This can lead to an infinite to renderer recreation, so discard all
// SDL_WINDOWEVENT events after SDL_CreateRenderer().
Session* session = Session::get();
if (session != nullptr) {
// If we get here during a session, we need to synchronize with the event loop
// to ensure we don't drop any important events.
session->flushWindowEvents();
}
else {
// If we get here prior to the start of a session, just pump and flush ourselves.
SDL_PumpEvents();
SDL_FlushEvent(SDL_WINDOWEVENT);
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(params->window, &info)) {
EGL_LOG(Error, "SDL_GetWindowWMInfo() failed: %s", SDL_GetError());
return false;
}
switch (info.subsystem) {
#ifdef SDL_VIDEO_DRIVER_WAYLAND
case SDL_SYSWM_WAYLAND:
if (!openDisplay(EGL_PLATFORM_WAYLAND_KHR, info.info.wl.display)) {
return false;
}
break;
#endif
#ifdef SDL_VIDEO_DRIVER_X11
case SDL_SYSWM_X11:
if (!openDisplay(EGL_PLATFORM_X11_KHR, info.info.x11.display)) {
return false;
}
break;
#endif
#if SDL_VERSION_ATLEAST(2, 0, 15) && defined(SDL_VIDEO_DRIVER_KMSDRM)
case SDL_SYSWM_KMSDRM:
if (!openDisplay(EGL_PLATFORM_GBM_KHR, info.info.kmsdrm.gbm_dev)) {
return false;
}
break;
#endif
default:
EGL_LOG(Error, "not compatible with SYSWM");
return false;
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Error, "Cannot get EGL display: %d", eglGetError());
return false;
}
if (!(m_Context = SDL_GL_CreateContext(params->window))) {
EGL_LOG(Error, "Cannot create OpenGL context: %s", SDL_GetError());
return false;
}
if (SDL_GL_MakeCurrent(params->window, m_Context)) {
EGL_LOG(Error, "Cannot use created EGL context: %s", SDL_GetError());
return false;
}
{
int r, g, b, a;
SDL_GL_GetAttribute(SDL_GL_RED_SIZE, &r);
SDL_GL_GetAttribute(SDL_GL_GREEN_SIZE, &g);
SDL_GL_GetAttribute(SDL_GL_BLUE_SIZE, &b);
SDL_GL_GetAttribute(SDL_GL_ALPHA_SIZE, &a);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Color buffer is: R%dG%dB%dA%d",
r, g, b, a);
}
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, &m_GlesMajorVersion);
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, &m_GlesMinorVersion);
// We can use GL_UNPACK_ROW_LENGTH for a more optimized upload of non-tightly-packed textures
m_HasExtUnpackSubimage = SDL_GL_ExtensionSupported("GL_EXT_unpack_subimage");
const EGLExtensions eglExtensions(m_EGLDisplay);
if (!eglExtensions.isSupported("EGL_KHR_image_base") &&
!eglExtensions.isSupported("EGL_KHR_image")) {
EGL_LOG(Error, "EGL_KHR_image unsupported");
return false;
}
else if (!SDL_GL_ExtensionSupported("GL_OES_EGL_image")) {
EGL_LOG(Error, "GL_OES_EGL_image unsupported");
return false;
}
if (!m_Backend->initializeEGL(m_EGLDisplay, eglExtensions))
return false;
if (!(m_glEGLImageTargetTexture2DOES = (typeof(m_glEGLImageTargetTexture2DOES))eglGetProcAddress("glEGLImageTargetTexture2DOES"))) {
EGL_LOG(Error,
"EGL: cannot retrieve `glEGLImageTargetTexture2DOES` address");
return false;
}
// Vertex arrays are an extension on OpenGL ES 2.0
if (SDL_GL_ExtensionSupported("GL_OES_vertex_array_object")) {
m_glGenVertexArraysOES = (typeof(m_glGenVertexArraysOES))eglGetProcAddress("glGenVertexArraysOES");
m_glBindVertexArrayOES = (typeof(m_glBindVertexArrayOES))eglGetProcAddress("glBindVertexArrayOES");
m_glDeleteVertexArraysOES = (typeof(m_glDeleteVertexArraysOES))eglGetProcAddress("glDeleteVertexArraysOES");
}
else {
// They are included in OpenGL ES 3.0 as part of the standard
m_glGenVertexArraysOES = (typeof(m_glGenVertexArraysOES))eglGetProcAddress("glGenVertexArrays");
m_glBindVertexArrayOES = (typeof(m_glBindVertexArrayOES))eglGetProcAddress("glBindVertexArray");
m_glDeleteVertexArraysOES = (typeof(m_glDeleteVertexArraysOES))eglGetProcAddress("glDeleteVertexArrays");
}
if (!m_glGenVertexArraysOES || !m_glBindVertexArrayOES || !m_glDeleteVertexArraysOES) {
EGL_LOG(Error, "Failed to find VAO functions");
return false;
}
/* Compute the video region size in order to keep the aspect ratio of the
* video stream.
*/
SDL_Rect src, dst;
src.x = src.y = dst.x = dst.y = 0;
src.w = params->width;
src.h = params->height;
SDL_GL_GetDrawableSize(m_Window, &dst.w, &dst.h);
StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
glViewport(dst.x, dst.y, dst.w, dst.h);
m_ViewportWidth = dst.w;
m_ViewportHeight = dst.h;
glClearColor(0, 0, 0, 1);
glClear(GL_COLOR_BUFFER_BIT);
if (params->enableVsync) {
SDL_GL_SetSwapInterval(1);
#if SDL_VERSION_ATLEAST(2, 0, 15) && defined(SDL_VIDEO_DRIVER_KMSDRM)
// The SDL KMSDRM backend already enforces double buffering (due to
// SDL_HINT_VIDEO_DOUBLE_BUFFER=1), so calling glFinish() after
// SDL_GL_SwapWindow() will block an extra frame and lock rendering
// at 1/2 the display refresh rate.
if (info.subsystem != SDL_SYSWM_KMSDRM)
#endif
{
m_BlockingSwapBuffers = true;
}
} else {
SDL_GL_SetSwapInterval(0);
}
SDL_GL_SwapWindow(params->window);
glGenTextures(EGL_MAX_PLANES, m_Textures);
for (size_t i = 0; i < EGL_MAX_PLANES; ++i) {
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
glGenBuffers(Overlay::OverlayMax, m_OverlayVbos);
glGenTextures(Overlay::OverlayMax, m_OverlayTextures);
for (size_t i = 0; i < Overlay::OverlayMax; ++i) {
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
GLenum err = glGetError();
if (err != GL_NO_ERROR)
EGL_LOG(Error, "OpenGL error: %d", err);
// Detach the context from this thread, so the render thread can attach it
SDL_GL_MakeCurrent(m_Window, nullptr);
return err == GL_NO_ERROR;
}
const float *EGLRenderer::getColorOffsets(const AVFrame* frame) {
static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f };
static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f };
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
return (frame->color_range == AVCOL_RANGE_JPEG) ? fullOffsets : limitedOffsets;
}
const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
/* The conversion matrices are shamelessly stolen from linux:
* drivers/media/platform/imx-pxp.c:pxp_setup_csc
*/
static const float bt601Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.3917f, 2.0172f,
1.5960f, -0.8129f, 0.0f
};
static const float bt601Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.3441f, 1.7720f,
1.4020f, -0.7141f, 0.0f
};
static const float bt709Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.2132f, 2.1124f,
1.7927f, -0.5329f, 0.0f
};
static const float bt709Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1873f, 1.8556f,
1.5748f, -0.4681f, 0.0f
};
static const float bt2020Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.1874f, 2.1418f,
1.6781f, -0.6505f, 0.0f
};
static const float bt2020Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1646f, 1.8814f,
1.4746f, -0.5714f, 0.0f
};
// This handles the case where the color range is unknown,
// so that we use Limited color range which is the default
// behavior for Moonlight.
bool fullRange = (frame->color_range == AVCOL_RANGE_JPEG);
switch (frame->colorspace) {
case AVCOL_SPC_SMPTE170M:
case AVCOL_SPC_BT470BG:
return fullRange ? bt601Full : bt601Lim;
case AVCOL_SPC_BT709:
return fullRange ? bt709Full : bt709Lim;
case AVCOL_SPC_BT2020_NCL:
case AVCOL_SPC_BT2020_CL:
return fullRange ? bt2020Full : bt2020Lim;
default:
// Some backends don't populate this, so we'll assume
// the host gave us what we asked for by default.
switch (getDecoderColorspace()) {
case COLORSPACE_REC_601:
return fullRange ? bt601Full : bt601Lim;
case COLORSPACE_REC_709:
return fullRange ? bt709Full : bt709Lim;
case COLORSPACE_REC_2020:
return fullRange ? bt2020Full : bt2020Lim;
default:
SDL_assert(false);
}
};
return bt601Lim;
}
bool EGLRenderer::specialize() {
SDL_assert(!m_VAO);
if (!compileShaders())
return false;
// The viewport should have the aspect ratio of the video stream
static const float vertices[] = {
// pos .... // tex coords
1.0f, 1.0f, 1.0f, 0.0f,
1.0f, -1.0f, 1.0f, 1.0f,
-1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 0.0f,
};
static const unsigned int indices[] = {
0, 1, 3,
1, 2, 3,
};
glUseProgram(m_ShaderProgram);
unsigned int VBO, EBO;
m_glGenVertexArraysOES(1, &m_VAO);
glGenBuffers(1, &VBO);
glGenBuffers(1, &EBO);
m_glBindVertexArrayOES(m_VAO);
glBindBuffer(GL_ARRAY_BUFFER, VBO);
glBufferData(GL_ARRAY_BUFFER, sizeof (vertices), vertices, GL_STATIC_DRAW);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, GL_STATIC_DRAW);
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
glEnableVertexAttribArray(0);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof (float)));
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, 0);
m_glBindVertexArrayOES(0);
glDeleteBuffers(1, &VBO);
glDeleteBuffers(1, &EBO);
GLenum err = glGetError();
if (err != GL_NO_ERROR) {
EGL_LOG(Error, "OpenGL error: %d", err);
}
return err == GL_NO_ERROR;
}
void EGLRenderer::renderFrame(AVFrame* frame)
{
EGLImage imgs[EGL_MAX_PLANES];
if (frame == nullptr) {
// End of stream - unbind the GL context
SDL_GL_MakeCurrent(m_Window, nullptr);
return;
}
// Attach our GL context to the render thread
// NB: It should already be current, unless the SDL render event watcher
// performs a rendering operation (like a viewport update on resize) on
// our fake SDL_Renderer. If it's already current, this is a no-op.
SDL_GL_MakeCurrent(m_Window, m_Context);
// Find the native read-back format and load the shaders
if (m_EGLImagePixelFormat == AV_PIX_FMT_NONE) {
m_EGLImagePixelFormat = m_Backend->getEGLImagePixelFormat();
EGL_LOG(Info, "EGLImage pixel format: %d", m_EGLImagePixelFormat);
SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
if (!specialize()) {
m_EGLImagePixelFormat = AV_PIX_FMT_NONE;
return;
}
}
ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs);
if (plane_count < 0)
return;
for (ssize_t i = 0; i < plane_count; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
m_glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]);
}
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram(m_ShaderProgram);
m_glBindVertexArrayOES(m_VAO);
// Bind parameters for the shaders
if (m_EGLImagePixelFormat == AV_PIX_FMT_NV12 || m_EGLImagePixelFormat == AV_PIX_FMT_P010) {
glUniformMatrix3fv(m_ShaderProgramParams[NV12_PARAM_YUVMAT], 1, GL_FALSE, getColorMatrix(frame));
glUniform3fv(m_ShaderProgramParams[NV12_PARAM_OFFSET], 1, getColorOffsets(frame));
glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE1], 0);
glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE2], 1);
}
else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
glUniform1i(m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE], 0);
}
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
m_glBindVertexArrayOES(0);
for (int i = 0; i < Overlay::OverlayMax; i++) {
renderOverlay((Overlay::OverlayType)i);
}
SDL_GL_SwapWindow(m_Window);
if (m_BlockingSwapBuffers) {
// This glClear() forces us to block until the buffer swap is
// complete to continue rendering. Mesa won't actually wait
// for the swap with just glFinish() alone. Waiting here keeps us
// in lock step with the display refresh rate. If we don't wait
// here, we'll stall on the first GL call next frame. Doing the
// wait here instead allows more time for a newer frame to arrive
// for next renderFrame() call.
glClear(GL_COLOR_BUFFER_BIT);
glFinish();
}
m_Backend->freeEGLImages(m_EGLDisplay, imgs);
// Free the DMA-BUF backing the last frame now that it is definitely
// no longer being used anymore. While the PRIME FD stays around until
// EGL is done with it, the memory backing it may be reused by FFmpeg
// before the GPU has read it. This is particularly noticeable on the
// RK3288-based TinkerBoard when V-Sync is disabled.
av_frame_unref(m_LastFrame);
av_frame_move_ref(m_LastFrame, frame);
}
bool EGLRenderer::testRenderFrame(AVFrame* frame)
{
EGLImage imgs[EGL_MAX_PLANES];
// Make sure we can get working EGLImages from the backend renderer.
// Some devices (Raspberry Pi) will happily decode into DRM formats that
// its own GL implementation won't accept in eglCreateImage().
ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs);
if (plane_count <= 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Backend failed to export EGL image for test frame");
return false;
}
m_Backend->freeEGLImages(m_EGLDisplay, imgs);
return true;
}
Reference in a new issue View git blame Copy permalink