Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2024-12-22 09:03:07 +00:00
Code Issues Projects Releases Packages Wiki Activity
moonlight-qt/app/streaming/video/ffmpeg-renderers/eglvid.cpp
Cameron Gutman 5970ca49e9 Don't use swap interval on Wayland
2022-11-12 15:45:54 -06:00

1018 lines
37 KiB
C++
Raw Blame History

// vim: noai:ts=4:sw=4:softtabstop=4:expandtab
#include "eglvid.h"
#include "path.h"
#include "streaming/session.h"
#include "streaming/streamutils.h"
#include <QDir>
#include <Limelight.h>
#include <unistd.h>
#include <SDL_egl.h>
#include <SDL_opengles2.h>
#include <SDL_render.h>
#include <SDL_syswm.h>
// These are extensions, so some platform headers may not provide them
#ifndef EGL_PLATFORM_WAYLAND_KHR
#define EGL_PLATFORM_WAYLAND_KHR 0x31D8
#endif
#ifndef EGL_PLATFORM_X11_KHR
#define EGL_PLATFORM_X11_KHR 0x31D5
#endif
#ifndef EGL_PLATFORM_GBM_KHR
#define EGL_PLATFORM_GBM_KHR 0x31D7
#endif
#ifndef GL_UNPACK_ROW_LENGTH_EXT
#define GL_UNPACK_ROW_LENGTH_EXT 0x0CF2
#endif
typedef struct _OVERLAY_VERTEX
{
float x, y;
float u, v;
} OVERLAY_VERTEX, *POVERLAY_VERTEX;
/* TODO:
* - handle more pixel formats
* - handle software decoding
*/
/* DOC/misc:
* - https://kernel-recipes.org/en/2016/talks/video-and-colorspaces/
* - http://www.brucelindbloom.com/
* - https://learnopengl.com/Getting-started/Shaders
* - https://github.com/stunpix/yuvit
* - https://en.wikipedia.org/wiki/YCbCr#ITU-R_BT.601_conversion
* - https://www.renesas.com/eu/en/www/doc/application-note/an9717.pdf
* - https://www.xilinx.com/support/documentation/application_notes/xapp283.pdf
* - https://www.itu.int/dms_pubrec/itu-r/rec/bt/R-REC-BT.709-6-201506-I!!PDF-E.pdf
* - https://www.khronos.org/registry/OpenGL/extensions/OES/OES_EGL_image_external.txt
* - https://gist.github.com/rexguo/6696123
* - https://wiki.libsdl.org/CategoryVideo
*/
#define EGL_LOG(Category, ...) SDL_Log ## Category(\
SDL_LOG_CATEGORY_APPLICATION, \
"EGLRenderer: " __VA_ARGS__)
SDL_Window* EGLRenderer::s_LastFailedWindow = nullptr;
int EGLRenderer::s_LastFailedVideoFormat = 0;
EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
:
m_EGLImagePixelFormat(AV_PIX_FMT_NONE),
m_EGLDisplay(EGL_NO_DISPLAY),
m_Textures{0},
m_OverlayTextures{0},
m_OverlayVbos{0},
m_OverlayHasValidData{},
m_ShaderProgram(0),
m_OverlayShaderProgram(0),
m_Context(0),
m_Window(nullptr),
m_Backend(backendRenderer),
m_VAO(0),
m_BlockingSwapBuffers(false),
m_LastRenderSync(EGL_NO_SYNC),
m_LastFrame(av_frame_alloc()),
m_glEGLImageTargetTexture2DOES(nullptr),
m_glGenVertexArraysOES(nullptr),
m_glBindVertexArrayOES(nullptr),
m_glDeleteVertexArraysOES(nullptr),
m_eglCreateSync(nullptr),
m_eglCreateSyncKHR(nullptr),
m_eglDestroySync(nullptr),
m_eglClientWaitSync(nullptr),
m_GlesMajorVersion(0),
m_GlesMinorVersion(0),
m_HasExtUnpackSubimage(false),
m_DummyRenderer(nullptr)
{
SDL_assert(backendRenderer);
SDL_assert(backendRenderer->canExportEGL());
// Save these global parameters so we can restore them in our destructor
SDL_GL_GetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, &m_OldContextProfileMask);
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, &m_OldContextMajorVersion);
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, &m_OldContextMinorVersion);
}
EGLRenderer::~EGLRenderer()
{
if (m_Context) {
// Reattach the GL context to the main thread for destruction
SDL_GL_MakeCurrent(m_Window, m_Context);
if (m_LastRenderSync != EGL_NO_SYNC) {
SDL_assert(m_eglDestroySync != nullptr);
m_eglDestroySync(m_EGLDisplay, m_LastRenderSync);
}
if (m_ShaderProgram) {
glDeleteProgram(m_ShaderProgram);
}
if (m_OverlayShaderProgram) {
glDeleteProgram(m_OverlayShaderProgram);
}
if (m_VAO) {
SDL_assert(m_glDeleteVertexArraysOES != nullptr);
m_glDeleteVertexArraysOES(1, &m_VAO);
}
for (int i = 0; i < EGL_MAX_PLANES; i++) {
if (m_Textures[i] != 0) {
glDeleteTextures(1, &m_Textures[i]);
}
}
for (int i = 0; i < Overlay::OverlayMax; i++) {
if (m_OverlayTextures[i] != 0) {
glDeleteTextures(1, &m_OverlayTextures[i]);
}
if (m_OverlayVbos[i] != 0) {
glDeleteBuffers(1, &m_OverlayVbos[i]);
}
}
SDL_GL_DeleteContext(m_Context);
}
if (m_DummyRenderer) {
SDL_DestroyRenderer(m_DummyRenderer);
}
av_frame_free(&m_LastFrame);
// Reset the global properties back to what they were before
SDL_SetHint(SDL_HINT_OPENGL_ES_DRIVER, "0");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, m_OldContextProfileMask);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, m_OldContextMajorVersion);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, m_OldContextMinorVersion);
}
bool EGLRenderer::prepareDecoderContext(AVCodecContext*, AVDictionary**)
{
/* Nothing to do */
EGL_LOG(Info, "Using EGL renderer");
return true;
}
void EGLRenderer::notifyOverlayUpdated(Overlay::OverlayType type)
{
// We handle uploading the updated overlay texture in renderOverlay().
// notifyOverlayUpdated() is called on an arbitrary thread, which may
// not be have the OpenGL context current on it.
if (!Session::get()->getOverlayManager().isOverlayEnabled(type)) {
// If the overlay has been disabled, mark the data as invalid/stale.
SDL_AtomicSet(&m_OverlayHasValidData[type], 0);
return;
}
}
bool EGLRenderer::isPixelFormatSupported(int videoFormat, AVPixelFormat pixelFormat)
{
// Pixel format support should be determined by the backend renderer
return m_Backend->isPixelFormatSupported(videoFormat, pixelFormat);
}
AVPixelFormat EGLRenderer::getPreferredPixelFormat(int videoFormat)
{
// Pixel format preference should be determined by the backend renderer
return m_Backend->getPreferredPixelFormat(videoFormat);
}
void EGLRenderer::renderOverlay(Overlay::OverlayType type)
{
// Do nothing if this overlay is disabled
if (!Session::get()->getOverlayManager().isOverlayEnabled(type)) {
return;
}
// Upload a new overlay texture if needed
SDL_Surface* newSurface = Session::get()->getOverlayManager().getUpdatedOverlaySurface(type);
if (newSurface != nullptr) {
SDL_assert(!SDL_MUSTLOCK(newSurface));
SDL_assert(newSurface->format->format == SDL_PIXELFORMAT_ARGB8888);
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[type]);
void* packedPixelData = nullptr;
if (m_GlesMajorVersion >= 3 || m_HasExtUnpackSubimage) {
// If we are GLES 3.0+ or have GL_EXT_unpack_subimage, GL can handle any pitch
SDL_assert(newSurface->pitch % newSurface->format->BytesPerPixel == 0);
glPixelStorei(GL_UNPACK_ROW_LENGTH_EXT, newSurface->pitch / newSurface->format->BytesPerPixel);
}
else if (newSurface->pitch != newSurface->w * newSurface->format->BytesPerPixel) {
// If we can't use GL_UNPACK_ROW_LENGTH and the surface isn't tightly packed,
// we must allocate a tightly packed buffer and copy our pixels there.
packedPixelData = malloc(newSurface->w * newSurface->h * newSurface->format->BytesPerPixel);
if (!packedPixelData) {
SDL_FreeSurface(newSurface);
return;
}
SDL_ConvertPixels(newSurface->w, newSurface->h,
newSurface->format->format, newSurface->pixels, newSurface->pitch,
newSurface->format->format, packedPixelData, newSurface->w * newSurface->format->BytesPerPixel);
}
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, newSurface->w, newSurface->h, 0, GL_RGBA, GL_UNSIGNED_BYTE,
packedPixelData ? packedPixelData : newSurface->pixels);
if (packedPixelData) {
free(packedPixelData);
}
SDL_FRect overlayRect;
// These overlay positions differ from the other renderers because OpenGL
// places the origin in the lower-left corner instead of the upper-left.
if (type == Overlay::OverlayStatusUpdate) {
// Bottom Left
overlayRect.x = 0;
overlayRect.y = 0;
}
else if (type == Overlay::OverlayDebug) {
// Top left
overlayRect.x = 0;
overlayRect.y = m_ViewportHeight - newSurface->h;
} else {
SDL_assert(false);
}
overlayRect.w = newSurface->w;
overlayRect.h = newSurface->h;
SDL_FreeSurface(newSurface);
// Convert screen space to normalized device coordinates
StreamUtils::screenSpaceToNormalizedDeviceCoords(&overlayRect, m_ViewportWidth, m_ViewportHeight);
OVERLAY_VERTEX verts[] =
{
{overlayRect.x + overlayRect.w, overlayRect.y + overlayRect.h, 1.0f, 0.0f},
{overlayRect.x, overlayRect.y + overlayRect.h, 0.0f, 0.0f},
{overlayRect.x, overlayRect.y, 0.0f, 1.0f},
{overlayRect.x, overlayRect.y, 0.0f, 1.0f},
{overlayRect.x + overlayRect.w, overlayRect.y, 1.0f, 1.0f},
{overlayRect.x + overlayRect.w, overlayRect.y + overlayRect.h, 1.0f, 0.0f}
};
glBindBuffer(GL_ARRAY_BUFFER, m_OverlayVbos[type]);
glBufferData(GL_ARRAY_BUFFER, sizeof(verts), verts, GL_STATIC_DRAW);
SDL_AtomicSet(&m_OverlayHasValidData[type], 1);
}
if (!SDL_AtomicGet(&m_OverlayHasValidData[type])) {
// If the overlay is not populated yet or is stale, don't render it.
return;
}
glUseProgram(m_OverlayShaderProgram);
glBindBuffer(GL_ARRAY_BUFFER, m_OverlayVbos[type]);
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, sizeof(OVERLAY_VERTEX), (void*)offsetof(OVERLAY_VERTEX, x));
glEnableVertexAttribArray(0);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, sizeof(OVERLAY_VERTEX), (void*)offsetof(OVERLAY_VERTEX, u));
glEnableVertexAttribArray(1);
glActiveTexture(GL_TEXTURE0);
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[type]);
glUniform1i(m_OverlayShaderProgramParams[OVERLAY_PARAM_TEXTURE], 0);
glDrawArrays(GL_TRIANGLES, 0, 6);
}
int EGLRenderer::loadAndBuildShader(int shaderType,
const char *file) {
GLuint shader = glCreateShader(shaderType);
if (!shader || shader == GL_INVALID_ENUM) {
EGL_LOG(Error, "Can't create shader: %d", glGetError());
return 0;
}
auto sourceData = Path::readDataFile(file);
GLint len = sourceData.size();
const char *buf = sourceData.data();
glShaderSource(shader, 1, &buf, &len);
glCompileShader(shader);
GLint status;
glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
if (!status) {
char shaderLog[512];
glGetShaderInfoLog(shader, sizeof (shaderLog), nullptr, shaderLog);
EGL_LOG(Error, "Cannot load shader \"%s\": %s", file, shaderLog);
return 0;
}
return shader;
}
bool EGLRenderer::openDisplay(unsigned int platform, void* nativeDisplay)
{
PFNEGLGETPLATFORMDISPLAYPROC eglGetPlatformDisplayProc;
PFNEGLGETPLATFORMDISPLAYEXTPROC eglGetPlatformDisplayEXTProc;
m_EGLDisplay = EGL_NO_DISPLAY;
// NB: eglGetPlatformDisplay() and eglGetPlatformDisplayEXT() have slightly different definitions
eglGetPlatformDisplayProc = (typeof(eglGetPlatformDisplayProc))eglGetProcAddress("eglGetPlatformDisplay");
eglGetPlatformDisplayEXTProc = (typeof(eglGetPlatformDisplayEXTProc))eglGetProcAddress("eglGetPlatformDisplayEXT");
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// eglGetPlatformDisplay() is part of the EGL 1.5 core specification
if (eglGetPlatformDisplayProc != nullptr) {
m_EGLDisplay = eglGetPlatformDisplayProc(platform, nativeDisplay, nullptr);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Warn, "eglGetPlatformDisplay() failed: %d", eglGetError());
}
}
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// eglGetPlatformDisplayEXT() is an extension for EGL 1.4
const EGLExtensions eglExtensions(EGL_NO_DISPLAY);
if (eglExtensions.isSupported("EGL_EXT_platform_base")) {
if (eglGetPlatformDisplayEXTProc != nullptr) {
m_EGLDisplay = eglGetPlatformDisplayEXTProc(platform, nativeDisplay, nullptr);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Warn, "eglGetPlatformDisplayEXT() failed: %d", eglGetError());
}
}
else {
EGL_LOG(Warn, "EGL_EXT_platform_base supported but no eglGetPlatformDisplayEXT() export!");
}
}
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
// Finally, if all else fails, use eglGetDisplay()
m_EGLDisplay = eglGetDisplay((EGLNativeDisplayType)nativeDisplay);
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Error, "eglGetDisplay() failed: %d", eglGetError());
}
}
return m_EGLDisplay != EGL_NO_DISPLAY;
}
unsigned EGLRenderer::compileShader(const char* vertexShaderSrc, const char* fragmentShaderSrc) {
unsigned shader = 0;
GLuint vertexShader = loadAndBuildShader(GL_VERTEX_SHADER, vertexShaderSrc);
if (!vertexShader)
return false;
GLuint fragmentShader = loadAndBuildShader(GL_FRAGMENT_SHADER, fragmentShaderSrc);
if (!fragmentShader)
goto fragError;
shader = glCreateProgram();
if (!shader) {
EGL_LOG(Error, "Cannot create shader program");
goto progFailCreate;
}
glAttachShader(shader, vertexShader);
glAttachShader(shader, fragmentShader);
glLinkProgram(shader);
int status;
glGetProgramiv(shader, GL_LINK_STATUS, &status);
if (!status) {
char shader_log[512];
glGetProgramInfoLog(shader, sizeof (shader_log), nullptr, shader_log);
EGL_LOG(Error, "Cannot link shader program: %s", shader_log);
glDeleteProgram(shader);
shader = 0;
}
progFailCreate:
glDeleteShader(fragmentShader);
fragError:
glDeleteShader(vertexShader);
return shader;
}
bool EGLRenderer::compileShaders() {
SDL_assert(!m_ShaderProgram);
SDL_assert(!m_OverlayShaderProgram);
SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
// XXX: TODO: other formats
if (m_EGLImagePixelFormat == AV_PIX_FMT_NV12 || m_EGLImagePixelFormat == AV_PIX_FMT_P010) {
m_ShaderProgram = compileShader("egl_nv12.vert", "egl_nv12.frag");
if (!m_ShaderProgram) {
return false;
}
m_ShaderProgramParams[NV12_PARAM_YUVMAT] = glGetUniformLocation(m_ShaderProgram, "yuvmat");
m_ShaderProgramParams[NV12_PARAM_OFFSET] = glGetUniformLocation(m_ShaderProgram, "offset");
m_ShaderProgramParams[NV12_PARAM_PLANE1] = glGetUniformLocation(m_ShaderProgram, "plane1");
m_ShaderProgramParams[NV12_PARAM_PLANE2] = glGetUniformLocation(m_ShaderProgram, "plane2");
}
else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
m_ShaderProgram = compileShader("egl_opaque.vert", "egl_opaque.frag");
if (!m_ShaderProgram) {
return false;
}
m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE] = glGetUniformLocation(m_ShaderProgram, "uTexture");
}
else {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unsupported EGL pixel format: %d",
m_EGLImagePixelFormat);
SDL_assert(false);
return false;
}
m_OverlayShaderProgram = compileShader("egl_overlay.vert", "egl_overlay.frag");
if (!m_OverlayShaderProgram) {
return false;
}
m_OverlayShaderProgramParams[OVERLAY_PARAM_TEXTURE] = glGetUniformLocation(m_OverlayShaderProgram, "uTexture");
return true;
}
bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
{
m_Window = params->window;
// It's not safe to attempt to opportunistically create a GLES2
// renderer prior to 2.0.10. If GLES2 isn't available, SDL will
// attempt to dereference a null pointer and crash Moonlight.
// https://bugzilla.libsdl.org/show_bug.cgi?id=4350
// https://hg.libsdl.org/SDL/rev/84618d571795
if (!SDL_VERSION_ATLEAST(2, 0, 10)) {
EGL_LOG(Error, "Not supported until SDL 2.0.10");
return false;
}
// HACK: Work around bug where renderer will repeatedly fail with:
// SDL_CreateRenderer() failed: Could not create GLES window surface
// Don't retry if we've already failed to create a renderer for this
// window *unless* the format has changed from 10-bit to 8-bit.
if (m_Window == s_LastFailedWindow &&
!!(params->videoFormat & VIDEO_FORMAT_MASK_10BIT) ==
!!(s_LastFailedVideoFormat & VIDEO_FORMAT_MASK_10BIT)) {
EGL_LOG(Error, "SDL_CreateRenderer() already failed on this window!");
return false;
}
// This hint will ensure we use EGL to retreive our GL context,
// even on X11 where that is not the default. EGL is required
// to avoid a crash in Mesa.
// https://gitlab.freedesktop.org/mesa/mesa/issues/1011
SDL_SetHint(SDL_HINT_OPENGL_ES_DRIVER, "1");
SDL_GL_SetAttribute(SDL_GL_CONTEXT_PROFILE_MASK, SDL_GL_CONTEXT_PROFILE_ES);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, 3);
SDL_GL_SetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, 0);
int renderIndex;
int maxRenderers = SDL_GetNumRenderDrivers();
SDL_assert(maxRenderers >= 0);
SDL_RendererInfo renderInfo;
for (renderIndex = 0; renderIndex < maxRenderers; ++renderIndex) {
if (SDL_GetRenderDriverInfo(renderIndex, &renderInfo))
continue;
if (!strcmp(renderInfo.name, "opengles2")) {
SDL_assert(renderInfo.flags & SDL_RENDERER_ACCELERATED);
break;
}
}
if (renderIndex == maxRenderers) {
EGL_LOG(Error, "Could not find a suitable SDL_Renderer");
return false;
}
m_DummyRenderer = SDL_CreateRenderer(m_Window, renderIndex, SDL_RENDERER_ACCELERATED);
if (!m_DummyRenderer) {
// Print the error here (before it gets clobbered), but ensure that we flush window
// events just in case SDL re-created the window before eventually failing.
EGL_LOG(Error, "SDL_CreateRenderer() failed: %s", SDL_GetError());
}
// SDL_CreateRenderer() can end up having to recreate our window (SDL_RecreateWindow())
// to ensure it's compatible with the renderer's OpenGL context. If that happens, we
// can get spurious SDL_WINDOWEVENT events that will cause us to (again) recreate our
// renderer. This can lead to an infinite to renderer recreation, so discard all
// SDL_WINDOWEVENT events after SDL_CreateRenderer().
Session* session = Session::get();
if (session != nullptr) {
// If we get here during a session, we need to synchronize with the event loop
// to ensure we don't drop any important events.
session->flushWindowEvents();
}
else {
// If we get here prior to the start of a session, just pump and flush ourselves.
SDL_PumpEvents();
SDL_FlushEvent(SDL_WINDOWEVENT);
}
// Now we finally bail if we failed during SDL_CreateRenderer() above.
if (!m_DummyRenderer) {
s_LastFailedWindow = m_Window;
s_LastFailedVideoFormat = params->videoFormat;
return false;
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(params->window, &info)) {
EGL_LOG(Error, "SDL_GetWindowWMInfo() failed: %s", SDL_GetError());
return false;
}
switch (info.subsystem) {
#ifdef SDL_VIDEO_DRIVER_WAYLAND
case SDL_SYSWM_WAYLAND:
if (!openDisplay(EGL_PLATFORM_WAYLAND_KHR, info.info.wl.display)) {
return false;
}
break;
#endif
#ifdef SDL_VIDEO_DRIVER_X11
case SDL_SYSWM_X11:
if (!openDisplay(EGL_PLATFORM_X11_KHR, info.info.x11.display)) {
return false;
}
break;
#endif
#if SDL_VERSION_ATLEAST(2, 0, 15) && defined(SDL_VIDEO_DRIVER_KMSDRM)
case SDL_SYSWM_KMSDRM:
if (!openDisplay(EGL_PLATFORM_GBM_KHR, info.info.kmsdrm.gbm_dev)) {
return false;
}
break;
#endif
default:
EGL_LOG(Error, "not compatible with SYSWM");
return false;
}
if (m_EGLDisplay == EGL_NO_DISPLAY) {
EGL_LOG(Error, "Cannot get EGL display: %d", eglGetError());
return false;
}
if (!(m_Context = SDL_GL_CreateContext(params->window))) {
EGL_LOG(Error, "Cannot create OpenGL context: %s", SDL_GetError());
return false;
}
if (SDL_GL_MakeCurrent(params->window, m_Context)) {
EGL_LOG(Error, "Cannot use created EGL context: %s", SDL_GetError());
return false;
}
{
int r, g, b, a;
SDL_GL_GetAttribute(SDL_GL_RED_SIZE, &r);
SDL_GL_GetAttribute(SDL_GL_GREEN_SIZE, &g);
SDL_GL_GetAttribute(SDL_GL_BLUE_SIZE, &b);
SDL_GL_GetAttribute(SDL_GL_ALPHA_SIZE, &a);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Color buffer is: R%dG%dB%dA%d",
r, g, b, a);
}
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MAJOR_VERSION, &m_GlesMajorVersion);
SDL_GL_GetAttribute(SDL_GL_CONTEXT_MINOR_VERSION, &m_GlesMinorVersion);
// We can use GL_UNPACK_ROW_LENGTH for a more optimized upload of non-tightly-packed textures
m_HasExtUnpackSubimage = SDL_GL_ExtensionSupported("GL_EXT_unpack_subimage");
const EGLExtensions eglExtensions(m_EGLDisplay);
if (!eglExtensions.isSupported("EGL_KHR_image_base") &&
!eglExtensions.isSupported("EGL_KHR_image")) {
EGL_LOG(Error, "EGL_KHR_image unsupported");
return false;
}
else if (!SDL_GL_ExtensionSupported("GL_OES_EGL_image")) {
EGL_LOG(Error, "GL_OES_EGL_image unsupported");
return false;
}
if (!m_Backend->initializeEGL(m_EGLDisplay, eglExtensions))
return false;
if (!(m_glEGLImageTargetTexture2DOES = (typeof(m_glEGLImageTargetTexture2DOES))eglGetProcAddress("glEGLImageTargetTexture2DOES"))) {
EGL_LOG(Error,
"EGL: cannot retrieve `glEGLImageTargetTexture2DOES` address");
return false;
}
// Vertex arrays are an extension on OpenGL ES 2.0
if (SDL_GL_ExtensionSupported("GL_OES_vertex_array_object")) {
m_glGenVertexArraysOES = (typeof(m_glGenVertexArraysOES))eglGetProcAddress("glGenVertexArraysOES");
m_glBindVertexArrayOES = (typeof(m_glBindVertexArrayOES))eglGetProcAddress("glBindVertexArrayOES");
m_glDeleteVertexArraysOES = (typeof(m_glDeleteVertexArraysOES))eglGetProcAddress("glDeleteVertexArraysOES");
}
else {
// They are included in OpenGL ES 3.0 as part of the standard
m_glGenVertexArraysOES = (typeof(m_glGenVertexArraysOES))eglGetProcAddress("glGenVertexArrays");
m_glBindVertexArrayOES = (typeof(m_glBindVertexArrayOES))eglGetProcAddress("glBindVertexArray");
m_glDeleteVertexArraysOES = (typeof(m_glDeleteVertexArraysOES))eglGetProcAddress("glDeleteVertexArrays");
}
if (!m_glGenVertexArraysOES || !m_glBindVertexArrayOES || !m_glDeleteVertexArraysOES) {
EGL_LOG(Error, "Failed to find VAO functions");
return false;
}
// EGL_KHR_fence_sync is an extension for EGL 1.1+
if (eglExtensions.isSupported("EGL_KHR_fence_sync")) {
// eglCreateSyncKHR() has a slightly different prototype to eglCreateSync()
m_eglCreateSyncKHR = (typeof(m_eglCreateSyncKHR))eglGetProcAddress("eglCreateSyncKHR");
m_eglDestroySync = (typeof(m_eglDestroySync))eglGetProcAddress("eglDestroySyncKHR");
m_eglClientWaitSync = (typeof(m_eglClientWaitSync))eglGetProcAddress("eglClientWaitSyncKHR");
}
else {
// EGL 1.5 introduced sync support to the core specification
m_eglCreateSync = (typeof(m_eglCreateSync))eglGetProcAddress("eglCreateSync");
m_eglDestroySync = (typeof(m_eglDestroySync))eglGetProcAddress("eglDestroySync");
m_eglClientWaitSync = (typeof(m_eglClientWaitSync))eglGetProcAddress("eglClientWaitSync");
}
if (!(m_eglCreateSync || m_eglCreateSyncKHR) || !m_eglDestroySync || !m_eglClientWaitSync) {
EGL_LOG(Warn, "Failed to find sync functions");
// Sub-optimal, but not fatal
m_eglCreateSync = nullptr;
m_eglCreateSyncKHR = nullptr;
m_eglDestroySync = nullptr;
m_eglClientWaitSync = nullptr;
}
/* Compute the video region size in order to keep the aspect ratio of the
* video stream.
*/
SDL_Rect src, dst;
src.x = src.y = dst.x = dst.y = 0;
src.w = params->width;
src.h = params->height;
SDL_GL_GetDrawableSize(m_Window, &dst.w, &dst.h);
StreamUtils::scaleSourceToDestinationSurface(&src, &dst);
glViewport(dst.x, dst.y, dst.w, dst.h);
m_ViewportWidth = dst.w;
m_ViewportHeight = dst.h;
// SDL always uses swap interval 0 under the hood on Wayland systems,
// because the compositor guarantees tear-free rendering. In this
// situation, swap interval > 0 behaves as a frame pacing option
// rather than a way to eliminate tearing as SDL will block in
// SwapBuffers until the compositor consumes the frame. This will
// needlessly increases latency, so we should avoid it.
//
// HACK: In SDL 2.0.22+ on GNOME systems with fractional DPI scaling,
// the Wayland viewport can be stale when using Super+Left/Right/Up
// to resize the window. This seems to happen significantly more often
// with vsync enabled, so this also mitigates that problem too.
if (params->enableVsync
#ifdef SDL_VIDEO_DRIVER_WAYLAND
&& info.subsystem != SDL_SYSWM_WAYLAND
#endif
) {
SDL_GL_SetSwapInterval(1);
#if SDL_VERSION_ATLEAST(2, 0, 15) && defined(SDL_VIDEO_DRIVER_KMSDRM)
// The SDL KMSDRM backend already enforces double buffering (due to
// SDL_HINT_VIDEO_DOUBLE_BUFFER=1), so calling glFinish() after
// SDL_GL_SwapWindow() will block an extra frame and lock rendering
// at 1/2 the display refresh rate.
if (info.subsystem != SDL_SYSWM_KMSDRM)
#endif
{
m_BlockingSwapBuffers = true;
}
} else {
SDL_GL_SetSwapInterval(0);
}
if (!params->testOnly) {
// Draw a black frame until the video stream starts rendering
glClearColor(0, 0, 0, 1);
glClear(GL_COLOR_BUFFER_BIT);
SDL_GL_SwapWindow(params->window);
}
glGenTextures(EGL_MAX_PLANES, m_Textures);
for (size_t i = 0; i < EGL_MAX_PLANES; ++i) {
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
glGenBuffers(Overlay::OverlayMax, m_OverlayVbos);
glGenTextures(Overlay::OverlayMax, m_OverlayTextures);
for (size_t i = 0; i < Overlay::OverlayMax; ++i) {
glBindTexture(GL_TEXTURE_2D, m_OverlayTextures[i]);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
}
glEnable(GL_BLEND);
glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
GLenum err = glGetError();
if (err != GL_NO_ERROR)
EGL_LOG(Error, "OpenGL error: %d", err);
// Detach the context from this thread, so the render thread can attach it
SDL_GL_MakeCurrent(m_Window, nullptr);
#ifdef SDL_HINT_VIDEO_X11_FORCE_EGL
if (err == GL_NO_ERROR) {
// If we got a working GL implementation via EGL, avoid using GLX from now on.
// GLX will cause problems if we later want to use EGL again on this window.
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "EGL passed preflight checks. Using EGL for GL context creation.");
SDL_SetHint(SDL_HINT_VIDEO_X11_FORCE_EGL, "1");
}
#endif
return err == GL_NO_ERROR;
}
const float *EGLRenderer::getColorOffsets(const AVFrame* frame) {
static const float limitedOffsets[] = { 16.0f / 255.0f, 128.0f / 255.0f, 128.0f / 255.0f };
static const float fullOffsets[] = { 0.0f, 128.0f / 255.0f, 128.0f / 255.0f };
return isFrameFullRange(frame) ? fullOffsets : limitedOffsets;
}
const float *EGLRenderer::getColorMatrix(const AVFrame* frame) {
/* The conversion matrices are shamelessly stolen from linux:
* drivers/media/platform/imx-pxp.c:pxp_setup_csc
*/
static const float bt601Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.3917f, 2.0172f,
1.5960f, -0.8129f, 0.0f
};
static const float bt601Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.3441f, 1.7720f,
1.4020f, -0.7141f, 0.0f
};
static const float bt709Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.2132f, 2.1124f,
1.7927f, -0.5329f, 0.0f
};
static const float bt709Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1873f, 1.8556f,
1.5748f, -0.4681f, 0.0f
};
static const float bt2020Lim[] = {
1.1644f, 1.1644f, 1.1644f,
0.0f, -0.1874f, 2.1418f,
1.6781f, -0.6505f, 0.0f
};
static const float bt2020Full[] = {
1.0f, 1.0f, 1.0f,
0.0f, -0.1646f, 1.8814f,
1.4746f, -0.5714f, 0.0f
};
bool fullRange = isFrameFullRange(frame);
switch (getFrameColorspace(frame)) {
case COLORSPACE_REC_601:
return fullRange ? bt601Full : bt601Lim;
case COLORSPACE_REC_709:
return fullRange ? bt709Full : bt709Lim;
case COLORSPACE_REC_2020:
return fullRange ? bt2020Full : bt2020Lim;
default:
SDL_assert(false);
}
return bt601Lim;
}
bool EGLRenderer::specialize() {
SDL_assert(!m_VAO);
if (!compileShaders())
return false;
// The viewport should have the aspect ratio of the video stream
static const float vertices[] = {
// pos .... // tex coords
1.0f, 1.0f, 1.0f, 0.0f,
1.0f, -1.0f, 1.0f, 1.0f,
-1.0f, -1.0f, 0.0f, 1.0f,
-1.0f, 1.0f, 0.0f, 0.0f,
};
static const unsigned int indices[] = {
0, 1, 3,
1, 2, 3,
};
glUseProgram(m_ShaderProgram);
unsigned int VBO, EBO;
m_glGenVertexArraysOES(1, &m_VAO);
glGenBuffers(1, &VBO);
glGenBuffers(1, &EBO);
m_glBindVertexArrayOES(m_VAO);
glBindBuffer(GL_ARRAY_BUFFER, VBO);
glBufferData(GL_ARRAY_BUFFER, sizeof (vertices), vertices, GL_STATIC_DRAW);
glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, EBO);
glBufferData(GL_ELEMENT_ARRAY_BUFFER, sizeof (indices), indices, GL_STATIC_DRAW);
glVertexAttribPointer(0, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)0);
glEnableVertexAttribArray(0);
glVertexAttribPointer(1, 2, GL_FLOAT, GL_FALSE, 4 * sizeof(float), (void*)(2 * sizeof (float)));
glEnableVertexAttribArray(1);
glBindBuffer(GL_ARRAY_BUFFER, 0);
m_glBindVertexArrayOES(0);
glDeleteBuffers(1, &VBO);
glDeleteBuffers(1, &EBO);
GLenum err = glGetError();
if (err != GL_NO_ERROR) {
EGL_LOG(Error, "OpenGL error: %d", err);
}
return err == GL_NO_ERROR;
}
void EGLRenderer::cleanupRenderContext()
{
// Detach the context from the render thread so the destructor can attach it
SDL_GL_MakeCurrent(m_Window, nullptr);
}
void EGLRenderer::waitToRender()
{
// Ensure our GL context is active on this thread
// See comment in renderFrame() for more details.
SDL_GL_MakeCurrent(m_Window, m_Context);
// Wait for the previous buffer swap to finish before picking the next frame to render.
// This way we'll get the latest available frame and render it without blocking.
if (m_BlockingSwapBuffers) {
// Try to use eglClientWaitSync() if the driver supports it
if (m_LastRenderSync != EGL_NO_SYNC) {
SDL_assert(m_eglClientWaitSync != nullptr);
m_eglClientWaitSync(m_EGLDisplay, m_LastRenderSync, EGL_SYNC_FLUSH_COMMANDS_BIT, EGL_FOREVER);
}
else {
// Use glFinish() if fences aren't available
glFinish();
}
}
}
void EGLRenderer::renderFrame(AVFrame* frame)
{
EGLImage imgs[EGL_MAX_PLANES];
// Attach our GL context to the render thread
// NB: It should already be current, unless the SDL render event watcher
// performs a rendering operation (like a viewport update on resize) on
// our fake SDL_Renderer. If it's already current, this is a no-op.
SDL_GL_MakeCurrent(m_Window, m_Context);
// Find the native read-back format and load the shaders
if (m_EGLImagePixelFormat == AV_PIX_FMT_NONE) {
m_EGLImagePixelFormat = m_Backend->getEGLImagePixelFormat();
EGL_LOG(Info, "EGLImage pixel format: %d", m_EGLImagePixelFormat);
SDL_assert(m_EGLImagePixelFormat != AV_PIX_FMT_NONE);
if (!specialize()) {
m_EGLImagePixelFormat = AV_PIX_FMT_NONE;
// Failure to specialize is fatal. We must reset the renderer
// to recover successfully.
//
// Note: This seems to be easy to trigger when transitioning from
// maximized mode by dragging the window down on GNOME 42 using
// XWayland. Other strategies like calling glGetError() don't seem
// to be able to detect this situation for some reason.
SDL_Event event;
event.type = SDL_RENDER_TARGETS_RESET;
SDL_PushEvent(&event);
return;
}
}
ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs);
if (plane_count < 0)
return;
for (ssize_t i = 0; i < plane_count; ++i) {
glActiveTexture(GL_TEXTURE0 + i);
glBindTexture(GL_TEXTURE_EXTERNAL_OES, m_Textures[i]);
m_glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, imgs[i]);
}
glClear(GL_COLOR_BUFFER_BIT);
glUseProgram(m_ShaderProgram);
m_glBindVertexArrayOES(m_VAO);
// Bind parameters for the shaders
if (m_EGLImagePixelFormat == AV_PIX_FMT_NV12 || m_EGLImagePixelFormat == AV_PIX_FMT_P010) {
glUniformMatrix3fv(m_ShaderProgramParams[NV12_PARAM_YUVMAT], 1, GL_FALSE, getColorMatrix(frame));
glUniform3fv(m_ShaderProgramParams[NV12_PARAM_OFFSET], 1, getColorOffsets(frame));
glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE1], 0);
glUniform1i(m_ShaderProgramParams[NV12_PARAM_PLANE2], 1);
}
else if (m_EGLImagePixelFormat == AV_PIX_FMT_DRM_PRIME) {
glUniform1i(m_ShaderProgramParams[OPAQUE_PARAM_TEXTURE], 0);
}
glDrawElements(GL_TRIANGLES, 6, GL_UNSIGNED_INT, 0);
m_glBindVertexArrayOES(0);
for (int i = 0; i < Overlay::OverlayMax; i++) {
renderOverlay((Overlay::OverlayType)i);
}
SDL_GL_SwapWindow(m_Window);
if (m_BlockingSwapBuffers) {
// This glClear() requires the new back buffer to complete. This ensures
// our eglClientWaitSync() or glFinish() call in waitToRender() will not
// return before the new buffer is actually ready for rendering.
glClear(GL_COLOR_BUFFER_BIT);
// If we this EGL implementation supports fences, use those to delay
// rendering the next frame until this one is completed. If not, we'll
// have to just use glFinish().
if (m_eglClientWaitSync != nullptr) {
// Delete the sync object from last render
if (m_LastRenderSync != EGL_NO_SYNC) {
m_eglDestroySync(m_EGLDisplay, m_LastRenderSync);
}
// Create a new sync object that will be signalled when the buffer swap is completed
if (m_eglCreateSync != nullptr) {
m_LastRenderSync = m_eglCreateSync(m_EGLDisplay, EGL_SYNC_FENCE, nullptr);
}
else {
SDL_assert(m_eglCreateSyncKHR != nullptr);
m_LastRenderSync = m_eglCreateSyncKHR(m_EGLDisplay, EGL_SYNC_FENCE, nullptr);
}
}
}
m_Backend->freeEGLImages(m_EGLDisplay, imgs);
// Free the DMA-BUF backing the last frame now that it is definitely
// no longer being used anymore. While the PRIME FD stays around until
// EGL is done with it, the memory backing it may be reused by FFmpeg
// before the GPU has read it. This is particularly noticeable on the
// RK3288-based TinkerBoard when V-Sync is disabled.
av_frame_unref(m_LastFrame);
av_frame_move_ref(m_LastFrame, frame);
}
bool EGLRenderer::testRenderFrame(AVFrame* frame)
{
EGLImage imgs[EGL_MAX_PLANES];
#ifdef HAVE_MMAL
// EGL rendering is so slow on the Raspberry Pi that we should basically
// never use it. It is suitable for 1080p 30 FPS on a good day, and much
// much less than that if you decide to do something crazy like stream
// in full-screen. It's nice that it at least works now on Bullseye, but
// it's so slow that we actually wish it didn't.
if (qgetenv("RPI_ALLOW_EGL_RENDER") != "1") {
return false;
}
#endif
// Make sure we can get working EGLImages from the backend renderer.
// Some devices (Raspberry Pi) will happily decode into DRM formats that
// its own GL implementation won't accept in eglCreateImage().
ssize_t plane_count = m_Backend->exportEGLImages(frame, m_EGLDisplay, imgs);
if (plane_count <= 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Backend failed to export EGL image for test frame");
return false;
}
m_Backend->freeEGLImages(m_EGLDisplay, imgs);
return true;
}
Reference in a new issue View git blame Copy permalink