Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2025-01-09 09:48:43 +00:00
Code Issues Projects Releases Packages Wiki Activity

Export composed VA surface layers for EGL import if supported

Browse source 
Importing composed formats is more efficient and performant because it allows
tiled and/or compressed surfaces to be directly sampled by shaders without
requiring a conversion to a linear planar format first.
This commit is contained in:
Cameron Gutman 2023-09-26 22:58:13 -05:00
parent b01dfea343
commit 1efdeeb9dc
4 changed files with 275 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

160
app/streaming/video/ffmpeg-renderers/eglimagefactory.cpp
View file

@ -1,9 +1,21 @@
#include "eglimagefactory.h"
// Don't take a dependency on libdrm just for this constant
// Don't take a dependency on libdrm just for these constants
#ifndef DRM_FORMAT_MOD_INVALID
#define DRM_FORMAT_MOD_INVALID ((1ULL << 56) - 1)
#endif
#ifndef DRM_FORMAT_MOD_LINEAR
#define DRM_FORMAT_MOD_LINEAR 0
#endif
#ifndef fourcc_code
#define fourcc_code(a, b, c, d) ((uint32_t)(a) | ((uint32_t)(b) << 8) | ((uint32_t)(c) << 16) | ((uint32_t)(d) << 24))
#endif
#ifndef DRM_FORMAT_R8
#define DRM_FORMAT_R8 fourcc_code('R', '8', ' ', ' ')
#endif
#ifndef DRM_FORMAT_GR88
#define DRM_FORMAT_GR88 fourcc_code('G', 'R', '8', '8')
#endif
EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) :
m_Renderer(renderer),
@ -11,7 +23,9 @@ EglImageFactory::EglImageFactory(IFFmpegRenderer* renderer) :
m_eglCreateImage(nullptr),
m_eglDestroyImage(nullptr),
m_eglCreateImageKHR(nullptr),
m_eglDestroyImageKHR(nullptr)
m_eglDestroyImageKHR(nullptr),
m_eglQueryDmaBufFormatsEXT(nullptr),
m_eglQueryDmaBufModifiersEXT(nullptr)
{
}
@ -25,6 +39,10 @@ bool EglImageFactory::initializeEGL(EGLDisplay,
}
m_EGLExtDmaBuf = ext.isSupported("EGL_EXT_image_dma_buf_import_modifiers");
if (m_EGLExtDmaBuf) {
m_eglQueryDmaBufFormatsEXT = (typeof(m_eglQueryDmaBufFormatsEXT))eglGetProcAddress("eglQueryDmaBufFormatsEXT");
m_eglQueryDmaBufModifiersEXT = (typeof(m_eglQueryDmaBufModifiersEXT))eglGetProcAddress("eglQueryDmaBufModifiersEXT");
}
// NB: eglCreateImage() and eglCreateImageKHR() have slightly different definitions
m_eglCreateImage = (typeof(m_eglCreateImage))eglGetProcAddress("eglCreateImage");
@ -42,6 +60,8 @@ bool EglImageFactory::initializeEGL(EGLDisplay,
return true;
}
#ifdef HAVE_DRM
ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
{
memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES);
@ -216,9 +236,11 @@ ssize_t EglImageFactory::exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* d
return 1;
}
#endif
#ifdef HAVE_LIBVA
ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[])
ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescriptor *vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES])
{
ssize_t count = 0;
@ -229,8 +251,8 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
for (size_t i = 0; i < vaFrame->num_layers; ++i) {
const auto &layer = vaFrame->layers[i];
// Max 31 attributes (1 key + 1 value for each)
const int EGL_ATTRIB_COUNT = 31 * 2;
// Max 33 attributes (1 key + 1 value for each)
const int EGL_ATTRIB_COUNT = 33 * 2;
EGLAttrib attribs[EGL_ATTRIB_COUNT] = {
EGL_LINUX_DRM_FOURCC_EXT, layer.drm_format,
EGL_WIDTH, i == 0 ? frame->width : frame->width / 2,
@ -308,6 +330,61 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
}
}
// For composed exports, add the YUV metadata
if (vaFrame->num_layers == 1) {
// Add colorspace metadata
switch (m_Renderer->getFrameColorspace(frame)) {
case COLORSPACE_REC_601:
attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT;
attribs[attribIndex++] = EGL_ITU_REC601_EXT;
break;
case COLORSPACE_REC_709:
attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT;
attribs[attribIndex++] = EGL_ITU_REC709_EXT;
break;
case COLORSPACE_REC_2020:
attribs[attribIndex++] = EGL_YUV_COLOR_SPACE_HINT_EXT;
attribs[attribIndex++] = EGL_ITU_REC2020_EXT;
break;
}
// Add color range metadata
attribs[attribIndex++] = EGL_SAMPLE_RANGE_HINT_EXT;
attribs[attribIndex++] = m_Renderer->isFrameFullRange(frame) ? EGL_YUV_FULL_RANGE_EXT : EGL_YUV_NARROW_RANGE_EXT;
// Add chroma siting metadata
switch (frame->chroma_location) {
case AVCHROMA_LOC_LEFT:
case AVCHROMA_LOC_TOPLEFT:
attribs[attribIndex++] = EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT;
attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_EXT;
break;
case AVCHROMA_LOC_CENTER:
case AVCHROMA_LOC_TOP:
attribs[attribIndex++] = EGL_YUV_CHROMA_HORIZONTAL_SITING_HINT_EXT;
attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_5_EXT;
break;
default:
break;
}
switch (frame->chroma_location) {
case AVCHROMA_LOC_TOPLEFT:
case AVCHROMA_LOC_TOP:
attribs[attribIndex++] = EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT;
attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_EXT;
break;
case AVCHROMA_LOC_LEFT:
case AVCHROMA_LOC_CENTER:
attribs[attribIndex++] = EGL_YUV_CHROMA_VERTICAL_SITING_HINT_EXT;
attribs[attribIndex++] = EGL_YUV_CHROMA_SITING_0_5_EXT;
break;
default:
break;
}
}
// Terminate the attribute list
attribs[attribIndex++] = EGL_NONE;
SDL_assert(attribIndex <= EGL_ATTRIB_COUNT);
@ -341,6 +418,7 @@ ssize_t EglImageFactory::exportVAImages(AVFrame *frame, VADRMPRIMESurfaceDescrip
++count;
}
return count;
fail:
@ -348,6 +426,77 @@ fail:
return -1;
}
bool EglImageFactory::supportsImportingFormat(EGLDisplay dpy, EGLint format)
{
if (!m_eglQueryDmaBufFormatsEXT) {
// These are the standard formats used for importing separate layers of NV12.
// We will assume all EGL implementations can handle these.
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming R8 and GR88 format support because eglQueryDmaBufFormatsEXT() is not supported");
return format == DRM_FORMAT_R8 || format == DRM_FORMAT_GR88;
}
// Get the number of formats
EGLint numFormats;
if (!m_eglQueryDmaBufFormatsEXT(dpy, 0, nullptr, &numFormats)) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglQueryDmaBufFormatsEXT() #1 failed: %d", eglGetError());
return false;
}
EGLint formats[numFormats];
if (!m_eglQueryDmaBufFormatsEXT(dpy, numFormats, formats, &numFormats)) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglQueryDmaBufFormatsEXT() #2 failed: %d", eglGetError());
return false;
}
for (EGLint i = 0; i < numFormats; i++) {
if (format == formats[i]) {
return true;
}
}
return false;
}
bool EglImageFactory::supportsImportingModifier(EGLDisplay dpy, EGLint format, EGLuint64KHR modifier)
{
// We assume linear and no modifiers are always supported
if (modifier == DRM_FORMAT_MOD_LINEAR || modifier == DRM_FORMAT_MOD_INVALID) {
return true;
}
if (!m_eglQueryDmaBufModifiersEXT) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming linear modifier support because eglQueryDmaBufModifiersEXT() is not supported");
return false;
}
// Get the number of modifiers
EGLint numModifiers;
if (!m_eglQueryDmaBufModifiersEXT(dpy, format, 0, nullptr, nullptr, &numModifiers)) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglQueryDmaBufModifiersEXT() #1 failed: %d", eglGetError());
return false;
}
EGLuint64KHR modifiers[numModifiers];
if (!m_eglQueryDmaBufModifiersEXT(dpy, format, numModifiers, modifiers, nullptr, &numModifiers)) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"eglQueryDmaBufModifiersEXT() #2 failed: %d", eglGetError());
return false;
}
for (EGLint i = 0; i < numModifiers; i++) {
if (modifier == modifiers[i]) {
return true;
}
}
return false;
}
#endif
void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
@ -361,4 +510,5 @@ void EglImageFactory::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLAN
}
}
}
memset(images, 0, sizeof(EGLImage) * EGL_MAX_PLANES);
}

8
app/streaming/video/ffmpeg-renderers/eglimagefactory.h
View file

@ -14,12 +14,18 @@ class EglImageFactory
public:
EglImageFactory(IFFmpegRenderer* renderer);
bool initializeEGL(EGLDisplay, const EGLExtensions &ext);
#ifdef HAVE_DRM
ssize_t exportDRMImages(AVFrame* frame, AVDRMFrameDescriptor* drmFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
#endif
#ifdef HAVE_LIBVA
ssize_t exportVAImages(AVFrame* frame, VADRMPRIMESurfaceDescriptor* vaFrame, EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
#endif
bool supportsImportingFormat(EGLDisplay dpy, EGLint format);
bool supportsImportingModifier(EGLDisplay dpy, EGLint format, EGLuint64KHR modifier);
void freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]);
private:
@ -29,4 +35,6 @@ private:
PFNEGLDESTROYIMAGEPROC m_eglDestroyImage;
PFNEGLCREATEIMAGEKHRPROC m_eglCreateImageKHR;
PFNEGLDESTROYIMAGEKHRPROC m_eglDestroyImageKHR;
PFNEGLQUERYDMABUFFORMATSEXTPROC m_eglQueryDmaBufFormatsEXT;
PFNEGLQUERYDMABUFMODIFIERSEXTPROC m_eglQueryDmaBufModifiersEXT;
};

122
app/streaming/video/ffmpeg-renderers/vaapi.cpp
View file

@ -18,12 +18,12 @@ VAAPIRenderer::VAAPIRenderer(int decoderSelectionPass)
m_BlacklistedForDirectRendering(false),
m_OverlayMutex(nullptr)
#ifdef HAVE_EGL
, m_EglImageFactory(this)
, m_EglExportType(EglExportType::Unknown),
m_EglImageFactory(this)
#endif
{
#ifdef HAVE_EGL
m_PrimeDescriptor.num_layers = 0;
m_PrimeDescriptor.num_objects = 0;
SDL_zero(m_PrimeDescriptor);
#endif
SDL_zero(m_OverlayImage);
@ -795,12 +795,11 @@ VAAPIRenderer::renderFrame(AVFrame* frame)
// Ensure that vaExportSurfaceHandle() is supported by the VA-API driver
bool
VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) {
VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag, VADRMPRIMESurfaceDescriptor* descriptor) {
AVHWDeviceContext* deviceContext = (AVHWDeviceContext*)m_HwContext->data;
AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)deviceContext->hwctx;
VASurfaceID surfaceId;
VAStatus st;
VADRMPRIMESurfaceDescriptor descriptor;
VASurfaceAttrib attrs[2];
int attributeCount = 0;
@ -848,7 +847,7 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) {
surfaceId,
VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
VA_EXPORT_SURFACE_READ_ONLY | layerTypeFlag,
&descriptor);
descriptor);
vaDestroySurfaces(vaDeviceContext->display, &surfaceId, 1);
@ -858,8 +857,9 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) {
return false;
}
for (size_t i = 0; i < descriptor.num_objects; ++i) {
close(descriptor.objects[i].fd);
for (size_t i = 0; i < descriptor->num_objects; ++i) {
close(descriptor->objects[i].fd);
descriptor->objects[i].fd = -1;
}
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
@ -874,33 +874,122 @@ VAAPIRenderer::canExportSurfaceHandle(int layerTypeFlag) {
bool
VAAPIRenderer::canExportEGL() {
// Our EGL export logic requires exporting separate layers
return canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS);
VADRMPRIMESurfaceDescriptor descriptor;
return (qgetenv("VAAPI_EGL_SEPARATE_LAYERS") != "1" && canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor)) ||
canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS, &descriptor);
}
AVPixelFormat VAAPIRenderer::getEGLImagePixelFormat() {
return (m_VideoFormat & VIDEO_FORMAT_MASK_10BIT) ?
AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
switch (m_EglExportType) {
case EglExportType::Separate:
return (m_VideoFormat & VIDEO_FORMAT_MASK_10BIT) ?
AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
case EglExportType::Composed:
// This tells EGLRenderer to treat the EGLImage as a single opaque texture
return AV_PIX_FMT_DRM_PRIME;
case EglExportType::Unknown:
SDL_assert(m_EglExportType != EglExportType::Unknown);
break;
}
return AV_PIX_FMT_NONE;
}
bool
VAAPIRenderer::initializeEGL(EGLDisplay dpy,
const EGLExtensions &ext) {
return m_EglImageFactory.initializeEGL(dpy, ext);
VADRMPRIMESurfaceDescriptor descriptor;
if (!m_EglImageFactory.initializeEGL(dpy, ext)) {
return false;
}
// Prefer exporting composed images absent a user override or lack of support for exporting or importing
if (qgetenv("VAAPI_EGL_SEPARATE_LAYERS") == "1") {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Exporting separate layers due to environment variable override");
m_EglExportType = EglExportType::Separate;
}
else if (!canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Exporting separate layers due to lack of support for VA_EXPORT_SURFACE_COMPOSED_LAYERS");
m_EglExportType = EglExportType::Separate;
}
else if (!m_EglImageFactory.supportsImportingFormat(dpy, descriptor.layers[0].drm_format)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Exporting separate layers due to lack of support for importing format: %08x", descriptor.layers[0].drm_format);
m_EglExportType = EglExportType::Separate;
}
else if (!m_EglImageFactory.supportsImportingModifier(dpy, descriptor.layers[0].drm_format, descriptor.objects[0].drm_format_modifier)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Exporting separate layers due to lack of support for importing format and modifier: %08x %016lx",
descriptor.layers[0].drm_format,
descriptor.objects[0].drm_format_modifier);
m_EglExportType = EglExportType::Separate;
}
else {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Exporting composed layers with format and modifier: %08x %016lx",
descriptor.layers[0].drm_format,
descriptor.objects[0].drm_format_modifier);
m_EglExportType = EglExportType::Composed;
}
// Let's probe for EGL import support on separate layers too, but only warn if it's not supported
if (m_EglExportType == EglExportType::Separate) {
if (!canExportSurfaceHandle(VA_EXPORT_SURFACE_SEPARATE_LAYERS, &descriptor)) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Exporting separate layers is not supported by the VAAPI driver");
return false;
}
for (int i = 0; i < descriptor.num_layers; i++) {
if (!m_EglImageFactory.supportsImportingFormat(dpy, descriptor.layers[i].drm_format)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGL implementation lacks support for importing format: %08x", descriptor.layers[0].drm_format);
}
else if (!m_EglImageFactory.supportsImportingModifier(dpy, descriptor.layers[i].drm_format,
descriptor.objects[descriptor.layers[i].object_index[0]].drm_format_modifier)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"EGL implementation lacks support for importing format and modifier: %08x %016lx",
descriptor.layers[i].drm_format,
descriptor.objects[descriptor.layers[i].object_index[0]].drm_format_modifier);
}
}
}
return true;
}
ssize_t
VAAPIRenderer::exportEGLImages(AVFrame *frame, EGLDisplay dpy,
EGLImage images[EGL_MAX_PLANES]) {
ssize_t count;
uint32_t exportFlags = VA_EXPORT_SURFACE_READ_ONLY;
switch (m_EglExportType) {
case EglExportType::Separate:
exportFlags |= VA_EXPORT_SURFACE_SEPARATE_LAYERS;
break;
case EglExportType::Composed:
exportFlags |= VA_EXPORT_SURFACE_COMPOSED_LAYERS;
break;
case EglExportType::Unknown:
SDL_assert(m_EglExportType != EglExportType::Unknown);
return -1;
}
auto hwFrameCtx = (AVHWFramesContext*)frame->hw_frames_ctx->data;
AVVAAPIDeviceContext* vaDeviceContext = (AVVAAPIDeviceContext*)hwFrameCtx->device_ctx->hwctx;
VASurfaceID surface_id = (VASurfaceID)(uintptr_t)frame->data[3];
VAStatus st = vaExportSurfaceHandle(vaDeviceContext->display,
surface_id,
VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
VA_EXPORT_SURFACE_READ_ONLY | VA_EXPORT_SURFACE_SEPARATE_LAYERS,
exportFlags,
&m_PrimeDescriptor);
if (st != VA_STATUS_SUCCESS) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
@ -948,7 +1037,8 @@ VAAPIRenderer::freeEGLImages(EGLDisplay dpy, EGLImage images[EGL_MAX_PLANES]) {
bool VAAPIRenderer::canExportDrmPrime()
{
// Our DRM renderer requires composed layers
return canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS);
VADRMPRIMESurfaceDescriptor descriptor;
return canExportSurfaceHandle(VA_EXPORT_SURFACE_COMPOSED_LAYERS, &descriptor);
}
bool VAAPIRenderer::mapDrmPrimeFrame(AVFrame* frame, AVDRMFrameDescriptor* drmDescriptor)

7
app/streaming/video/ffmpeg-renderers/vaapi.h
View file

@ -69,7 +69,7 @@ private:
void renderOverlay(VADisplay display, VASurfaceID surface, Overlay::OverlayType type);
#if defined(HAVE_EGL) || defined(HAVE_DRM)
bool canExportSurfaceHandle(int layerTypeFlag);
bool canExportSurfaceHandle(int layerTypeFlag, VADRMPRIMESurfaceDescriptor* descriptor);
#endif
int m_DecoderSelectionPass;
@ -96,6 +96,11 @@ private:
int m_DisplayHeight;
#ifdef HAVE_EGL
enum class EglExportType {
Unknown,
Separate,
Composed
} m_EglExportType;
VADRMPRIMESurfaceDescriptor m_PrimeDescriptor;
EglImageFactory m_EglImageFactory;
#endif