moonlight-qt/app/streaming/video/ffmpeg-renderers/vt.mm

315 lines
11 KiB
Text

// Nasty hack to avoid conflict between AVFoundation and
// libavutil both defining AVMediaType
#define AVMediaType AVMediaType_FFmpeg
#include "vt.h"
#include "pacer/pacer.h"
#undef AVMediaType
#include <SDL_syswm.h>
#include <Limelight.h>
#include <streaming/session.h>
#include <mach/mach_time.h>
#import <Cocoa/Cocoa.h>
#import <VideoToolbox/VideoToolbox.h>
#import <AVFoundation/AVFoundation.h>
#import <dispatch/dispatch.h>
class VTRenderer : public IFFmpegRenderer
{
public:
VTRenderer()
: m_HwContext(nullptr),
m_DisplayLayer(nullptr),
m_FormatDesc(nullptr),
m_StreamView(nullptr)
{
SDL_zero(m_OverlayTextFields);
}
virtual ~VTRenderer() override
{
if (m_HwContext != nullptr) {
av_buffer_unref(&m_HwContext);
}
if (m_FormatDesc != nullptr) {
CFRelease(m_FormatDesc);
}
for (int i = 0; i < Overlay::OverlayMax; i++) {
if (m_OverlayTextFields[i] != nullptr) {
[m_OverlayTextFields[i] removeFromSuperview];
}
}
if (m_StreamView != nullptr) {
[m_StreamView removeFromSuperview];
}
}
// Caller frees frame after we return
virtual void renderFrame(AVFrame* frame) override
{
OSStatus status;
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Resetting failed AVSampleBufferDisplay layer");
// Trigger the main thread to recreate the decoder
SDL_Event event;
event.type = SDL_RENDER_TARGETS_RESET;
SDL_PushEvent(&event);
return;
}
// If the format has changed or doesn't exist yet, construct it with the
// pixel buffer data
if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) {
if (m_FormatDesc != nullptr) {
CFRelease(m_FormatDesc);
}
status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault,
pixBuf, &m_FormatDesc);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMVideoFormatDescriptionCreateForImageBuffer() failed: %d",
status);
return;
}
}
// Queue this sample for the next v-sync
CMSampleTimingInfo timingInfo = {
.duration = kCMTimeInvalid,
.decodeTimeStamp = kCMTimeInvalid,
.presentationTimeStamp = CMTimeMake(mach_absolute_time(), 1000 * 1000 * 1000)
};
CMSampleBufferRef sampleBuffer;
status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault,
pixBuf,
m_FormatDesc,
&timingInfo,
&sampleBuffer);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMSampleBufferCreateReadyWithImageBuffer() failed: %d",
status);
return;
}
[m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
virtual bool initialize(SDL_Window* window,
int videoFormat,
int,
int,
int,
bool) override
{
int err;
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
// Prior to 10.13, we'll just assume everything has
// H.264 support and fail open to allow VT decode.
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated H.264 decode via VT");
return false;
}
}
else
#endif
{
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming H.264 HW decode on < macOS 10.13");
}
}
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated HEVC decode via VT");
return false;
}
}
else
#endif
{
// Fail closed for HEVC if we're not on 10.13+
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HEVC support on < macOS 10.13");
return false;
}
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(window, &info)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"SDL_GetWindowWMInfo() failed: %s",
SDL_GetError());
return false;
}
SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
// SDL adds its own content view to listen for events.
// We need to add a subview for our display layer.
NSView* contentView = info.info.cocoa.window.contentView;
m_StreamView = [[NSView alloc] initWithFrame:contentView.bounds];
m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init];
m_DisplayLayer.bounds = m_StreamView.bounds;
m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_StreamView.bounds), CGRectGetMidY(m_StreamView.bounds));
m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
// Create a layer-hosted view by setting the layer before wantsLayer
// This avoids us having to add our AVSampleBufferDisplayLayer as a
// sublayer of a layer-backed view which leaves a useless layer in
// the middle.
m_StreamView.layer = m_DisplayLayer;
m_StreamView.wantsLayer = YES;
[contentView addSubview: m_StreamView];
err = av_hwdevice_ctx_create(&m_HwContext,
AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
nullptr,
nullptr,
0);
if (err < 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"av_hwdevice_ctx_create() failed for VT decoder: %d",
err);
return false;
}
return true;
}
void updateOverlayOnMainThread(Overlay::OverlayType type)
{
// Lazy initialization for the overlay
if (m_OverlayTextFields[type] == nullptr) {
m_OverlayTextFields[type] = [[NSTextField alloc] initWithFrame:m_StreamView.bounds];
[m_OverlayTextFields[type] setBezeled:NO];
[m_OverlayTextFields[type] setDrawsBackground:NO];
[m_OverlayTextFields[type] setEditable:NO];
[m_OverlayTextFields[type] setSelectable:NO];
switch (type) {
case Overlay::OverlayDebug:
[m_OverlayTextFields[type] setAlignment:NSLeftTextAlignment];
break;
case Overlay::OverlayStatusUpdate:
[m_OverlayTextFields[type] setAlignment:NSRightTextAlignment];
break;
default:
break;
}
SDL_Color color = Session::get()->getOverlayManager().getOverlayColor(type);
[m_OverlayTextFields[type] setTextColor:[NSColor colorWithSRGBRed:color.r / 255.0 green:color.g / 255.0 blue:color.b / 255.0 alpha:color.a / 255.0]];
[m_OverlayTextFields[type] setFont:[NSFont messageFontOfSize:Session::get()->getOverlayManager().getOverlayFontSize(type)]];
[m_StreamView addSubview: m_OverlayTextFields[type]];
}
// Update text contents
[m_OverlayTextFields[type] setStringValue: [NSString stringWithUTF8String:Session::get()->getOverlayManager().getOverlayText(type)]];
// Unhide if it's enabled
[m_OverlayTextFields[type] setHidden: !Session::get()->getOverlayManager().isOverlayEnabled(type)];
}
static void updateDebugOverlayOnMainThread(void* context)
{
VTRenderer* me = (VTRenderer*)context;
me->updateOverlayOnMainThread(Overlay::OverlayDebug);
}
static void updateStatusOverlayOnMainThread(void* context)
{
VTRenderer* me = (VTRenderer*)context;
me->updateOverlayOnMainThread(Overlay::OverlayStatusUpdate);
}
virtual void notifyOverlayUpdated(Overlay::OverlayType type) override
{
// We must do the actual UI updates on the main thread, so queue an
// async callback on the main thread via GCD to do the UI update.
switch (type) {
case Overlay::OverlayDebug:
dispatch_async_f(dispatch_get_main_queue(), this, updateDebugOverlayOnMainThread);
break;
case Overlay::OverlayStatusUpdate:
dispatch_async_f(dispatch_get_main_queue(), this, updateStatusOverlayOnMainThread);
break;
default:
break;
}
}
virtual bool prepareDecoderContext(AVCodecContext* context) override
{
context->hw_device_ctx = av_buffer_ref(m_HwContext);
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Using VideoToolbox accelerated renderer");
return true;
}
virtual bool needsTestFrame() override
{
// We used to trust VT to tell us whether decode will work, but
// there are cases where it can lie because the hardware technically
// can decode the format but VT is unserviceable for some other reason.
// Decoding the test frame will tell us for sure whether it will work.
return true;
}
virtual int getDecoderCapabilities() override
{
return 0;
}
virtual IFFmpegRenderer::FramePacingConstraint getFramePacingConstraint() override
{
// This renderer is inherently tied to V-sync due how we're
// rendering with AVSampleBufferDisplay layer. Running without
// the V-Sync source leads to massive stuttering.
return PACING_FORCE_ON;
}
virtual bool isRenderThreadSupported() override
{
// renderFrame() may be called outside of the main thread
return true;
}
private:
AVBufferRef* m_HwContext;
AVSampleBufferDisplayLayer* m_DisplayLayer;
CMVideoFormatDescriptionRef m_FormatDesc;
NSView* m_StreamView;
NSTextField* m_OverlayTextFields[Overlay::OverlayMax];
};
IFFmpegRenderer* VTRendererFactory::createRenderer() {
return new VTRenderer();
}