// Nasty hack to avoid conflict between AVFoundation and // libavutil both defining AVMediaType #define AVMediaType AVMediaType_FFmpeg #include "vt.h" #include "pacer/pacer.h" #undef AVMediaType #include #include #include #include #import #import #import #import class VTRenderer : public IFFmpegRenderer { public: VTRenderer() : m_HwContext(nullptr), m_DisplayLayer(nullptr), m_FormatDesc(nullptr), m_StreamView(nullptr) { SDL_zero(m_OverlayTextFields); } virtual ~VTRenderer() override { if (m_HwContext != nullptr) { av_buffer_unref(&m_HwContext); } if (m_FormatDesc != nullptr) { CFRelease(m_FormatDesc); } for (int i = 0; i < Overlay::OverlayMax; i++) { if (m_OverlayTextFields[i] != nullptr) { [m_OverlayTextFields[i] removeFromSuperview]; } } if (m_StreamView != nullptr) { [m_StreamView removeFromSuperview]; } } // Caller frees frame after we return virtual void renderFrame(AVFrame* frame) override { OSStatus status; CVPixelBufferRef pixBuf = reinterpret_cast(frame->data[3]); if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Resetting failed AVSampleBufferDisplay layer"); // Trigger the main thread to recreate the decoder SDL_Event event; event.type = SDL_RENDER_TARGETS_RESET; SDL_PushEvent(&event); return; } // If the format has changed or doesn't exist yet, construct it with the // pixel buffer data if (!m_FormatDesc || !CMVideoFormatDescriptionMatchesImageBuffer(m_FormatDesc, pixBuf)) { if (m_FormatDesc != nullptr) { CFRelease(m_FormatDesc); } status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixBuf, &m_FormatDesc); if (status != noErr) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "CMVideoFormatDescriptionCreateForImageBuffer() failed: %d", status); return; } } // Queue this sample for the next v-sync CMSampleTimingInfo timingInfo = { .duration = kCMTimeInvalid, .decodeTimeStamp = kCMTimeInvalid, .presentationTimeStamp = CMTimeMake(mach_absolute_time(), 1000 * 1000 * 1000) }; CMSampleBufferRef sampleBuffer; status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixBuf, m_FormatDesc, &timingInfo, &sampleBuffer); if (status != noErr) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "CMSampleBufferCreateReadyWithImageBuffer() failed: %d", status); return; } [m_DisplayLayer enqueueSampleBuffer:sampleBuffer]; CFRelease(sampleBuffer); } virtual bool initialize(SDL_Window* window, int videoFormat, int, int, int, bool) override { int err; if (videoFormat & VIDEO_FORMAT_MASK_H264) { // Prior to 10.13, we'll just assume everything has // H.264 support and fail open to allow VT decode. #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 if (__builtin_available(macOS 10.13, *)) { if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "No HW accelerated H.264 decode via VT"); return false; } } else #endif { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Assuming H.264 HW decode on < macOS 10.13"); } } else if (videoFormat & VIDEO_FORMAT_MASK_H265) { #if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300 if (__builtin_available(macOS 10.13, *)) { if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "No HW accelerated HEVC decode via VT"); return false; } } else #endif { // Fail closed for HEVC if we're not on 10.13+ SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "No HEVC support on < macOS 10.13"); return false; } } SDL_SysWMinfo info; SDL_VERSION(&info.version); if (!SDL_GetWindowWMInfo(window, &info)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "SDL_GetWindowWMInfo() failed: %s", SDL_GetError()); return false; } SDL_assert(info.subsystem == SDL_SYSWM_COCOA); // SDL adds its own content view to listen for events. // We need to add a subview for our display layer. NSView* contentView = info.info.cocoa.window.contentView; m_StreamView = [[NSView alloc] initWithFrame:contentView.bounds]; m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init]; m_DisplayLayer.bounds = m_StreamView.bounds; m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_StreamView.bounds), CGRectGetMidY(m_StreamView.bounds)); m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect; // Create a layer-hosted view by setting the layer before wantsLayer // This avoids us having to add our AVSampleBufferDisplayLayer as a // sublayer of a layer-backed view which leaves a useless layer in // the middle. m_StreamView.layer = m_DisplayLayer; m_StreamView.wantsLayer = YES; [contentView addSubview: m_StreamView]; err = av_hwdevice_ctx_create(&m_HwContext, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, nullptr, nullptr, 0); if (err < 0) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "av_hwdevice_ctx_create() failed for VT decoder: %d", err); return false; } return true; } void updateOverlayOnMainThread(Overlay::OverlayType type) { // Lazy initialization for the overlay if (m_OverlayTextFields[type] == nullptr) { m_OverlayTextFields[type] = [[NSTextField alloc] initWithFrame:m_StreamView.bounds]; [m_OverlayTextFields[type] setBezeled:NO]; [m_OverlayTextFields[type] setDrawsBackground:NO]; [m_OverlayTextFields[type] setEditable:NO]; [m_OverlayTextFields[type] setSelectable:NO]; switch (type) { case Overlay::OverlayDebug: [m_OverlayTextFields[type] setAlignment:NSLeftTextAlignment]; break; case Overlay::OverlayStatusUpdate: [m_OverlayTextFields[type] setAlignment:NSRightTextAlignment]; break; default: break; } SDL_Color color = Session::get()->getOverlayManager().getOverlayColor(type); [m_OverlayTextFields[type] setTextColor:[NSColor colorWithSRGBRed:color.r / 255.0 green:color.g / 255.0 blue:color.b / 255.0 alpha:color.a / 255.0]]; [m_OverlayTextFields[type] setFont:[NSFont messageFontOfSize:Session::get()->getOverlayManager().getOverlayFontSize(type)]]; [m_StreamView addSubview: m_OverlayTextFields[type]]; } // Update text contents [m_OverlayTextFields[type] setStringValue: [NSString stringWithUTF8String:Session::get()->getOverlayManager().getOverlayText(type)]]; // Unhide if it's enabled [m_OverlayTextFields[type] setHidden: !Session::get()->getOverlayManager().isOverlayEnabled(type)]; } static void updateDebugOverlayOnMainThread(void* context) { VTRenderer* me = (VTRenderer*)context; me->updateOverlayOnMainThread(Overlay::OverlayDebug); } static void updateStatusOverlayOnMainThread(void* context) { VTRenderer* me = (VTRenderer*)context; me->updateOverlayOnMainThread(Overlay::OverlayStatusUpdate); } virtual void notifyOverlayUpdated(Overlay::OverlayType type) override { // We must do the actual UI updates on the main thread, so queue an // async callback on the main thread via GCD to do the UI update. switch (type) { case Overlay::OverlayDebug: dispatch_async_f(dispatch_get_main_queue(), this, updateDebugOverlayOnMainThread); break; case Overlay::OverlayStatusUpdate: dispatch_async_f(dispatch_get_main_queue(), this, updateStatusOverlayOnMainThread); break; default: break; } } virtual bool prepareDecoderContext(AVCodecContext* context) override { context->hw_device_ctx = av_buffer_ref(m_HwContext); SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Using VideoToolbox accelerated renderer"); return true; } virtual bool needsTestFrame() override { // We used to trust VT to tell us whether decode will work, but // there are cases where it can lie because the hardware technically // can decode the format but VT is unserviceable for some other reason. // Decoding the test frame will tell us for sure whether it will work. return true; } virtual int getDecoderCapabilities() override { return 0; } virtual IFFmpegRenderer::FramePacingConstraint getFramePacingConstraint() override { // This renderer is inherently tied to V-sync due how we're // rendering with AVSampleBufferDisplay layer. Running without // the V-Sync source leads to massive stuttering. return PACING_FORCE_ON; } virtual bool isRenderThreadSupported() override { // renderFrame() may be called outside of the main thread return true; } private: AVBufferRef* m_HwContext; AVSampleBufferDisplayLayer* m_DisplayLayer; CMVideoFormatDescriptionRef m_FormatDesc; NSView* m_StreamView; NSTextField* m_OverlayTextFields[Overlay::OverlayMax]; }; IFFmpegRenderer* VTRendererFactory::createRenderer() { return new VTRenderer(); }