Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2024-09-20 06:01:57 +00:00
Code Issues Projects Releases Packages Wiki Activity

Experimental AVFoundation renderer

Browse source
This commit is contained in:
Cameron Gutman 2018-07-17 22:57:15 -07:00
parent a89cadc520
commit 57137412b5
5 changed files with 554 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
app/app.pro
View file

@ -41,7 +41,7 @@ win32 {
}
macx {
LIBS += -lssl -lcrypto -lSDL2 -lavcodec.58 -lavdevice.58 -lavformat.58 -lavutil.56
LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreGraphics -framework CoreMedia -framework AppKit
LIBS += -lobjc -framework VideoToolbox -framework AVFoundation -framework CoreVideo -framework CoreMedia -framework AppKit
}
SOURCES += \
@ -64,7 +64,7 @@ win32 {
SOURCES += streaming/video/ffmpeg-renderers/dxva2.cpp
}
macx {
SOURCES += streaming/video/ffmpeg-renderers/vt.mm
SOURCES += streaming/video/avfoundation.mm
}
HEADERS += \
@ -87,7 +87,7 @@ win32 {
HEADERS += streaming/video/ffmpeg-renderers/dxva2.h
}
macx {
HEADERS += streaming/video/ffmpeg-renderers/vt.h
HEADERS += streaming/video/avfoundation.h
}
RESOURCES += \

19
app/streaming/session.cpp
View file

@ -5,6 +5,10 @@
#include <SDL.h>
#include "utils.h"
#ifdef __APPLE__
#include "video/avfoundation.h"
#endif
#include "video/ffmpeg.h"
#include <QRandomGenerator>
@ -82,6 +86,21 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window, int videoFormat, int width, int height,
int frameRate, IVideoDecoder*& chosenDecoder)
{
#ifdef __APPLE__
chosenDecoder = AVFoundationDecoderFactory::createDecoder();
if (chosenDecoder->initialize(vds, window, videoFormat, width, height, frameRate)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"AVFoundation-based video decoder chosen");
return true;
}
else {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to load AVFoundation decoder");
delete chosenDecoder;
chosenDecoder = nullptr;
}
#endif
chosenDecoder = new FFmpegVideoDecoder();
if (chosenDecoder->initialize(vds, window, videoFormat, width, height, frameRate)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,

11
app/streaming/video/avfoundation.h Normal file
View file

@ -0,0 +1,11 @@
#pragma once
#include "decoder.h"
// A factory is required to avoid pulling in
// incompatible Objective-C headers.
class AVFoundationDecoderFactory {
public:
static
IVideoDecoder* createDecoder();
};

521
app/streaming/video/avfoundation.mm Normal file
View file

@ -0,0 +1,521 @@
#include "avfoundation.h"
#include <SDL_syswm.h>
#import <Cocoa/Cocoa.h>
#import <VideoToolbox/VideoToolbox.h>
#import <AVFoundation/AVFoundation.h>
#import <CoreVideo/CoreVideo.h>
#define FRAME_START_PREFIX_SIZE 4
#define NALU_START_PREFIX_SIZE 3
#define NAL_LENGTH_PREFIX_SIZE 4
#define FUTURE_FRAMES_IN_VSYNC 3
class AVFoundationDecoder : public IVideoDecoder
{
public:
AVFoundationDecoder()
: m_DisplayLayer(nullptr),
m_FormatDesc(nullptr),
m_SpsData(nullptr),
m_PpsData(nullptr),
m_VpsData(nullptr),
m_View(nullptr),
m_NeedsIdr(false),
m_DisplayLink(nullptr),
m_VsyncTime(0),
m_LastVsyncTime(0),
m_VsyncStatsLock(0)
{
SDL_zero(m_FrameCountOnVsync);
}
virtual ~AVFoundationDecoder()
{
if (m_FormatDesc != nullptr) {
CFRelease(m_FormatDesc);
}
if (m_DisplayLink != nullptr) {
CVDisplayLinkStop(m_DisplayLink);
CVDisplayLinkRelease(m_DisplayLink);
}
delete m_SpsData;
delete m_PpsData;
delete m_VpsData;
}
static
CVReturn
displayLinkOutputCallback(
CVDisplayLinkRef,
const CVTimeStamp*,
const CVTimeStamp* vsyncTime,
CVOptionFlags,
CVOptionFlags*,
void *displayLinkContext)
{
AVFoundationDecoder* me = reinterpret_cast<AVFoundationDecoder*>(displayLinkContext);
SDL_AtomicLock(&me->m_VsyncStatsLock);
me->m_LastVsyncTime = me->m_VsyncTime;
me->m_VsyncTime = vsyncTime->hostTime;
int framesThisVsync = *(volatile int*)&me->m_FrameCountOnVsync[0];
memmove(&me->m_FrameCountOnVsync[0],
&me->m_FrameCountOnVsync[1],
sizeof(me->m_FrameCountOnVsync) - sizeof(me->m_FrameCountOnVsync[0]));
me->m_FrameCountOnVsync[FUTURE_FRAMES_IN_VSYNC - 1] = 0;
SDL_AtomicUnlock(&me->m_VsyncStatsLock);
if (framesThisVsync != 1)
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"V-Sync interval frames: %d",
framesThisVsync);
return kCVReturnSuccess;
}
virtual bool initialize(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int, int, int) override
{
m_VideoFormat = videoFormat;
// Fail if the user wants software decode only
if (vds == StreamingPreferences::VDS_FORCE_SOFTWARE) {
return false;
}
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
// Prior to 10.13, we'll just assume everything has
// H.264 support and fail open to allow VT decode.
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_H264)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated H.264 decode via VT");
return false;
}
}
else
#endif
{
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Assuming H.264 HW decode on < macOS 10.13");
}
}
else if (videoFormat & VIDEO_FORMAT_MASK_H265) {
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
if (__builtin_available(macOS 10.13, *)) {
if (!VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HW accelerated HEVC decode via VT");
return false;
}
}
else
#endif
{
// Fail closed for HEVC if we're not on 10.13+
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"No HEVC support on < macOS 10.13");
return false;
}
}
SDL_SysWMinfo info;
SDL_VERSION(&info.version);
if (!SDL_GetWindowWMInfo(window, &info)) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"SDL_GetWindowWMInfo() failed: %s",
SDL_GetError());
return false;
}
SDL_assert(info.subsystem == SDL_SYSWM_COCOA);
// SDL adds its own content view to listen for events.
// We need to add a subview for our display layer.
NSView* contentView = info.info.cocoa.window.contentView;
m_View = [[NSView alloc] initWithFrame:contentView.bounds];
m_View.wantsLayer = YES;
[contentView addSubview: m_View];
setupDisplayLayer();
CVDisplayLinkCreateWithActiveCGDisplays(&m_DisplayLink);
CVDisplayLinkSetOutputCallback(m_DisplayLink, displayLinkOutputCallback, (void*)this);
CVDisplayLinkStart(m_DisplayLink);
return true;
}
virtual bool isHardwareAccelerated() override
{
// We only use AVFoundation for HW acceleration
return true;
}
virtual int submitDecodeUnit(PDECODE_UNIT du) override
{
OSStatus status;
unsigned char* data = (unsigned char*)malloc(du->fullLength);
if (!data) {
return DR_NEED_IDR;
}
PLENTRY entry = du->bufferList;
int pictureDataLength = 0;
while (entry != nullptr) {
if (entry->bufferType == BUFFER_TYPE_VPS) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Got VPS");
delete m_VpsData;
m_VpsData = new QByteArray(&entry->data[FRAME_START_PREFIX_SIZE], entry->length - FRAME_START_PREFIX_SIZE);
// We got a new VPS so wait for a new SPS to match it
delete m_SpsData;
delete m_PpsData;
m_SpsData = m_PpsData = nullptr;
}
else if (entry->bufferType == BUFFER_TYPE_SPS) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Got SPS");
delete m_SpsData;
m_SpsData = new QByteArray(&entry->data[FRAME_START_PREFIX_SIZE], entry->length - FRAME_START_PREFIX_SIZE);
// We got a new SPS so wait for a new PPS to match it
delete m_PpsData;
m_PpsData = nullptr;
}
else if (entry->bufferType == BUFFER_TYPE_PPS) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Got PPS");
delete m_PpsData;
m_PpsData = new QByteArray(&entry->data[FRAME_START_PREFIX_SIZE], entry->length - FRAME_START_PREFIX_SIZE);
}
else {
// Picture data
SDL_assert(entry->bufferType == BUFFER_TYPE_PICDATA);
memcpy(&data[pictureDataLength],
entry->data,
entry->length);
pictureDataLength += entry->length;
}
entry = entry->next;
}
SDL_assert(pictureDataLength <= du->fullLength);
if (du->frameType == FRAME_TYPE_IDR) {
m_NeedsIdr = false;
if (m_FormatDesc != nullptr) {
CFRelease(m_FormatDesc);
}
if (m_VideoFormat & VIDEO_FORMAT_MASK_H264) {
const uint8_t* const parameterSetPointers[] = { (uint8_t*)m_SpsData->data(),
(uint8_t*)m_PpsData->data() };
const size_t parameterSetSizes[] = { (size_t)m_SpsData->length(),
(size_t)m_PpsData->length() };
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Constructing new H264 format description");
status = CMVideoFormatDescriptionCreateFromH264ParameterSets(kCFAllocatorDefault,
2, /* count of parameter sets */
parameterSetPointers,
parameterSetSizes,
NAL_LENGTH_PREFIX_SIZE,
&m_FormatDesc);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to create H264 format description: %d",
(int)status);
m_FormatDesc = NULL;
}
}
else {
#if __MAC_OS_X_VERSION_MAX_ALLOWED >= 101300
const uint8_t* const parameterSetPointers[] = { (uint8_t*)m_VpsData->data(),
(uint8_t*)m_SpsData->data(),
(uint8_t*)m_PpsData->data() };
const size_t parameterSetSizes[] = { (size_t)m_VpsData->length(),
(size_t)m_SpsData->length(),
(size_t)m_PpsData->length() };
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Constructing new HEVC format description");
if (__builtin_available(macOS 10.13, *)) {
status = CMVideoFormatDescriptionCreateFromHEVCParameterSets(kCFAllocatorDefault,
3, /* count of parameter sets */
parameterSetPointers,
parameterSetSizes,
NAL_LENGTH_PREFIX_SIZE,
nil,
&m_FormatDesc);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to create HEVC format description: %d",
(int)status);
m_FormatDesc = NULL;
}
}
else
#endif
{
// This means Moonlight-common-c decided to give us an HEVC stream
// even though we said we couldn't support it. All we can do is abort().
abort();
}
}
}
if (m_FormatDesc == nullptr || m_NeedsIdr) {
// Can't decode if we haven't gotten our parameter sets yet
free(data);
return DR_NEED_IDR;
}
// Now we're decoding actual frame data here
CMBlockBufferRef blockBuffer;
status = CMBlockBufferCreateEmpty(NULL, 0, 0, &blockBuffer);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferCreateEmpty failed: %d",
(int)status);
free(data);
return DR_NEED_IDR;
}
int lastOffset = -1;
for (int i = 0; i < pictureDataLength - FRAME_START_PREFIX_SIZE; i++) {
// Search for a NALU
if (data[i] == 0 && data[i+1] == 0 && data[i+2] == 1) {
// It's the start of a new NALU
if (lastOffset != -1) {
// We've seen a start before this so enqueue that NALU
updateBufferForRange(blockBuffer, data, lastOffset, i - lastOffset);
}
lastOffset = i;
}
}
if (lastOffset != -1) {
// Enqueue the remaining data
updateBufferForRange(blockBuffer, data, lastOffset, pictureDataLength - lastOffset);
}
// From now on, CMBlockBuffer owns the data pointer and will free it when it's released
CMSampleBufferRef sampleBuffer;
CMSampleTimingInfo time = {
.decodeTimeStamp = kCMTimeInvalid,
.duration = kCMTimeInvalid
};
SDL_AtomicLock(&m_VsyncStatsLock);
// Find a slot to put this frame
for (int i = 0; i < FUTURE_FRAMES_IN_VSYNC; i++) {
if (m_FrameCountOnVsync[i] == 0) {
time.presentationTimeStamp =
CMTimeMake(m_VsyncTime + (i + 1) * (m_VsyncTime - m_LastVsyncTime),
1000 * 1000 * 1000);
m_FrameCountOnVsync[i]++;
}
}
SDL_AtomicUnlock(&m_VsyncStatsLock);
status = CMSampleBufferCreate(kCFAllocatorDefault,
blockBuffer,
true, NULL,
NULL, m_FormatDesc, 1, 1,
&time, 0, NULL,
&sampleBuffer);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMSampleBufferCreate failed: %d",
(int)status);
CFRelease(blockBuffer);
return DR_NEED_IDR;
}
// Submit for callback on the main thread
queueFrame(sampleBuffer, (void*)(long)du->frameType);
return DR_OK;
}
// This is called on the main thread, so it's safe to interact with
// the AVSampleBufferDisplayLayer here.
void submitBuffer(CMSampleBufferRef sampleBuffer, int frameType)
{
if (m_DisplayLayer.status == AVQueuedSampleBufferRenderingStatusFailed) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Resetting failed AVSampleBufferDisplay layer");
setupDisplayLayer();
CFRelease(sampleBuffer);
return;
}
CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, YES);
CFMutableDictionaryRef dict = (CFMutableDictionaryRef)CFArrayGetValueAtIndex(attachments, 0);
CFDictionarySetValue(dict, kCMSampleAttachmentKey_IsDependedOnByOthers, kCFBooleanTrue);
if (frameType == FRAME_TYPE_PFRAME) {
// P-frame
CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanTrue);
CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanTrue);
} else {
// I-frame
CFDictionarySetValue(dict, kCMSampleAttachmentKey_NotSync, kCFBooleanFalse);
CFDictionarySetValue(dict, kCMSampleAttachmentKey_DependsOnOthers, kCFBooleanFalse);
}
[m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer);
}
virtual void renderFrame(SDL_UserEvent* event) override
{
submitBuffer((CMSampleBufferRef)event->data1, (int)(long)event->data2);
}
virtual void dropFrame(SDL_UserEvent* event) override
{
// Submit anyway and let the drop happen in the render pipeline
submitBuffer((CMSampleBufferRef)event->data1, (int)(long)event->data2);
}
private:
void setupDisplayLayer()
{
CALayer* oldLayer = m_DisplayLayer;
m_DisplayLayer = [[AVSampleBufferDisplayLayer alloc] init];
m_DisplayLayer.bounds = m_View.bounds;
m_DisplayLayer.position = CGPointMake(CGRectGetMidX(m_View.bounds), CGRectGetMidY(m_View.bounds));
m_DisplayLayer.videoGravity = AVLayerVideoGravityResizeAspect;
CALayer* viewLayer = m_View.layer;
if (oldLayer != nil) {
[viewLayer replaceSublayer:oldLayer with:m_DisplayLayer];
}
else {
[viewLayer addSublayer:m_DisplayLayer];
}
// Wait for another IDR frame
m_NeedsIdr = true;
delete m_SpsData;
delete m_PpsData;
delete m_VpsData;
m_SpsData = m_PpsData = m_VpsData = nullptr;
}
void updateBufferForRange(CMBlockBufferRef existingBuffer, unsigned char* data, int offset, int nalLength)
{
OSStatus status;
size_t oldOffset = CMBlockBufferGetDataLength(existingBuffer);
// If we're at index 1 (first NALU in frame), enqueue this buffer to the memory block
// so it can handle freeing it when the block buffer is destroyed
if (offset == 1) {
int dataLength = nalLength - NALU_START_PREFIX_SIZE;
// Pass the real buffer pointer directly (no offset)
// This will give it to the block buffer to free when it's released.
// All further calls to CMBlockBufferAppendMemoryBlock will do so
// at an offset and will not be asking the buffer to be freed.
status = CMBlockBufferAppendMemoryBlock(existingBuffer, data,
nalLength + 1, // Add 1 for the offset we decremented
kCFAllocatorDefault,
NULL, 0, nalLength + 1, 0);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferReplaceDataBytes failed: %d",
(int)status);
return;
}
// Write the length prefix to existing buffer
const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16),
(uint8_t)(dataLength >> 8), (uint8_t)dataLength};
status = CMBlockBufferReplaceDataBytes(lengthBytes, existingBuffer,
oldOffset, NAL_LENGTH_PREFIX_SIZE);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferReplaceDataBytes failed: %d",
(int)status);
return;
}
} else {
// Append a 4 byte buffer to this block for the length prefix
status = CMBlockBufferAppendMemoryBlock(existingBuffer, NULL,
NAL_LENGTH_PREFIX_SIZE,
kCFAllocatorDefault, NULL, 0,
NAL_LENGTH_PREFIX_SIZE, 0);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferAppendMemoryBlock failed: %d",
(int)status);
return;
}
// Write the length prefix to the new buffer
int dataLength = nalLength - NALU_START_PREFIX_SIZE;
const uint8_t lengthBytes[] = {(uint8_t)(dataLength >> 24), (uint8_t)(dataLength >> 16),
(uint8_t)(dataLength >> 8), (uint8_t)dataLength};
status = CMBlockBufferReplaceDataBytes(lengthBytes, existingBuffer,
oldOffset, NAL_LENGTH_PREFIX_SIZE);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferReplaceDataBytes failed: %d",
(int)status);
return;
}
// Attach the buffer by reference to the block buffer
status = CMBlockBufferAppendMemoryBlock(existingBuffer, &data[offset+NALU_START_PREFIX_SIZE],
dataLength,
kCFAllocatorNull, // Don't deallocate data on free
NULL, 0, dataLength, 0);
if (status != noErr) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"CMBlockBufferReplaceDataBytes failed: %d",
(int)status);
return;
}
}
}
AVSampleBufferDisplayLayer* m_DisplayLayer;
CMVideoFormatDescriptionRef m_FormatDesc;
int m_VideoFormat;
QByteArray* m_SpsData;
QByteArray* m_PpsData;
QByteArray* m_VpsData;
NSView* m_View;
bool m_NeedsIdr;
CVDisplayLinkRef m_DisplayLink;
uint64_t m_VsyncTime, m_LastVsyncTime;
int m_FrameCountOnVsync[FUTURE_FRAMES_IN_VSYNC];
SDL_SpinLock m_VsyncStatsLock;
};
IVideoDecoder* AVFoundationDecoderFactory::createDecoder() {
return new AVFoundationDecoder();
}

9
app/streaming/video/ffmpeg.cpp
View file

@ -5,10 +5,6 @@
#include "ffmpeg-renderers/dxva2.h"
#endif
#ifdef __APPLE__
#include "ffmpeg-renderers/vt.h"
#endif
bool FFmpegVideoDecoder::chooseDecoder(
StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
@ -64,11 +60,6 @@ bool FFmpegVideoDecoder::chooseDecoder(
case AV_HWDEVICE_TYPE_DXVA2:
newRenderer = new DXVA2Renderer();
break;
#endif
#ifdef __APPLE__
case AV_HWDEVICE_TYPE_VIDEOTOOLBOX:
newRenderer = VTRendererFactory::createRenderer();
break;
#endif
default:
continue;