Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2024-11-10 05:34:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

Separate FFmpeg decoder from the Session class (#4)

Browse source
This commit is contained in:
Cameron Gutman 2018-07-17 20:00:16 -07:00 committed by GitHub
parent ec68f2ae89
commit a89cadc520
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 276 additions and 166 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
app/app.pro
View file

@ -55,16 +55,16 @@ SOURCES += \
streaming/input.cpp \
streaming/session.cpp \
streaming/audio.cpp \
streaming/video.cpp \
streaming/video/ffmpeg.cpp \
gui/computermodel.cpp \
gui/appmodel.cpp \
streaming/renderers/sdl.cpp
streaming/video/ffmpeg-renderers/sdl.cpp
win32 {
SOURCES += streaming/renderers/dxva2.cpp
SOURCES += streaming/video/ffmpeg-renderers/dxva2.cpp
}
macx {
SOURCES += streaming/renderers/vt.mm
SOURCES += streaming/video/ffmpeg-renderers/vt.mm
}
HEADERS += \
@ -79,13 +79,15 @@ HEADERS += \
streaming/session.hpp \
gui/computermodel.h \
gui/appmodel.h \
streaming/renderers/renderer.h
streaming/video/decoder.h \
streaming/video/ffmpeg.h \
streaming/video/ffmpeg-renderers/renderer.h
win32 {
HEADERS += streaming/renderers/dxva2.h
HEADERS += streaming/video/ffmpeg-renderers/dxva2.h
}
macx {
HEADERS += streaming/renderers/vt.h
HEADERS += streaming/video/ffmpeg-renderers/vt.h
}
RESOURCES += \

110
app/streaming/session.cpp
View file

@ -5,6 +5,8 @@
#include <SDL.h>
#include "utils.h"
#include "video/ffmpeg.h"
#include <QRandomGenerator>
#include <QtEndian>
#include <QCoreApplication>
@ -76,16 +78,99 @@ void Session::clLogMessage(const char* format, ...)
va_end(ap);
}
bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window, int videoFormat, int width, int height,
int frameRate, IVideoDecoder*& chosenDecoder)
{
chosenDecoder = new FFmpegVideoDecoder();
if (chosenDecoder->initialize(vds, window, videoFormat, width, height, frameRate)) {
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"FFmpeg-based video decoder chosen");
return true;
}
else {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to load FFmpeg decoder");
delete chosenDecoder;
chosenDecoder = nullptr;
}
// If we reach this, we didn't initialize any decoders successfully
return false;
}
int Session::drSetup(int videoFormat, int width, int height, int frameRate, void *, int)
{
if (!chooseDecoder(s_ActiveSession->m_Preferences.videoDecoderSelection,
s_ActiveSession->m_Window,
videoFormat, width, height, frameRate,
s_ActiveSession->m_VideoDecoder)) {
return -1;
}
return 0;
}
int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
{
// Use a lock since we'll be yanking this decoder out
// from underneath the session when we initiate destruction.
// We need to destroy the decoder on the main thread to satisfy
// some API constraints (like DXVA2).
SDL_AtomicLock(&s_ActiveSession->m_DecoderLock);
IVideoDecoder* decoder = s_ActiveSession->m_VideoDecoder;
if (decoder != nullptr) {
int ret = decoder->submitDecodeUnit(du);
SDL_AtomicUnlock(&s_ActiveSession->m_DecoderLock);
return ret;
}
else {
SDL_AtomicUnlock(&s_ActiveSession->m_DecoderLock);
return DR_OK;
}
}
bool Session::isHardwareDecodeAvailable(StreamingPreferences::VideoDecoderSelection vds,
int videoFormat, int width, int height, int frameRate)
{
IVideoDecoder* decoder;
SDL_Window* window = SDL_CreateWindow("", 0, 0, width, height, SDL_WINDOW_HIDDEN);
if (!window) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to create window for hardware decode test: %s",
SDL_GetError());
return false;
}
if (!chooseDecoder(vds, window, videoFormat, width, height, frameRate, decoder)) {
SDL_DestroyWindow(window);
return false;
}
SDL_DestroyWindow(window);
bool ret = decoder->isHardwareAccelerated();
delete decoder;
return ret;
}
Session::Session(NvComputer* computer, NvApp& app)
: m_Computer(computer),
m_App(app),
m_Window(nullptr)
m_Window(nullptr),
m_VideoDecoder(nullptr),
m_DecoderLock(0)
{
LiInitializeVideoCallbacks(&m_VideoCallbacks);
m_VideoCallbacks.setup = drSetup;
m_VideoCallbacks.cleanup = drCleanup;
m_VideoCallbacks.submitDecodeUnit = drSubmitDecodeUnit;
m_VideoCallbacks.capabilities = getDecoderCapabilities();
// Submit for decode without using a separate thread
m_VideoCallbacks.capabilities |= CAPABILITY_DIRECT_SUBMIT;
// Slice up to 4 times for parallel decode, once slice per core
m_VideoCallbacks.capabilities |= CAPABILITY_SLICES_PER_FRAME(qMin(MAX_SLICES, SDL_GetCPUCount()));
LiInitializeStreamConfiguration(&m_StreamConfig);
m_StreamConfig.width = m_Preferences.width;
@ -119,7 +204,8 @@ Session::Session(NvComputer* computer, NvApp& app)
isHardwareDecodeAvailable(m_Preferences.videoDecoderSelection,
VIDEO_FORMAT_H265,
m_StreamConfig.width,
m_StreamConfig.height);
m_StreamConfig.height,
m_StreamConfig.fps);
m_StreamConfig.enableHdr = false;
break;
case StreamingPreferences::VCC_FORCE_H264:
@ -166,7 +252,8 @@ bool Session::validateLaunch()
else if (!isHardwareDecodeAvailable(m_Preferences.videoDecoderSelection,
VIDEO_FORMAT_H265,
m_StreamConfig.width,
m_StreamConfig.height)) {
m_StreamConfig.height,
m_StreamConfig.fps)) {
// NOTE: HEVC currently uses only 1 slice regardless of what
// we provide in CAPABILITY_SLICES_PER_FRAME(), so we should
// never use it for software decoding (unless common-c starts
@ -191,7 +278,8 @@ bool Session::validateLaunch()
else if (!isHardwareDecodeAvailable(m_Preferences.videoDecoderSelection,
VIDEO_FORMAT_H265_MAIN10,
m_StreamConfig.width,
m_StreamConfig.height)) {
m_StreamConfig.height,
m_StreamConfig.fps)) {
emit displayLaunchWarning("Your client PC GPU doesn't support HEVC Main10 decoding for HDR streaming.");
}
else {
@ -414,12 +502,12 @@ void Session::exec()
SDL_GETEVENT,
SDL_USEREVENT,
SDL_USEREVENT) == 1) {
dropFrame(&event.user);
m_VideoDecoder->dropFrame(&event.user);
event = nextEvent;
}
// Render the last frame
renderFrame(&event.user);
m_VideoDecoder->renderFrame(&event.user);
break;
}
case SDL_KEYUP:
@ -463,6 +551,12 @@ DispatchDeferredCleanup:
SDL_HideWindow(m_Window);
}
// Destroy the decoder, since this must be done on the main thread
SDL_AtomicLock(&m_DecoderLock);
delete m_VideoDecoder;
m_VideoDecoder = nullptr;
SDL_AtomicUnlock(&m_DecoderLock);
// Cleanup can take a while, so dispatch it to a worker thread.
// When it is complete, it will release our s_ActiveSessionSemaphore
// reference.

39
app/streaming/session.hpp
View file

@ -7,13 +7,7 @@
#include "backend/computermanager.h"
#include "settings/streamingpreferences.h"
#include "input.hpp"
#include "renderers/renderer.h"
extern "C" {
#include <libavcodec/avcodec.h>
}
#define SDL_CODE_FRAME_READY 0
#include "video/decoder.h"
class Session : public QObject
{
@ -45,26 +39,14 @@ private:
int sdlDetermineAudioConfiguration();
static
bool chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width, int height,
AVCodec*& chosenDecoder,
const AVCodecHWConfig*& chosenHwConfig,
IRenderer*& newRenderer);
static
enum AVPixelFormat getHwFormat(AVCodecContext*,
const enum AVPixelFormat* pixFmts);
static
bool isHardwareDecodeAvailable(StreamingPreferences::VideoDecoderSelection vds,
int videoFormat, int width, int height);
int videoFormat, int width, int height, int frameRate);
void renderFrame(SDL_UserEvent* event);
void dropFrame(SDL_UserEvent* event);
static
bool chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window, int videoFormat, int width, int height,
int frameRate, IVideoDecoder*& chosenDecoder);
static
void clStageStarting(int stage);
@ -110,13 +92,8 @@ private:
NvComputer* m_Computer;
NvApp m_App;
SDL_Window* m_Window;
static AVPacket s_Pkt;
static AVCodecContext* s_VideoDecoderCtx;
static QByteArray s_DecodeBuffer;
static AVBufferRef* s_HwDeviceCtx;
static const AVCodecHWConfig* s_HwDecodeCfg;
static IRenderer* s_Renderer;
IVideoDecoder* m_VideoDecoder;
SDL_SpinLock m_DecoderLock;
static SDL_AudioDeviceID s_AudioDevice;
static OpusMSDecoder* s_OpusDecoder;

37
app/streaming/video/decoder.h Normal file
View file

@ -0,0 +1,37 @@
#pragma once
#include <Limelight.h>
#include <SDL.h>
#include "settings/streamingpreferences.h"
#define SDL_CODE_FRAME_READY 0
#define MAX_SLICES 4
class IVideoDecoder {
public:
virtual ~IVideoDecoder() {}
virtual bool initialize(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width,
int height,
int frameRate) = 0;
virtual bool isHardwareAccelerated() = 0;
virtual int submitDecodeUnit(PDECODE_UNIT du) = 0;
virtual void renderFrame(SDL_UserEvent* event) = 0;
virtual void dropFrame(SDL_UserEvent* event) = 0;
virtual void queueFrame(void* data1 = nullptr,
void* data2 = nullptr)
{
SDL_Event event;
event.type = SDL_USEREVENT;
event.user.code = SDL_CODE_FRAME_READY;
event.user.data1 = data1;
event.user.data2 = data2;
SDL_PushEvent(&event);
}
};

0
app/streaming/renderers/dxva2.cpp → app/streaming/video/ffmpeg-renderers/dxva2.cpp
View file

2
app/streaming/renderers/dxva2.h → app/streaming/video/ffmpeg-renderers/dxva2.h
View file

@ -9,7 +9,7 @@ extern "C" {
#include <libavcodec/dxva2.h>
}
class DXVA2Renderer : public IRenderer
class DXVA2Renderer : public IFFmpegRenderer
{
public:
DXVA2Renderer();

6
app/streaming/renderers/renderer.h → app/streaming/video/ffmpeg-renderers/renderer.h
View file

@ -6,9 +6,9 @@ extern "C" {
#include <libavcodec/avcodec.h>
}
class IRenderer {
class IFFmpegRenderer {
public:
virtual ~IRenderer() {}
virtual ~IFFmpegRenderer() {}
virtual bool initialize(SDL_Window* window,
int videoFormat,
int width,
@ -17,7 +17,7 @@ public:
virtual void renderFrame(AVFrame* frame) = 0;
};
class SdlRenderer : public IRenderer {
class SdlRenderer : public IFFmpegRenderer {
public:
SdlRenderer();
virtual ~SdlRenderer();

2
app/streaming/renderers/sdl.cpp → app/streaming/video/ffmpeg-renderers/sdl.cpp
View file

@ -1,4 +1,4 @@
#include "streaming/session.hpp"
#include "renderer.h"
SdlRenderer::SdlRenderer()
: m_Renderer(nullptr),

2
app/streaming/renderers/vt.h → app/streaming/video/ffmpeg-renderers/vt.h
View file

@ -7,5 +7,5 @@
class VTRendererFactory {
public:
static
IRenderer* createRenderer();
IFFmpegRenderer* createRenderer();
};

4
app/streaming/renderers/vt.mm → app/streaming/video/ffmpeg-renderers/vt.mm
View file

@ -11,7 +11,7 @@
#import <VideoToolbox/VideoToolbox.h>
#import <AVFoundation/AVFoundation.h>
class VTRenderer : public IRenderer
class VTRenderer : public IFFmpegRenderer
{
public:
VTRenderer()
@ -194,6 +194,6 @@ private:
NSView* m_View;
};
IRenderer* VTRendererFactory::createRenderer() {
IFFmpegRenderer* VTRendererFactory::createRenderer() {
return new VTRenderer();
}

183
app/streaming/video.cpp → app/streaming/video/ffmpeg.cpp
View file

@ -1,42 +1,22 @@
#include <Limelight.h>
#include "session.hpp"
#include "ffmpeg.h"
#ifdef _WIN32
#include "renderers/dxva2.h"
#include "ffmpeg-renderers/dxva2.h"
#endif
#ifdef __APPLE__
#include "renderers/vt.h"
#include "ffmpeg-renderers/vt.h"
#endif
AVPacket Session::s_Pkt;
AVCodecContext* Session::s_VideoDecoderCtx;
QByteArray Session::s_DecodeBuffer;
const AVCodecHWConfig* Session::s_HwDecodeCfg;
IRenderer* Session::s_Renderer;
#define MAX_SLICES 4
int Session::getDecoderCapabilities()
{
int caps = 0;
// Submit for decode without using a separate thread
caps |= CAPABILITY_DIRECT_SUBMIT;
// Slice up to 4 times for parallel decode, once slice per core
caps |= CAPABILITY_SLICES_PER_FRAME(qMin(MAX_SLICES, SDL_GetCPUCount()));
return caps;
}
bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width, int height,
AVCodec*& chosenDecoder,
const AVCodecHWConfig*& chosenHwConfig,
IRenderer*& newRenderer)
bool FFmpegVideoDecoder::chooseDecoder(
StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width, int height,
AVCodec*& chosenDecoder,
const AVCodecHWConfig*& chosenHwConfig,
IFFmpegRenderer*& newRenderer)
{
if (videoFormat & VIDEO_FORMAT_MASK_H264) {
chosenDecoder = avcodec_find_decoder(AV_CODEC_ID_H264);
@ -106,125 +86,104 @@ bool Session::chooseDecoder(StreamingPreferences::VideoDecoderSelection vds,
}
}
bool Session::isHardwareDecodeAvailable(
StreamingPreferences::VideoDecoderSelection vds,
int videoFormat, int width, int height)
bool FFmpegVideoDecoder::isHardwareAccelerated()
{
AVCodec* decoder;
const AVCodecHWConfig* hwConfig;
IRenderer* renderer;
// Create temporary window to instantiate the decoder
SDL_Window* window = SDL_CreateWindow("", 0, 0, width, height, SDL_WINDOW_HIDDEN);
if (!window) {
return false;
}
if (chooseDecoder(vds, window, videoFormat, width, height, decoder, hwConfig, renderer)) {
// The renderer may have referenced the window, so
// we must delete the renderer before the window.
delete renderer;
SDL_DestroyWindow(window);
return hwConfig != nullptr;
}
else {
SDL_DestroyWindow(window);
// Failed to find *any* decoder, including software
return false;
}
return m_HwDecodeCfg != nullptr;
}
int Session::drSetup(int videoFormat, int width, int height, int /* frameRate */, void*, int)
FFmpegVideoDecoder::FFmpegVideoDecoder()
: m_VideoDecoderCtx(nullptr),
m_DecodeBuffer(1024 * 1024, 0),
m_HwDecodeCfg(nullptr),
m_Renderer(nullptr)
{
AVCodec* decoder;
av_init_packet(&m_Pkt);
// Use linear filtering when renderer scaling is required
SDL_SetHint(SDL_HINT_RENDER_SCALE_QUALITY, "1");
}
av_init_packet(&s_Pkt);
FFmpegVideoDecoder::~FFmpegVideoDecoder()
{
avcodec_close(m_VideoDecoderCtx);
av_free(m_VideoDecoderCtx);
m_VideoDecoderCtx = nullptr;
if (!chooseDecoder(s_ActiveSession->m_Preferences.videoDecoderSelection,
s_ActiveSession->m_Window,
videoFormat, width, height,
decoder, s_HwDecodeCfg, s_Renderer)) {
m_HwDecodeCfg = nullptr;
delete m_Renderer;
m_Renderer = nullptr;
}
bool FFmpegVideoDecoder::initialize(
StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width,
int height,
int)
{
AVCodec* decoder;
if (!chooseDecoder(vds, window, videoFormat, width, height,
decoder, m_HwDecodeCfg, m_Renderer)) {
// Error logged in chooseDecoder()
return -1;
return false;
}
s_VideoDecoderCtx = avcodec_alloc_context3(decoder);
if (!s_VideoDecoderCtx) {
m_VideoDecoderCtx = avcodec_alloc_context3(decoder);
if (!m_VideoDecoderCtx) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to allocate video decoder context");
delete s_Renderer;
return -1;
return false;
}
// Always request low delay decoding
s_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
m_VideoDecoderCtx->flags |= AV_CODEC_FLAG_LOW_DELAY;
// Enable slice multi-threading for software decoding
if (!s_HwDecodeCfg) {
s_VideoDecoderCtx->thread_type = FF_THREAD_SLICE;
s_VideoDecoderCtx->thread_count = qMin(MAX_SLICES, SDL_GetCPUCount());
if (!m_HwDecodeCfg) {
m_VideoDecoderCtx->thread_type = FF_THREAD_SLICE;
m_VideoDecoderCtx->thread_count = qMin(MAX_SLICES, SDL_GetCPUCount());
}
else {
// No threading for HW decode
s_VideoDecoderCtx->thread_count = 1;
m_VideoDecoderCtx->thread_count = 1;
}
// Setup decoding parameters
s_VideoDecoderCtx->width = width;
s_VideoDecoderCtx->height = height;
s_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
m_VideoDecoderCtx->width = width;
m_VideoDecoderCtx->height = height;
m_VideoDecoderCtx->pix_fmt = AV_PIX_FMT_YUV420P; // FIXME: HDR
// Allow the renderer to attach data to this decoder
if (!s_Renderer->prepareDecoderContext(s_VideoDecoderCtx)) {
delete s_Renderer;
av_free(s_VideoDecoderCtx);
return -1;
if (!m_Renderer->prepareDecoderContext(m_VideoDecoderCtx)) {
return false;
}
int err = avcodec_open2(s_VideoDecoderCtx, decoder, nullptr);
int err = avcodec_open2(m_VideoDecoderCtx, decoder, nullptr);
if (err < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Unable to open decoder for format: %x",
videoFormat);
delete s_Renderer;
av_free(s_VideoDecoderCtx);
return -1;
return false;
}
// 1MB frame buffer to start
s_DecodeBuffer = QByteArray(1024 * 1024, 0);
return 0;
return true;
}
void Session::drCleanup()
{
avcodec_close(s_VideoDecoderCtx);
av_free(s_VideoDecoderCtx);
s_VideoDecoderCtx = nullptr;
s_HwDecodeCfg = nullptr;
delete s_Renderer;
s_Renderer = nullptr;
}
int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
{
PLENTRY entry = du->bufferList;
int err;
if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > s_DecodeBuffer.length()) {
s_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0);
if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > m_DecodeBuffer.length()) {
m_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0);
}
int offset = 0;
while (entry != nullptr) {
memcpy(&s_DecodeBuffer.data()[offset],
memcpy(&m_DecodeBuffer.data()[offset],
entry->data,
entry->length);
offset += entry->length;
@ -233,10 +192,10 @@ int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
SDL_assert(offset == du->fullLength);
s_Pkt.data = reinterpret_cast<uint8_t*>(s_DecodeBuffer.data());
s_Pkt.size = du->fullLength;
m_Pkt.data = reinterpret_cast<uint8_t*>(m_DecodeBuffer.data());
m_Pkt.size = du->fullLength;
err = avcodec_send_packet(s_VideoDecoderCtx, &s_Pkt);
err = avcodec_send_packet(m_VideoDecoderCtx, &m_Pkt);
if (err < 0) {
SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
"Decoding failed: %d", err);
@ -256,7 +215,7 @@ int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
return DR_OK;
}
err = avcodec_receive_frame(s_VideoDecoderCtx, frame);
err = avcodec_receive_frame(m_VideoDecoderCtx, frame);
if (err == 0) {
SDL_Event event;
@ -275,15 +234,15 @@ int Session::drSubmitDecodeUnit(PDECODE_UNIT du)
}
// Called on main thread
void Session::renderFrame(SDL_UserEvent* event)
void FFmpegVideoDecoder::renderFrame(SDL_UserEvent* event)
{
AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
s_Renderer->renderFrame(frame);
m_Renderer->renderFrame(frame);
av_frame_free(&frame);
}
// Called on main thread
void Session::dropFrame(SDL_UserEvent* event)
void FFmpegVideoDecoder::dropFrame(SDL_UserEvent* event)
{
AVFrame* frame = reinterpret_cast<AVFrame*>(event->data1);
av_frame_free(&frame);

41
app/streaming/video/ffmpeg.h Normal file
View file

@ -0,0 +1,41 @@
#pragma once
#include "decoder.h"
#include "ffmpeg-renderers/renderer.h"
extern "C" {
#include <libavcodec/avcodec.h>
}
class FFmpegVideoDecoder : public IVideoDecoder {
public:
FFmpegVideoDecoder();
virtual ~FFmpegVideoDecoder();
virtual bool initialize(StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width,
int height,
int frameRate) override;
virtual bool isHardwareAccelerated() override;
virtual int submitDecodeUnit(PDECODE_UNIT du) override;
virtual void renderFrame(SDL_UserEvent* event) override;
virtual void dropFrame(SDL_UserEvent* event) override;
private:
bool
chooseDecoder(
StreamingPreferences::VideoDecoderSelection vds,
SDL_Window* window,
int videoFormat,
int width, int height,
AVCodec*& chosenDecoder,
const AVCodecHWConfig*& chosenHwConfig,
IFFmpegRenderer*& newRenderer);
AVPacket m_Pkt;
AVCodecContext* m_VideoDecoderCtx;
QByteArray m_DecodeBuffer;
const AVCodecHWConfig* m_HwDecodeCfg;
IFFmpegRenderer* m_Renderer;
};