Mirrors/moonlight-qt
2
0
Fork 0
mirror of https://github.com/moonlight-stream/moonlight-qt synced 2024-09-21 14:41:57 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor audio rendering into a separate class

Browse source
This commit is contained in:
Cameron Gutman 2018-09-13 06:23:06 -07:00
parent 9e2fd67487
commit 12496e4432
9 changed files with 244 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
app/app.pro
View file

@ -87,7 +87,8 @@ SOURCES += \
settings/streamingpreferences.cpp \
streaming/input.cpp \
streaming/session.cpp \
streaming/audio.cpp \
streaming/audio/audio.cpp \
streaming/audio/renderers/sdlaud.cpp \
gui/computermodel.cpp \
gui/appmodel.cpp \
streaming/streamutils.cpp \
@ -104,6 +105,8 @@ HEADERS += \
settings/streamingpreferences.h \
streaming/input.hpp \
streaming/session.hpp \
streaming/audio/renderers/renderer.h \
streaming/audio/renderers/sdl.h \
gui/computermodel.h \
gui/appmodel.h \
streaming/video/decoder.h \
@ -118,7 +121,7 @@ ffmpeg {
DEFINES += HAVE_FFMPEG
SOURCES += \
streaming/video/ffmpeg.cpp \
streaming/video/ffmpeg-renderers/sdl.cpp \
streaming/video/ffmpeg-renderers/sdlvid.cpp \
streaming/video/ffmpeg-renderers/pacer/pacer.cpp \
streaming/video/ffmpeg-renderers/pacer/nullthreadedvsyncsource.cpp

111
app/streaming/audio/audio.cpp Normal file
View file

@ -0,0 +1,111 @@
#include "../session.hpp"
#include "renderers/renderer.h"
#include "renderers/sdl.h"
#include <Limelight.h>
IAudioRenderer* Session::createAudioRenderer()
{
return new SdlAudioRenderer();
}
bool Session::testAudio(int audioConfiguration)
{
IAudioRenderer* audioRenderer;
audioRenderer = createAudioRenderer();
if (audioRenderer == nullptr) {
return false;
}
bool ret = audioRenderer->testAudio(audioConfiguration);
delete audioRenderer;
return ret;
}
int Session::detectAudioConfiguration()
{
IAudioRenderer* audioRenderer;
audioRenderer = createAudioRenderer();
if (audioRenderer == nullptr) {
// Hope for the best
return AUDIO_CONFIGURATION_STEREO;
}
int audioConfig = audioRenderer->detectAudioConfiguration();
delete audioRenderer;
return audioConfig;
}
int Session::arInit(int /* audioConfiguration */,
const POPUS_MULTISTREAM_CONFIGURATION opusConfig,
void* /* arContext */, int /* arFlags */)
{
int error;
SDL_memcpy(&s_ActiveSession->m_AudioConfig, opusConfig, sizeof(*opusConfig));
s_ActiveSession->m_OpusDecoder =
opus_multistream_decoder_create(opusConfig->sampleRate,
opusConfig->channelCount,
opusConfig->streams,
opusConfig->coupledStreams,
opusConfig->mapping,
&error);
if (s_ActiveSession->m_OpusDecoder == NULL) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to create decoder: %d",
error);
return -1;
}
s_ActiveSession->m_AudioRenderer = s_ActiveSession->createAudioRenderer();
if (s_ActiveSession->m_AudioRenderer == nullptr) {
opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
return -2;
}
s_ActiveSession->m_AudioRenderer->prepareForPlayback(opusConfig);
if (s_ActiveSession->m_AudioRenderer == nullptr) {
delete s_ActiveSession->m_AudioRenderer;
opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
return -3;
}
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Audio stream has %d channels",
opusConfig->channelCount);
return 0;
}
void Session::arCleanup()
{
delete s_ActiveSession->m_AudioRenderer;
opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder);
s_ActiveSession->m_OpusDecoder = nullptr;
}
void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength)
{
int samplesDecoded;
samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder,
(unsigned char*)sampleData,
sampleLength,
s_ActiveSession->m_OpusDecodeBuffer,
SAMPLES_PER_FRAME,
0);
if (samplesDecoded > 0) {
s_ActiveSession->m_AudioRenderer->submitAudio(s_ActiveSession->m_OpusDecodeBuffer,
static_cast<int>(sizeof(short) *
samplesDecoded *
s_ActiveSession->m_AudioConfig.channelCount));
}
}

20
app/streaming/audio/renderers/renderer.h Normal file
View file

@ -0,0 +1,20 @@
#pragma once
#include <Limelight.h>
#define MAX_CHANNELS 6
#define SAMPLES_PER_FRAME 240
class IAudioRenderer
{
public:
virtual ~IAudioRenderer() {}
virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) = 0;
virtual void submitAudio(short* audioBuffer, int audioSize) = 0;
virtual bool testAudio(int audioConfiguration) = 0;
virtual int detectAudioConfiguration() = 0;
};

28
app/streaming/audio/renderers/sdl.h Normal file
View file

@ -0,0 +1,28 @@
#pragma once
#include "renderer.h"
#include <SDL.h>
class SdlAudioRenderer : public IAudioRenderer
{
public:
SdlAudioRenderer();
virtual ~SdlAudioRenderer();
virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig);
virtual void submitAudio(short* audioBuffer, int audioSize);
virtual bool testAudio(int audioConfiguration);
virtual int detectAudioConfiguration();
private:
SDL_AudioDeviceID m_AudioDevice;
int m_ChannelCount;
int m_PendingDrops;
int m_PendingHardDrops;
unsigned int m_SampleIndex;
Uint32 m_BaselinePendingData;
};

140
app/streaming/audio.cpp → app/streaming/audio/renderers/sdlaud.cpp
View file

@ -1,25 +1,16 @@
#include "session.hpp"
#include "sdl.h"
#include <Limelight.h>
#include <SDL.h>
#define MAX_CHANNELS 6
#define SAMPLES_PER_FRAME 240
#include <QtGlobal>
#define MIN_QUEUED_FRAMES 2
#define MAX_QUEUED_FRAMES 4
#define STOP_THE_WORLD_LIMIT 20
#define DROP_RATIO_DENOM 32
SDL_AudioDeviceID Session::s_AudioDevice;
OpusMSDecoder* Session::s_OpusDecoder;
short Session::s_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME];
int Session::s_ChannelCount;
int Session::s_PendingDrops;
int Session::s_PendingHardDrops;
unsigned int Session::s_SampleIndex;
Uint32 Session::s_BaselinePendingData;
int Session::sdlDetermineAudioConfiguration()
int SdlAudioRenderer::detectAudioConfiguration()
{
SDL_AudioSpec want, have;
SDL_AudioDeviceID dev;
@ -70,7 +61,7 @@ Exit:
return ret;
}
bool Session::testAudio(int audioConfiguration)
bool SdlAudioRenderer::testAudio(int audioConfiguration)
{
SDL_AudioSpec want, have;
SDL_AudioDeviceID dev;
@ -124,12 +115,20 @@ Exit:
return ret;
}
int Session::sdlAudioInit(int /* audioConfiguration */,
POPUS_MULTISTREAM_CONFIGURATION opusConfig,
void* /* arContext */, int /* arFlags */)
SdlAudioRenderer::SdlAudioRenderer()
: m_AudioDevice(0),
m_ChannelCount(0),
m_PendingDrops(0),
m_PendingHardDrops(0),
m_SampleIndex(0),
m_BaselinePendingData(0)
{
}
bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig)
{
SDL_AudioSpec want, have;
int error;
SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) {
@ -150,8 +149,8 @@ int Session::sdlAudioInit(int /* audioConfiguration */,
// Specifying non-Po2 seems to work for our supported platforms.
want.samples = SAMPLES_PER_FRAME;
s_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0);
if (s_AudioDevice == 0) {
m_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0);
if (m_AudioDevice == 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to open audio device: %s",
SDL_GetError());
@ -159,126 +158,83 @@ int Session::sdlAudioInit(int /* audioConfiguration */,
return -1;
}
s_OpusDecoder = opus_multistream_decoder_create(opusConfig->sampleRate,
opusConfig->channelCount,
opusConfig->streams,
opusConfig->coupledStreams,
opusConfig->mapping,
&error);
if (s_OpusDecoder == NULL) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to create decoder: %d",
error);
SDL_CloseAudioDevice(s_AudioDevice);
s_AudioDevice = 0;
SDL_QuitSubSystem(SDL_INIT_AUDIO);
return -2;
}
// SDL counts pending samples in the queued
// audio size using the WASAPI backend. This
// includes silence, which can throw off our
// pending data count. Get a baseline so we
// can exclude that data.
s_BaselinePendingData = 0;
m_BaselinePendingData = 0;
#ifdef Q_OS_WIN32
for (int i = 0; i < 100; i++) {
s_BaselinePendingData = qMax(s_BaselinePendingData, SDL_GetQueuedAudioSize(s_AudioDevice));
m_BaselinePendingData = qMax(m_BaselinePendingData, SDL_GetQueuedAudioSize(m_AudioDevice));
SDL_Delay(10);
}
#endif
s_BaselinePendingData *= 2;
m_BaselinePendingData *= 2;
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Baseline pending audio data: %d bytes",
s_BaselinePendingData);
m_BaselinePendingData);
s_ChannelCount = opusConfig->channelCount;
s_SampleIndex = 0;
s_PendingDrops = s_PendingHardDrops = 0;
m_ChannelCount = opusConfig->channelCount;
m_SampleIndex = 0;
m_PendingDrops = m_PendingHardDrops = 0;
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Audio stream has %d channels",
opusConfig->channelCount);
// Start playback
SDL_PauseAudioDevice(m_AudioDevice, 0);
return 0;
}
void Session::sdlAudioStart()
SdlAudioRenderer::~SdlAudioRenderer()
{
// Unpause the audio device
SDL_PauseAudioDevice(s_AudioDevice, 0);
}
void Session::sdlAudioStop()
{
// Pause the audio device
SDL_PauseAudioDevice(s_AudioDevice, 1);
}
void Session::sdlAudioCleanup()
{
SDL_CloseAudioDevice(s_AudioDevice);
s_AudioDevice = 0;
opus_multistream_decoder_destroy(s_OpusDecoder);
s_OpusDecoder = nullptr;
// Stop playback
SDL_PauseAudioDevice(m_AudioDevice, 1);
SDL_CloseAudioDevice(m_AudioDevice);
SDL_QuitSubSystem(SDL_INIT_AUDIO);
SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO));
}
void Session::sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength)
void SdlAudioRenderer::submitAudio(short* audioBuffer, int audioSize)
{
int samplesDecoded;
m_SampleIndex++;
s_SampleIndex++;
Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(s_AudioDevice) - (int)s_BaselinePendingData, 0);
Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * s_ChannelCount * sizeof(short));
Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(m_AudioDevice) - (int)m_BaselinePendingData, 0);
Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * m_ChannelCount * sizeof(short));
// We must check this prior to the below checks to ensure we don't
// underflow if framesQueued - s_PendingHardDrops < 0.
// underflow if framesQueued - m_PendingHardDrops < 0.
if (framesQueued <= MIN_QUEUED_FRAMES) {
s_PendingDrops = s_PendingHardDrops = 0;
m_PendingDrops = m_PendingHardDrops = 0;
}
// Pend enough drops to get us back to MIN_QUEUED_FRAMES
else if (framesQueued - s_PendingHardDrops > STOP_THE_WORLD_LIMIT) {
s_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES;
else if (framesQueued - m_PendingHardDrops > STOP_THE_WORLD_LIMIT) {
m_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES;
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
"Pending hard drop of %u audio frames",
s_PendingHardDrops);
m_PendingHardDrops);
}
else if (framesQueued - s_PendingHardDrops - s_PendingDrops > MAX_QUEUED_FRAMES) {
s_PendingDrops = framesQueued - MIN_QUEUED_FRAMES;
else if (framesQueued - m_PendingHardDrops - m_PendingDrops > MAX_QUEUED_FRAMES) {
m_PendingDrops = framesQueued - MIN_QUEUED_FRAMES;
}
// Determine if this frame should be dropped
if (s_PendingHardDrops != 0) {
if (m_PendingHardDrops != 0) {
// Hard drops happen all at once to forcefully
// resync with the source.
s_PendingHardDrops--;
m_PendingHardDrops--;
return;
}
else if (s_PendingDrops != 0 && s_SampleIndex % DROP_RATIO_DENOM == 0) {
else if (m_PendingDrops != 0 && m_SampleIndex % DROP_RATIO_DENOM == 0) {
// Normal drops are interspersed with the audio data
// to hide the glitches.
s_PendingDrops--;
m_PendingDrops--;
return;
}
samplesDecoded = opus_multistream_decode(s_OpusDecoder,
(unsigned char*)sampleData,
sampleLength,
s_OpusDecodeBuffer,
SAMPLES_PER_FRAME,
0);
if (samplesDecoded > 0) {
if (SDL_QueueAudio(s_AudioDevice,
s_OpusDecodeBuffer,
sizeof(short) * samplesDecoded * s_ChannelCount) < 0) {
if (SDL_QueueAudio(m_AudioDevice, audioBuffer, audioSize) < 0) {
SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
"Failed to queue audio sample: %s",
SDL_GetError());
}
}
}

16
app/streaming/session.cpp
View file

@ -48,11 +48,11 @@ CONNECTION_LISTENER_CALLBACKS Session::k_ConnCallbacks = {
};
AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = {
Session::sdlAudioInit,
Session::sdlAudioStart,
Session::sdlAudioStop,
Session::sdlAudioCleanup,
Session::sdlAudioDecodeAndPlaySample,
Session::arInit,
nullptr,
nullptr,
Session::arCleanup,
Session::arDecodeAndPlaySample,
CAPABILITY_DIRECT_SUBMIT
};
@ -277,7 +277,9 @@ Session::Session(NvComputer* computer, NvApp& app)
m_AudioDisabled(false),
m_DisplayOriginX(0),
m_DisplayOriginY(0),
m_PendingWindowedTransition(false)
m_PendingWindowedTransition(false),
m_OpusDecoder(nullptr),
m_AudioRenderer(nullptr)
{
}
@ -321,7 +323,7 @@ void Session::initialize()
{
case StreamingPreferences::AC_AUTO:
SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Autodetecting audio configuration");
m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration();
m_StreamConfig.audioConfiguration = detectAudioConfiguration();
break;
case StreamingPreferences::AC_FORCE_STEREO:
m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_STEREO;

45
app/streaming/session.hpp
View file

@ -8,6 +8,7 @@
#include "settings/streamingpreferences.h"
#include "input.hpp"
#include "video/decoder.h"
#include "audio/renderers/renderer.h"
class Session : public QObject
{
@ -49,7 +50,9 @@ private:
int getDecoderCapabilities();
int sdlDetermineAudioConfiguration();
IAudioRenderer* createAudioRenderer();
int detectAudioConfiguration();
bool testAudio(int audioConfiguration);
@ -77,6 +80,17 @@ private:
static
void clLogMessage(const char* format, ...);
static
int arInit(int audioConfiguration,
const POPUS_MULTISTREAM_CONFIGURATION opusConfig,
void* arContext, int arFlags);
static
void arCleanup();
static
void arDecodeAndPlaySample(char* sampleData, int sampleLength);
static
int drSetup(int videoFormat, int width, int height, int frameRate, void*, int);
@ -86,23 +100,6 @@ private:
static
int drSubmitDecodeUnit(PDECODE_UNIT du);
static
int sdlAudioInit(int audioConfiguration,
POPUS_MULTISTREAM_CONFIGURATION opusConfig,
void* arContext, int arFlags);
static
void sdlAudioStart();
static
void sdlAudioStop();
static
void sdlAudioCleanup();
static
void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength);
StreamingPreferences m_Preferences;
STREAM_CONFIGURATION m_StreamConfig;
DECODER_RENDERER_CALLBACKS m_VideoCallbacks;
@ -123,14 +120,10 @@ private:
int m_ActiveVideoHeight;
int m_ActiveVideoFrameRate;
static SDL_AudioDeviceID s_AudioDevice;
static OpusMSDecoder* s_OpusDecoder;
static short s_OpusDecodeBuffer[];
static int s_ChannelCount;
static int s_PendingDrops;
static int s_PendingHardDrops;
static unsigned int s_SampleIndex;
static Uint32 s_BaselinePendingData;
OpusMSDecoder* m_OpusDecoder;
short m_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME];
IAudioRenderer* m_AudioRenderer;
OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig;
static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks;
static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks;

0
app/streaming/video/ffmpeg-renderers/sdl.cpp → app/streaming/video/ffmpeg-renderers/sdlvid.cpp
View file

2
moonlight-common-c/moonlight-common-c

@ -1 +1 @@
Subproject commit 8743ce5d083407c33c8b3c23529eaa9630361e7c
Subproject commit 718d6a4b28a209902848c8678207ab6a48474a63