diff --git a/app/app.pro b/app/app.pro index 26d4cafc..2e571dae 100644 --- a/app/app.pro +++ b/app/app.pro @@ -87,7 +87,8 @@ SOURCES += \ settings/streamingpreferences.cpp \ streaming/input.cpp \ streaming/session.cpp \ - streaming/audio.cpp \ + streaming/audio/audio.cpp \ + streaming/audio/renderers/sdlaud.cpp \ gui/computermodel.cpp \ gui/appmodel.cpp \ streaming/streamutils.cpp \ @@ -104,6 +105,8 @@ HEADERS += \ settings/streamingpreferences.h \ streaming/input.hpp \ streaming/session.hpp \ + streaming/audio/renderers/renderer.h \ + streaming/audio/renderers/sdl.h \ gui/computermodel.h \ gui/appmodel.h \ streaming/video/decoder.h \ @@ -118,7 +121,7 @@ ffmpeg { DEFINES += HAVE_FFMPEG SOURCES += \ streaming/video/ffmpeg.cpp \ - streaming/video/ffmpeg-renderers/sdl.cpp \ + streaming/video/ffmpeg-renderers/sdlvid.cpp \ streaming/video/ffmpeg-renderers/pacer/pacer.cpp \ streaming/video/ffmpeg-renderers/pacer/nullthreadedvsyncsource.cpp diff --git a/app/streaming/audio/audio.cpp b/app/streaming/audio/audio.cpp new file mode 100644 index 00000000..1df4e7af --- /dev/null +++ b/app/streaming/audio/audio.cpp @@ -0,0 +1,111 @@ +#include "../session.hpp" +#include "renderers/renderer.h" +#include "renderers/sdl.h" + +#include + +IAudioRenderer* Session::createAudioRenderer() +{ + return new SdlAudioRenderer(); +} + +bool Session::testAudio(int audioConfiguration) +{ + IAudioRenderer* audioRenderer; + + audioRenderer = createAudioRenderer(); + if (audioRenderer == nullptr) { + return false; + } + + bool ret = audioRenderer->testAudio(audioConfiguration); + + delete audioRenderer; + + return ret; +} + +int Session::detectAudioConfiguration() +{ + IAudioRenderer* audioRenderer; + + audioRenderer = createAudioRenderer(); + if (audioRenderer == nullptr) { + // Hope for the best + return AUDIO_CONFIGURATION_STEREO; + } + + int audioConfig = audioRenderer->detectAudioConfiguration(); + + delete audioRenderer; + + return audioConfig; +} + +int Session::arInit(int /* audioConfiguration */, + const POPUS_MULTISTREAM_CONFIGURATION opusConfig, + void* /* arContext */, int /* arFlags */) +{ + int error; + + SDL_memcpy(&s_ActiveSession->m_AudioConfig, opusConfig, sizeof(*opusConfig)); + + s_ActiveSession->m_OpusDecoder = + opus_multistream_decoder_create(opusConfig->sampleRate, + opusConfig->channelCount, + opusConfig->streams, + opusConfig->coupledStreams, + opusConfig->mapping, + &error); + if (s_ActiveSession->m_OpusDecoder == NULL) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to create decoder: %d", + error); + return -1; + } + + s_ActiveSession->m_AudioRenderer = s_ActiveSession->createAudioRenderer(); + if (s_ActiveSession->m_AudioRenderer == nullptr) { + opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder); + return -2; + } + + s_ActiveSession->m_AudioRenderer->prepareForPlayback(opusConfig); + if (s_ActiveSession->m_AudioRenderer == nullptr) { + delete s_ActiveSession->m_AudioRenderer; + opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder); + return -3; + } + + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "Audio stream has %d channels", + opusConfig->channelCount); + + return 0; +} + +void Session::arCleanup() +{ + delete s_ActiveSession->m_AudioRenderer; + + opus_multistream_decoder_destroy(s_ActiveSession->m_OpusDecoder); + s_ActiveSession->m_OpusDecoder = nullptr; +} + +void Session::arDecodeAndPlaySample(char* sampleData, int sampleLength) +{ + int samplesDecoded; + + samplesDecoded = opus_multistream_decode(s_ActiveSession->m_OpusDecoder, + (unsigned char*)sampleData, + sampleLength, + s_ActiveSession->m_OpusDecodeBuffer, + SAMPLES_PER_FRAME, + 0); + if (samplesDecoded > 0) { + s_ActiveSession->m_AudioRenderer->submitAudio(s_ActiveSession->m_OpusDecodeBuffer, + static_cast(sizeof(short) * + samplesDecoded * + s_ActiveSession->m_AudioConfig.channelCount)); + } +} diff --git a/app/streaming/audio/renderers/renderer.h b/app/streaming/audio/renderers/renderer.h new file mode 100644 index 00000000..95bc7f40 --- /dev/null +++ b/app/streaming/audio/renderers/renderer.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#define MAX_CHANNELS 6 +#define SAMPLES_PER_FRAME 240 + +class IAudioRenderer +{ +public: + virtual ~IAudioRenderer() {} + + virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) = 0; + + virtual void submitAudio(short* audioBuffer, int audioSize) = 0; + + virtual bool testAudio(int audioConfiguration) = 0; + + virtual int detectAudioConfiguration() = 0; +}; diff --git a/app/streaming/audio/renderers/sdl.h b/app/streaming/audio/renderers/sdl.h new file mode 100644 index 00000000..78c5d30e --- /dev/null +++ b/app/streaming/audio/renderers/sdl.h @@ -0,0 +1,28 @@ +#pragma once + +#include "renderer.h" +#include + +class SdlAudioRenderer : public IAudioRenderer +{ +public: + SdlAudioRenderer(); + + virtual ~SdlAudioRenderer(); + + virtual bool prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig); + + virtual void submitAudio(short* audioBuffer, int audioSize); + + virtual bool testAudio(int audioConfiguration); + + virtual int detectAudioConfiguration(); + +private: + SDL_AudioDeviceID m_AudioDevice; + int m_ChannelCount; + int m_PendingDrops; + int m_PendingHardDrops; + unsigned int m_SampleIndex; + Uint32 m_BaselinePendingData; +}; diff --git a/app/streaming/audio.cpp b/app/streaming/audio/renderers/sdlaud.cpp similarity index 56% rename from app/streaming/audio.cpp rename to app/streaming/audio/renderers/sdlaud.cpp index dd2958ab..a0c36adf 100644 --- a/app/streaming/audio.cpp +++ b/app/streaming/audio/renderers/sdlaud.cpp @@ -1,25 +1,16 @@ -#include "session.hpp" +#include "sdl.h" #include #include -#define MAX_CHANNELS 6 -#define SAMPLES_PER_FRAME 240 +#include + #define MIN_QUEUED_FRAMES 2 #define MAX_QUEUED_FRAMES 4 #define STOP_THE_WORLD_LIMIT 20 #define DROP_RATIO_DENOM 32 -SDL_AudioDeviceID Session::s_AudioDevice; -OpusMSDecoder* Session::s_OpusDecoder; -short Session::s_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME]; -int Session::s_ChannelCount; -int Session::s_PendingDrops; -int Session::s_PendingHardDrops; -unsigned int Session::s_SampleIndex; -Uint32 Session::s_BaselinePendingData; - -int Session::sdlDetermineAudioConfiguration() +int SdlAudioRenderer::detectAudioConfiguration() { SDL_AudioSpec want, have; SDL_AudioDeviceID dev; @@ -70,7 +61,7 @@ Exit: return ret; } -bool Session::testAudio(int audioConfiguration) +bool SdlAudioRenderer::testAudio(int audioConfiguration) { SDL_AudioSpec want, have; SDL_AudioDeviceID dev; @@ -124,12 +115,20 @@ Exit: return ret; } -int Session::sdlAudioInit(int /* audioConfiguration */, - POPUS_MULTISTREAM_CONFIGURATION opusConfig, - void* /* arContext */, int /* arFlags */) +SdlAudioRenderer::SdlAudioRenderer() + : m_AudioDevice(0), + m_ChannelCount(0), + m_PendingDrops(0), + m_PendingHardDrops(0), + m_SampleIndex(0), + m_BaselinePendingData(0) +{ + +} + +bool SdlAudioRenderer::prepareForPlayback(const OPUS_MULTISTREAM_CONFIGURATION* opusConfig) { SDL_AudioSpec want, have; - int error; SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO)); if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) { @@ -150,8 +149,8 @@ int Session::sdlAudioInit(int /* audioConfiguration */, // Specifying non-Po2 seems to work for our supported platforms. want.samples = SAMPLES_PER_FRAME; - s_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0); - if (s_AudioDevice == 0) { + m_AudioDevice = SDL_OpenAudioDevice(NULL, 0, &want, &have, 0); + if (m_AudioDevice == 0) { SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, "Failed to open audio device: %s", SDL_GetError()); @@ -159,126 +158,83 @@ int Session::sdlAudioInit(int /* audioConfiguration */, return -1; } - s_OpusDecoder = opus_multistream_decoder_create(opusConfig->sampleRate, - opusConfig->channelCount, - opusConfig->streams, - opusConfig->coupledStreams, - opusConfig->mapping, - &error); - if (s_OpusDecoder == NULL) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Failed to create decoder: %d", - error); - SDL_CloseAudioDevice(s_AudioDevice); - s_AudioDevice = 0; - SDL_QuitSubSystem(SDL_INIT_AUDIO); - return -2; - } - // SDL counts pending samples in the queued // audio size using the WASAPI backend. This // includes silence, which can throw off our // pending data count. Get a baseline so we // can exclude that data. - s_BaselinePendingData = 0; + m_BaselinePendingData = 0; #ifdef Q_OS_WIN32 for (int i = 0; i < 100; i++) { - s_BaselinePendingData = qMax(s_BaselinePendingData, SDL_GetQueuedAudioSize(s_AudioDevice)); + m_BaselinePendingData = qMax(m_BaselinePendingData, SDL_GetQueuedAudioSize(m_AudioDevice)); SDL_Delay(10); } #endif - s_BaselinePendingData *= 2; + m_BaselinePendingData *= 2; SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Baseline pending audio data: %d bytes", - s_BaselinePendingData); + m_BaselinePendingData); - s_ChannelCount = opusConfig->channelCount; - s_SampleIndex = 0; - s_PendingDrops = s_PendingHardDrops = 0; + m_ChannelCount = opusConfig->channelCount; + m_SampleIndex = 0; + m_PendingDrops = m_PendingHardDrops = 0; - SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, - "Audio stream has %d channels", - opusConfig->channelCount); + // Start playback + SDL_PauseAudioDevice(m_AudioDevice, 0); return 0; } -void Session::sdlAudioStart() +SdlAudioRenderer::~SdlAudioRenderer() { - // Unpause the audio device - SDL_PauseAudioDevice(s_AudioDevice, 0); -} - -void Session::sdlAudioStop() -{ - // Pause the audio device - SDL_PauseAudioDevice(s_AudioDevice, 1); -} - -void Session::sdlAudioCleanup() -{ - SDL_CloseAudioDevice(s_AudioDevice); - s_AudioDevice = 0; - - opus_multistream_decoder_destroy(s_OpusDecoder); - s_OpusDecoder = nullptr; + // Stop playback + SDL_PauseAudioDevice(m_AudioDevice, 1); + SDL_CloseAudioDevice(m_AudioDevice); SDL_QuitSubSystem(SDL_INIT_AUDIO); SDL_assert(!SDL_WasInit(SDL_INIT_AUDIO)); } -void Session::sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength) +void SdlAudioRenderer::submitAudio(short* audioBuffer, int audioSize) { - int samplesDecoded; + m_SampleIndex++; - s_SampleIndex++; - - Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(s_AudioDevice) - (int)s_BaselinePendingData, 0); - Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * s_ChannelCount * sizeof(short)); + Uint32 queuedAudio = qMax((int)SDL_GetQueuedAudioSize(m_AudioDevice) - (int)m_BaselinePendingData, 0); + Uint32 framesQueued = queuedAudio / (SAMPLES_PER_FRAME * m_ChannelCount * sizeof(short)); // We must check this prior to the below checks to ensure we don't - // underflow if framesQueued - s_PendingHardDrops < 0. + // underflow if framesQueued - m_PendingHardDrops < 0. if (framesQueued <= MIN_QUEUED_FRAMES) { - s_PendingDrops = s_PendingHardDrops = 0; + m_PendingDrops = m_PendingHardDrops = 0; } // Pend enough drops to get us back to MIN_QUEUED_FRAMES - else if (framesQueued - s_PendingHardDrops > STOP_THE_WORLD_LIMIT) { - s_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES; + else if (framesQueued - m_PendingHardDrops > STOP_THE_WORLD_LIMIT) { + m_PendingHardDrops = framesQueued - MIN_QUEUED_FRAMES; SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Pending hard drop of %u audio frames", - s_PendingHardDrops); + m_PendingHardDrops); } - else if (framesQueued - s_PendingHardDrops - s_PendingDrops > MAX_QUEUED_FRAMES) { - s_PendingDrops = framesQueued - MIN_QUEUED_FRAMES; + else if (framesQueued - m_PendingHardDrops - m_PendingDrops > MAX_QUEUED_FRAMES) { + m_PendingDrops = framesQueued - MIN_QUEUED_FRAMES; } // Determine if this frame should be dropped - if (s_PendingHardDrops != 0) { + if (m_PendingHardDrops != 0) { // Hard drops happen all at once to forcefully // resync with the source. - s_PendingHardDrops--; + m_PendingHardDrops--; return; } - else if (s_PendingDrops != 0 && s_SampleIndex % DROP_RATIO_DENOM == 0) { + else if (m_PendingDrops != 0 && m_SampleIndex % DROP_RATIO_DENOM == 0) { // Normal drops are interspersed with the audio data // to hide the glitches. - s_PendingDrops--; + m_PendingDrops--; return; } - samplesDecoded = opus_multistream_decode(s_OpusDecoder, - (unsigned char*)sampleData, - sampleLength, - s_OpusDecodeBuffer, - SAMPLES_PER_FRAME, - 0); - if (samplesDecoded > 0) { - if (SDL_QueueAudio(s_AudioDevice, - s_OpusDecodeBuffer, - sizeof(short) * samplesDecoded * s_ChannelCount) < 0) { - SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, - "Failed to queue audio sample: %s", - SDL_GetError()); - } + if (SDL_QueueAudio(m_AudioDevice, audioBuffer, audioSize) < 0) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Failed to queue audio sample: %s", + SDL_GetError()); } } diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 462102c5..8d98e358 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -48,11 +48,11 @@ CONNECTION_LISTENER_CALLBACKS Session::k_ConnCallbacks = { }; AUDIO_RENDERER_CALLBACKS Session::k_AudioCallbacks = { - Session::sdlAudioInit, - Session::sdlAudioStart, - Session::sdlAudioStop, - Session::sdlAudioCleanup, - Session::sdlAudioDecodeAndPlaySample, + Session::arInit, + nullptr, + nullptr, + Session::arCleanup, + Session::arDecodeAndPlaySample, CAPABILITY_DIRECT_SUBMIT }; @@ -277,7 +277,9 @@ Session::Session(NvComputer* computer, NvApp& app) m_AudioDisabled(false), m_DisplayOriginX(0), m_DisplayOriginY(0), - m_PendingWindowedTransition(false) + m_PendingWindowedTransition(false), + m_OpusDecoder(nullptr), + m_AudioRenderer(nullptr) { } @@ -321,7 +323,7 @@ void Session::initialize() { case StreamingPreferences::AC_AUTO: SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Autodetecting audio configuration"); - m_StreamConfig.audioConfiguration = sdlDetermineAudioConfiguration(); + m_StreamConfig.audioConfiguration = detectAudioConfiguration(); break; case StreamingPreferences::AC_FORCE_STEREO: m_StreamConfig.audioConfiguration = AUDIO_CONFIGURATION_STEREO; diff --git a/app/streaming/session.hpp b/app/streaming/session.hpp index 24d261e5..8e3038ba 100644 --- a/app/streaming/session.hpp +++ b/app/streaming/session.hpp @@ -8,6 +8,7 @@ #include "settings/streamingpreferences.h" #include "input.hpp" #include "video/decoder.h" +#include "audio/renderers/renderer.h" class Session : public QObject { @@ -49,7 +50,9 @@ private: int getDecoderCapabilities(); - int sdlDetermineAudioConfiguration(); + IAudioRenderer* createAudioRenderer(); + + int detectAudioConfiguration(); bool testAudio(int audioConfiguration); @@ -77,6 +80,17 @@ private: static void clLogMessage(const char* format, ...); + static + int arInit(int audioConfiguration, + const POPUS_MULTISTREAM_CONFIGURATION opusConfig, + void* arContext, int arFlags); + + static + void arCleanup(); + + static + void arDecodeAndPlaySample(char* sampleData, int sampleLength); + static int drSetup(int videoFormat, int width, int height, int frameRate, void*, int); @@ -86,23 +100,6 @@ private: static int drSubmitDecodeUnit(PDECODE_UNIT du); - static - int sdlAudioInit(int audioConfiguration, - POPUS_MULTISTREAM_CONFIGURATION opusConfig, - void* arContext, int arFlags); - - static - void sdlAudioStart(); - - static - void sdlAudioStop(); - - static - void sdlAudioCleanup(); - - static - void sdlAudioDecodeAndPlaySample(char* sampleData, int sampleLength); - StreamingPreferences m_Preferences; STREAM_CONFIGURATION m_StreamConfig; DECODER_RENDERER_CALLBACKS m_VideoCallbacks; @@ -123,14 +120,10 @@ private: int m_ActiveVideoHeight; int m_ActiveVideoFrameRate; - static SDL_AudioDeviceID s_AudioDevice; - static OpusMSDecoder* s_OpusDecoder; - static short s_OpusDecodeBuffer[]; - static int s_ChannelCount; - static int s_PendingDrops; - static int s_PendingHardDrops; - static unsigned int s_SampleIndex; - static Uint32 s_BaselinePendingData; + OpusMSDecoder* m_OpusDecoder; + short m_OpusDecodeBuffer[MAX_CHANNELS * SAMPLES_PER_FRAME]; + IAudioRenderer* m_AudioRenderer; + OPUS_MULTISTREAM_CONFIGURATION m_AudioConfig; static AUDIO_RENDERER_CALLBACKS k_AudioCallbacks; static CONNECTION_LISTENER_CALLBACKS k_ConnCallbacks; diff --git a/app/streaming/video/ffmpeg-renderers/sdl.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp similarity index 100% rename from app/streaming/video/ffmpeg-renderers/sdl.cpp rename to app/streaming/video/ffmpeg-renderers/sdlvid.cpp diff --git a/moonlight-common-c/moonlight-common-c b/moonlight-common-c/moonlight-common-c index 8743ce5d..718d6a4b 160000 --- a/moonlight-common-c/moonlight-common-c +++ b/moonlight-common-c/moonlight-common-c @@ -1 +1 @@ -Subproject commit 8743ce5d083407c33c8b3c23529eaa9630361e7c +Subproject commit 718d6a4b28a209902848c8678207ab6a48474a63