diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 2c479a2a..4ab101d9 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -1019,25 +1019,11 @@ void Session::exec(int displayOriginX, int displayOriginY) SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, "Quit event received"); goto DispatchDeferredCleanup; - case SDL_USEREVENT: { - SDL_Event nextEvent; + case SDL_USEREVENT: SDL_assert(event.user.code == SDL_CODE_FRAME_READY); - - // Drop any earlier frames - while (SDL_PeepEvents(&nextEvent, - 1, - SDL_GETEVENT, - SDL_USEREVENT, - SDL_USEREVENT) == 1) { - m_VideoDecoder->dropFrame(&event.user); - event = nextEvent; - } - - // Render the last frame - m_VideoDecoder->renderFrame(&event.user); + m_VideoDecoder->renderFrameOnMainThread(); break; - } case SDL_WINDOWEVENT: // Capture mouse cursor when user actives the window by clicking on diff --git a/app/streaming/video/decoder.h b/app/streaming/video/decoder.h index ae826f80..139c83e0 100644 --- a/app/streaming/video/decoder.h +++ b/app/streaming/video/decoder.h @@ -40,19 +40,5 @@ public: virtual bool isHardwareAccelerated() = 0; virtual int getDecoderCapabilities() = 0; virtual int submitDecodeUnit(PDECODE_UNIT du) = 0; - virtual void renderFrame(SDL_UserEvent* event) = 0; - virtual void dropFrame(SDL_UserEvent* event) = 0; - - virtual void queueFrame(void* data1 = nullptr, - void* data2 = nullptr) - { - SDL_Event event; - - event.type = SDL_USEREVENT; - event.user.code = SDL_CODE_FRAME_READY; - event.user.data1 = data1; - event.user.data2 = data2; - - SDL_PushEvent(&event); - } + virtual void renderFrameOnMainThread() = 0; }; diff --git a/app/streaming/video/ffmpeg-renderers/dxva2.cpp b/app/streaming/video/ffmpeg-renderers/dxva2.cpp index 643ff4a5..b8e58ec1 100644 --- a/app/streaming/video/ffmpeg-renderers/dxva2.cpp +++ b/app/streaming/video/ffmpeg-renderers/dxva2.cpp @@ -779,6 +779,12 @@ void DXVA2Renderer::notifyOverlayUpdated(Overlay::OverlayType type) } } +bool DXVA2Renderer::isRenderThreadSupported() +{ + // renderFrame() may be called outside of the main thread + return true; +} + void DXVA2Renderer::renderFrame(AVFrame *frame) { IDirect3DSurface9* surface = reinterpret_cast(frame->data[3]); diff --git a/app/streaming/video/ffmpeg-renderers/dxva2.h b/app/streaming/video/ffmpeg-renderers/dxva2.h index 810557a1..8e39bc37 100644 --- a/app/streaming/video/ffmpeg-renderers/dxva2.h +++ b/app/streaming/video/ffmpeg-renderers/dxva2.h @@ -28,6 +28,7 @@ public: virtual int getDecoderCapabilities() override; virtual FramePacingConstraint getFramePacingConstraint() override; virtual void notifyOverlayUpdated(Overlay::OverlayType) override; + virtual bool isRenderThreadSupported() override; private: bool initializeDecoder(); diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp index 140464bd..8f872221 100644 --- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp +++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp @@ -61,6 +61,24 @@ Pacer::~Pacer() } } +void Pacer::renderOnMainThread() +{ + // Ignore this call for renderers that work on a dedicated render thread + if (m_RenderThread != nullptr) { + return; + } + + m_FrameQueueLock.lock(); + + if (!m_RenderQueue.isEmpty()) { + // Releases m_FrameQueueLock + renderLastFrameAndUnlock(); + } + else { + m_FrameQueueLock.unlock(); + } +} + int Pacer::renderThread(void* context) { Pacer* me = reinterpret_cast(context); @@ -87,34 +105,61 @@ int Pacer::renderThread(void* context) break; } - // Dequeue the most recent frame for rendering and free the others. + // Render the latest frame and discard the others // NB: m_FrameQueueLock still held here! - AVFrame* lastFrame = nullptr; - while (!me->m_RenderQueue.isEmpty()) { - if (lastFrame != nullptr) { - // Don't hold the frame queue lock across av_frame_free(), - // since it could need to talk to the GPU driver. This is safe - // because we're guaranteed that the queue will not shrink during - // this time (and so dequeue() below will always get something). - me->m_FrameQueueLock.unlock(); - av_frame_free(&lastFrame); - me->m_VideoStats->pacerDroppedFrames++; - me->m_FrameQueueLock.lock(); - } - - lastFrame = me->m_RenderQueue.dequeue(); - } - - // Release the frame queue lock before rendering - me->m_FrameQueueLock.unlock(); - - // Render and free the mot current frame - me->renderFrame(lastFrame); + me->renderLastFrameAndUnlock(); } return 0; } +void Pacer::enqueueFrameForRenderingAndUnlock(AVFrame *frame) +{ + dropFrameForEnqueue(m_RenderQueue); + m_RenderQueue.enqueue(frame); + + m_FrameQueueLock.unlock(); + + if (m_RenderThread != nullptr) { + m_RenderQueueNotEmpty.wakeOne(); + } + else { + SDL_Event event; + + // For main thread rendering, we'll push an event to trigger a callback + event.type = SDL_USEREVENT; + event.user.code = SDL_CODE_FRAME_READY; + SDL_PushEvent(&event); + } +} + +// Caller must hold m_FrameQueueLock +void Pacer::renderLastFrameAndUnlock() +{ + // Dequeue the most recent frame for rendering and free the others. + AVFrame* lastFrame = nullptr; + while (!m_RenderQueue.isEmpty()) { + if (lastFrame != nullptr) { + // Don't hold the frame queue lock across av_frame_free(), + // since it could need to talk to the GPU driver. This is safe + // because we're guaranteed that the queue will not shrink during + // this time (and so dequeue() below will always get something). + m_FrameQueueLock.unlock(); + av_frame_free(&lastFrame); + m_VideoStats->pacerDroppedFrames++; + m_FrameQueueLock.lock(); + } + + lastFrame = m_RenderQueue.dequeue(); + } + + // Release the frame queue lock before rendering + m_FrameQueueLock.unlock(); + + // Render and free the mot current frame + renderFrame(lastFrame); +} + // Called in an arbitrary thread by the IVsyncSource on V-sync // or an event synchronized with V-sync void Pacer::vsyncCallback(int timeUntilNextVsyncMillis) @@ -172,10 +217,7 @@ void Pacer::vsyncCallback(int timeUntilNextVsyncMillis) } // Place the first frame on the render queue - dropFrameForEnqueue(m_RenderQueue); - m_RenderQueue.enqueue(m_PacingQueue.dequeue()); - m_FrameQueueLock.unlock(); - m_RenderQueueNotEmpty.wakeOne(); + enqueueFrameForRenderingAndUnlock(m_PacingQueue.dequeue()); } bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing) @@ -211,7 +253,9 @@ bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing) m_DisplayFps, m_MaxVideoFps); } - m_RenderThread = SDL_CreateThread(Pacer::renderThread, "Pacer Render Thread", this); + if (m_VsyncRenderer->isRenderThreadSupported()) { + m_RenderThread = SDL_CreateThread(Pacer::renderThread, "Pacer Render Thread", this); + } return true; } @@ -283,16 +327,10 @@ void Pacer::submitFrame(AVFrame* frame) if (m_VsyncSource != nullptr) { dropFrameForEnqueue(m_PacingQueue); m_PacingQueue.enqueue(frame); - } - else { - dropFrameForEnqueue(m_RenderQueue); - m_RenderQueue.enqueue(frame); - } - m_FrameQueueLock.unlock(); - if (m_VsyncSource != nullptr) { + m_FrameQueueLock.unlock(); m_PacingQueueNotEmpty.wakeOne(); } else { - m_RenderQueueNotEmpty.wakeOne(); + enqueueFrameForRenderingAndUnlock(frame); } } diff --git a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h index 5fd4e720..6702dbae 100644 --- a/app/streaming/video/ffmpeg-renderers/pacer/pacer.h +++ b/app/streaming/video/ffmpeg-renderers/pacer/pacer.h @@ -26,9 +26,15 @@ public: void vsyncCallback(int timeUntilNextVsyncMillis); + void renderOnMainThread(); + private: static int renderThread(void* context); + void enqueueFrameForRenderingAndUnlock(AVFrame* frame); + + void renderLastFrameAndUnlock(); + void renderFrame(AVFrame* frame); void dropFrameForEnqueue(QQueue& queue); diff --git a/app/streaming/video/ffmpeg-renderers/renderer.h b/app/streaming/video/ffmpeg-renderers/renderer.h index 96f0d78d..19b543af 100644 --- a/app/streaming/video/ffmpeg-renderers/renderer.h +++ b/app/streaming/video/ffmpeg-renderers/renderer.h @@ -27,6 +27,7 @@ public: virtual bool needsTestFrame() = 0; virtual int getDecoderCapabilities() = 0; virtual FramePacingConstraint getFramePacingConstraint() = 0; + virtual bool isRenderThreadSupported() = 0; // IOverlayRenderer virtual void notifyOverlayUpdated(Overlay::OverlayType) override { diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp index af6e2063..c80e1280 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.cpp +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.cpp @@ -62,7 +62,7 @@ bool SdlRenderer::prepareDecoderContext(AVCodecContext*) /* Nothing to do */ SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, - "Using SDL software renderer"); + "Using SDL renderer"); return true; } @@ -126,6 +126,24 @@ void SdlRenderer::notifyOverlayUpdated(Overlay::OverlayType type) } } +bool SdlRenderer::isRenderThreadSupported() +{ + SDL_RendererInfo info; + SDL_GetRendererInfo(m_Renderer, &info); + + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "SDL renderer backend: %s", + info.name); + + if (info.name != QString("direct3d")) { + SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION, + "SDL renderer backend requires main thread rendering"); + return false; + } + + return true; +} + bool SdlRenderer::initialize(SDL_Window* window, int, int width, diff --git a/app/streaming/video/ffmpeg-renderers/sdlvid.h b/app/streaming/video/ffmpeg-renderers/sdlvid.h index fe913f77..baac6977 100644 --- a/app/streaming/video/ffmpeg-renderers/sdlvid.h +++ b/app/streaming/video/ffmpeg-renderers/sdlvid.h @@ -20,6 +20,7 @@ public: virtual int getDecoderCapabilities() override; virtual FramePacingConstraint getFramePacingConstraint() override; virtual void notifyOverlayUpdated(Overlay::OverlayType) override; + virtual bool isRenderThreadSupported() override; private: void renderOverlay(Overlay::OverlayType type); diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.cpp b/app/streaming/video/ffmpeg-renderers/vaapi.cpp index d826e187..f4c9d06b 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.cpp +++ b/app/streaming/video/ffmpeg-renderers/vaapi.cpp @@ -182,6 +182,12 @@ IFFmpegRenderer::FramePacingConstraint VAAPIRenderer::getFramePacingConstraint() return PACING_ANY; } +bool VAAPIRenderer::isRenderThreadSupported() +{ + // renderFrame() may be called outside of the main thread + return true; +} + void VAAPIRenderer::renderFrame(AVFrame* frame) { diff --git a/app/streaming/video/ffmpeg-renderers/vaapi.h b/app/streaming/video/ffmpeg-renderers/vaapi.h index 2c536065..91eb98ee 100644 --- a/app/streaming/video/ffmpeg-renderers/vaapi.h +++ b/app/streaming/video/ffmpeg-renderers/vaapi.h @@ -41,6 +41,7 @@ public: virtual bool needsTestFrame(); virtual int getDecoderCapabilities(); virtual FramePacingConstraint getFramePacingConstraint(); + virtual bool isRenderThreadSupported(); private: int m_WindowSystem; diff --git a/app/streaming/video/ffmpeg-renderers/vdpau.cpp b/app/streaming/video/ffmpeg-renderers/vdpau.cpp index 229ca624..d6765ff8 100644 --- a/app/streaming/video/ffmpeg-renderers/vdpau.cpp +++ b/app/streaming/video/ffmpeg-renderers/vdpau.cpp @@ -251,6 +251,12 @@ IFFmpegRenderer::FramePacingConstraint VDPAURenderer::getFramePacingConstraint() return PACING_ANY; } +bool VDPAURenderer::isRenderThreadSupported() +{ + // renderFrame() may be called outside of the main thread + return true; +} + void VDPAURenderer::renderFrame(AVFrame* frame) { VdpStatus status; diff --git a/app/streaming/video/ffmpeg-renderers/vdpau.h b/app/streaming/video/ffmpeg-renderers/vdpau.h index fcace5d0..0620a3a8 100644 --- a/app/streaming/video/ffmpeg-renderers/vdpau.h +++ b/app/streaming/video/ffmpeg-renderers/vdpau.h @@ -24,6 +24,7 @@ public: virtual bool needsTestFrame(); virtual int getDecoderCapabilities(); virtual FramePacingConstraint getFramePacingConstraint(); + virtual bool isRenderThreadSupported(); private: uint32_t m_VideoWidth, m_VideoHeight; diff --git a/app/streaming/video/ffmpeg-renderers/vt.mm b/app/streaming/video/ffmpeg-renderers/vt.mm index 14af4c8e..690945c3 100644 --- a/app/streaming/video/ffmpeg-renderers/vt.mm +++ b/app/streaming/video/ffmpeg-renderers/vt.mm @@ -296,6 +296,12 @@ public: return PACING_FORCE_ON; } + virtual bool isRenderThreadSupported() override + { + // renderFrame() may be called outside of the main thread + return true; + } + private: AVBufferRef* m_HwContext; AVSampleBufferDisplayLayer* m_DisplayLayer; diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp index 335b0b95..fc9606b8 100644 --- a/app/streaming/video/ffmpeg.cpp +++ b/app/streaming/video/ffmpeg.cpp @@ -630,14 +630,8 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du) return DR_OK; } -// Called on main thread -void FFmpegVideoDecoder::renderFrame(SDL_UserEvent*) +void FFmpegVideoDecoder::renderFrameOnMainThread() { - SDL_assert(false); + m_Pacer->renderOnMainThread(); } -// Called on main thread -void FFmpegVideoDecoder::dropFrame(SDL_UserEvent*) -{ - SDL_assert(false); -} diff --git a/app/streaming/video/ffmpeg.h b/app/streaming/video/ffmpeg.h index 2792211d..9127a3be 100644 --- a/app/streaming/video/ffmpeg.h +++ b/app/streaming/video/ffmpeg.h @@ -23,8 +23,7 @@ public: virtual bool isHardwareAccelerated() override; virtual int getDecoderCapabilities() override; virtual int submitDecodeUnit(PDECODE_UNIT du) override; - virtual void renderFrame(SDL_UserEvent* event) override; - virtual void dropFrame(SDL_UserEvent* event) override; + virtual void renderFrameOnMainThread() override; virtual IFFmpegRenderer* getRenderer(); diff --git a/app/streaming/video/slvid.h b/app/streaming/video/slvid.h index a50fd5a3..1af92edf 100644 --- a/app/streaming/video/slvid.h +++ b/app/streaming/video/slvid.h @@ -22,8 +22,7 @@ public: virtual int submitDecodeUnit(PDECODE_UNIT du); // Unused since rendering is done directly from the decode thread - virtual void renderFrame(SDL_UserEvent*) {} - virtual void dropFrame(SDL_UserEvent*) {} + virtual void renderFrameOnMainThread() {} private: CSLVideoContext* m_VideoContext;