moonlight-qt/app/streaming/video/ffmpeg-renderers/pacer/pacer.cpp

#include "pacer.h"
#include "streaming/streamutils.h"

#include "nullthreadedvsyncsource.h"

#ifdef Q_OS_DARWIN
#include "displaylinkvsyncsource.h"
#endif

#ifdef Q_OS_WIN32
#include "dxvsyncsource.h"
#endif

#define FRAME_HISTORY_ENTRIES 8
#define RENDER_TIME_HISTORY_ENTRIES 20

// We may be woken up slightly late so don't go all the way
// up to the next V-sync since we may accidentally step into
// the next V-sync period. It also takes some amount of time
// to do the render itself, so we can't render right before
// V-sync happens.
#define TIMER_SLACK_MS 3

Pacer::Pacer(IFFmpegRenderer* renderer, PVIDEO_STATS videoStats) :
    m_RenderThread(nullptr),
    m_Stopping(false),
    m_VsyncSource(nullptr),
    m_VsyncRenderer(renderer),
    m_MaxVideoFps(0),
    m_DisplayFps(0),
    m_VideoStats(videoStats)
{

}

Pacer::~Pacer()
{
    // Stop V-sync callbacks
    delete m_VsyncSource;
    m_VsyncSource = nullptr;

    // Stop the render thread
    m_Stopping = true;
    if (m_RenderThread != nullptr) {
        m_FrameQueueNotEmpty.wakeAll();
        SDL_WaitThread(m_RenderThread, nullptr);
    }

    // Delete any remaining unconsumed frames
    while (!m_FrameQueue.isEmpty()) {
        AVFrame* frame = m_FrameQueue.dequeue();
        av_frame_free(&frame);
    }
}

int Pacer::renderThread(void* context)
{
    Pacer* me = reinterpret_cast<Pacer*>(context);

    while (!me->m_Stopping) {
        // Acquire the frame queue lock to protect the queue and
        // the not empty condition
        me->m_FrameQueueLock.lock();

        // Wait for a frame to be ready to render
        while (!me->m_Stopping && me->m_FrameQueue.isEmpty()) {
            me->m_FrameQueueNotEmpty.wait(&me->m_FrameQueueLock);
        }

        if (me->m_Stopping) {
            // Exit this thread
            me->m_FrameQueueLock.unlock();
            break;
        }

        // Dequeue the most recent frame for rendering and free the others.
        // NB: m_FrameQueueLock still held here!
        AVFrame* lastFrame = nullptr;
        while (!me->m_FrameQueue.isEmpty()) {
            if (lastFrame != nullptr) {
                // Don't hold the frame queue lock across av_frame_free(),
                // since it could need to talk to the GPU driver. This is safe
                // because we're guaranteed that the queue will not shrink during
                // this time (and so dequeue() below will always get something).
                me->m_FrameQueueLock.unlock();
                av_frame_free(&lastFrame);
                me->m_VideoStats->pacerDroppedFrames++;
                me->m_FrameQueueLock.lock();
            }

            lastFrame = me->m_FrameQueue.dequeue();
        }

        // Release the frame queue lock before rendering
        me->m_FrameQueueLock.unlock();

        // Render and free the mot current frame
        me->renderFrame(lastFrame);
    }

    return 0;
}

// Called in an arbitrary thread by the IVsyncSource on V-sync
// or an event synchronized with V-sync
void Pacer::vsyncCallback(int timeUntilNextVsyncMillis)
{
    // Make sure initialize() has been called
    SDL_assert(m_MaxVideoFps != 0);

    SDL_assert(timeUntilNextVsyncMillis >= TIMER_SLACK_MS);

    m_FrameQueueLock.lock();

    // If the queue length history entries are large, be strict
    // about dropping excess frames.
    int frameDropTarget = 1;

    // If we may get more frames per second than we can display, use
    // frame history to drop frames only if consistently above the
    // one queued frame mark.
    if (m_MaxVideoFps >= m_DisplayFps) {
        for (int queueHistoryEntry : m_FrameQueueHistory) {
            if (queueHistoryEntry <= 1) {
                // Be lenient as long as the queue length
                // resolves before the end of frame history
                frameDropTarget = 3;
                break;
            }
        }

        if (m_FrameQueueHistory.count() == FRAME_HISTORY_ENTRIES) {
            m_FrameQueueHistory.dequeue();
        }

        m_FrameQueueHistory.enqueue(m_FrameQueue.count());
    }

    // Catch up if we're several frames ahead
    while (m_FrameQueue.count() > frameDropTarget) {
        AVFrame* frame = m_FrameQueue.dequeue();

        // Drop the lock while we call av_frame_free()
        m_FrameQueueLock.unlock();
        m_VideoStats->pacerDroppedFrames++;
        av_frame_free(&frame);
        m_FrameQueueLock.lock();
    }

    if (m_FrameQueue.isEmpty()) {
        // Wait for a frame to arrive or our V-sync timeout to expire
        if (!m_FrameQueueNotEmpty.wait(&m_FrameQueueLock, timeUntilNextVsyncMillis - TIMER_SLACK_MS)) {
            // Wait timed out - unlock and bail
            m_FrameQueueLock.unlock();

            // Nothing to render at this time. This counts as a drop.
            addRenderTimeToHistory(0);
            return;
        }
    }

    // Grab the first frame
    AVFrame* frame = m_FrameQueue.dequeue();
    m_FrameQueueLock.unlock();

    renderFrame(frame);
}

bool Pacer::initialize(SDL_Window* window, int maxVideoFps, bool enablePacing)
{
    m_MaxVideoFps = maxVideoFps;
    m_DisplayFps = StreamUtils::getDisplayRefreshRate(window);

    if (enablePacing) {
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                    "Frame pacing active: target %d Hz with %d FPS stream",
                    m_DisplayFps, m_MaxVideoFps);

    #if defined(Q_OS_DARWIN)
        m_VsyncSource = DisplayLinkVsyncSourceFactory::createVsyncSource(this);
    #elif defined(Q_OS_WIN32)
        m_VsyncSource = new DxVsyncSource(this);
    #else
        // Platforms without a VsyncSource will just render frames
        // immediately like they used to.
    #endif

        if (m_VsyncSource != nullptr && !m_VsyncSource->initialize(window, m_DisplayFps)) {
            return false;
        }
    }
    else {
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                    "Frame pacing disabled: target %d Hz with %d FPS stream",
                    m_DisplayFps, m_MaxVideoFps);
    }

    if (m_VsyncSource == nullptr) {
        m_RenderThread = SDL_CreateThread(Pacer::renderThread, "Pacer Render Thread", this);
    }

    return true;
}

void Pacer::addRenderTimeToHistory(int renderTime)
{
    if (m_RenderTimeHistory.count() == RENDER_TIME_HISTORY_ENTRIES) {
        m_RenderTimeHistory.dequeue();
    }

    m_RenderTimeHistory.enqueue(renderTime);
}

void Pacer::renderFrame(AVFrame* frame)
{
    bool dropFrame = !m_RenderTimeHistory.isEmpty();

    // If rendering consistently takes longer than an entire V-sync interval,
    // there must have been frames pending in the pipeline that are blocking us.
    // Drop our this frame to clear the backed up display pipeline.
    for (int renderTime : m_RenderTimeHistory) {
        if (renderTime < 1000 / m_DisplayFps) {
            dropFrame = false;
            break;
        }
    }

    // Drop this frame if we believe we've run into V-sync waits
    if (dropFrame) {
        SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                    "Render time (%d ms) consistently exceeded V-sync period; dropping frame to reduce latency",
                    m_RenderTimeHistory[0]);

        // Add 0 render time to history to account for the drop
        addRenderTimeToHistory(0);

        m_VideoStats->pacerDroppedFrames++;
        av_frame_free(&frame);
        return;
    }

    // Count time spent in Pacer's queues
    Uint32 beforeRender = SDL_GetTicks();
    m_VideoStats->totalPacerTime += beforeRender - frame->pts;

    // Render it
    m_VsyncRenderer->renderFrameAtVsync(frame);
    Uint32 afterRender = SDL_GetTicks();

    addRenderTimeToHistory(afterRender - beforeRender);

    m_VideoStats->totalRenderTime += afterRender - beforeRender;
    m_VideoStats->renderedFrames++;
    av_frame_free(&frame);
}

void Pacer::submitFrame(AVFrame* frame)
{
    // Make sure initialize() has been called
    SDL_assert(m_MaxVideoFps != 0);

    // Queue the frame and possibly wake up the render thread
    m_FrameQueueLock.lock();
    m_FrameQueue.enqueue(frame);
    m_FrameQueueLock.unlock();
    m_FrameQueueNotEmpty.wakeOne();
}