Use EGL fences to reduce video latency

app/streaming/video/ffmpeg-renderers/eglvid.cpp

@@ -76,11 +76,16 @@ EGLRenderer::EGLRenderer(IFFmpegRenderer *backendRenderer)
         m_Backend(backendRenderer),
         m_VAO(0),
         m_BlockingSwapBuffers(false),
+        m_LastRenderSync(EGL_NO_SYNC),
         m_LastFrame(av_frame_alloc()),
         m_glEGLImageTargetTexture2DOES(nullptr),
         m_glGenVertexArraysOES(nullptr),
         m_glBindVertexArrayOES(nullptr),
         m_glDeleteVertexArraysOES(nullptr),
+        m_eglCreateSync(nullptr),
+        m_eglCreateSyncKHR(nullptr),
+        m_eglDestroySync(nullptr),
+        m_eglClientWaitSync(nullptr),
         m_GlesMajorVersion(0),
         m_GlesMinorVersion(0),
         m_HasExtUnpackSubimage(false),
@@ -100,6 +105,10 @@ EGLRenderer::~EGLRenderer()
     if (m_Context) {
         // Reattach the GL context to the main thread for destruction
         SDL_GL_MakeCurrent(m_Window, m_Context);
+        if (m_LastRenderSync != EGL_NO_SYNC) {
+            SDL_assert(m_eglDestroySync != nullptr);
+            m_eglDestroySync(m_EGLDisplay, m_LastRenderSync);
+        }
         if (m_ShaderProgram) {
             glDeleteProgram(m_ShaderProgram);
         }
@@ -615,6 +624,30 @@ bool EGLRenderer::initialize(PDECODER_PARAMETERS params)
         return false;
     }
 
+    // EGL_KHR_fence_sync is an extension for EGL 1.1+
+    if (eglExtensions.isSupported("EGL_KHR_fence_sync")) {
+        // eglCreateSyncKHR() has a slightly different prototype to eglCreateSync()
+        m_eglCreateSyncKHR = (typeof(m_eglCreateSyncKHR))eglGetProcAddress("eglCreateSyncKHR");
+        m_eglDestroySync = (typeof(m_eglDestroySync))eglGetProcAddress("eglDestroySyncKHR");
+        m_eglClientWaitSync = (typeof(m_eglClientWaitSync))eglGetProcAddress("eglClientWaitSyncKHR");
+    }
+    else {
+        // EGL 1.5 introduced sync support to the core specification
+        m_eglCreateSync = (typeof(m_eglCreateSync))eglGetProcAddress("eglCreateSync");
+        m_eglDestroySync = (typeof(m_eglDestroySync))eglGetProcAddress("eglDestroySync");
+        m_eglClientWaitSync = (typeof(m_eglClientWaitSync))eglGetProcAddress("eglClientWaitSync");
+    }
+
+    if (!(m_eglCreateSync || m_eglCreateSyncKHR) || !m_eglDestroySync || !m_eglClientWaitSync) {
+        EGL_LOG(Warn, "Failed to find sync functions");
+
+        // Sub-optimal, but not fatal
+        m_eglCreateSync = nullptr;
+        m_eglCreateSyncKHR = nullptr;
+        m_eglDestroySync = nullptr;
+        m_eglClientWaitSync = nullptr;
+    }
+
     /* Compute the video region size in order to keep the aspect ratio of the
      * video stream.
      */
@@ -820,6 +853,18 @@ void EGLRenderer::cleanupRenderContext()
     SDL_GL_MakeCurrent(m_Window, nullptr);
 }
 
+void EGLRenderer::waitToRender()
+{
+    // Ensure our GL context is active on this thread
+    // See comment in renderFrame() for more details.
+    SDL_GL_MakeCurrent(m_Window, m_Context);
+
+    if (m_LastRenderSync != 0) {
+        SDL_assert(m_eglClientWaitSync != nullptr);
+        m_eglClientWaitSync(m_EGLDisplay, m_LastRenderSync, EGL_SYNC_FLUSH_COMMANDS_BIT, EGL_FOREVER);
+    }
+}
+
 void EGLRenderer::renderFrame(AVFrame* frame)
 {
     EGLImage imgs[EGL_MAX_PLANES];
@@ -878,15 +923,34 @@ void EGLRenderer::renderFrame(AVFrame* frame)
     SDL_GL_SwapWindow(m_Window);
 
     if (m_BlockingSwapBuffers) {
-        // This glClear() forces us to block until the buffer swap is
+        // If we this EGL implementation supports fences, use those to delay
-        // complete to continue rendering. Mesa won't actually wait
+        // rendering the next frame until this one is completed.
-        // for the swap with just glFinish() alone. Waiting here keeps us
+        if (m_eglClientWaitSync != nullptr) {
-        // in lock step with the display refresh rate. If we don't wait
+            // Delete the sync object from last render
-        // here, we'll stall on the first GL call next frame. Doing the
+            if (m_LastRenderSync != EGL_NO_SYNC) {
-        // wait here instead allows more time for a newer frame to arrive
+                m_eglDestroySync(m_EGLDisplay, m_LastRenderSync);
-        // for next renderFrame() call.
+            }
-        glClear(GL_COLOR_BUFFER_BIT);
+
-        glFinish();
+            // Create a new sync object that will be signalled when the buffer swap is completed
+            if (m_eglCreateSync != nullptr) {
+                m_LastRenderSync = m_eglCreateSync(m_EGLDisplay, EGL_SYNC_FENCE, nullptr);
+            }
+            else {
+                SDL_assert(m_eglCreateSyncKHR != nullptr);
+                m_LastRenderSync = m_eglCreateSyncKHR(m_EGLDisplay, EGL_SYNC_FENCE, nullptr);
+            }
+        }
+        else {
+            // This glClear() forces us to block until the buffer swap is
+            // complete to continue rendering. Mesa won't actually wait
+            // for the swap with just glFinish() alone. Waiting here keeps us
+            // in lock step with the display refresh rate. If we don't wait
+            // here, we'll stall on the first GL call next frame. Doing the
+            // wait here instead allows more time for a newer frame to arrive
+            // for next renderFrame() call.
+            glClear(GL_COLOR_BUFFER_BIT);
+            glFinish();
+        }
     }
 
     m_Backend->freeEGLImages(m_EGLDisplay, imgs);

app/streaming/video/ffmpeg-renderers/eglvid.h

@@ -12,6 +12,7 @@ public:
     virtual bool initialize(PDECODER_PARAMETERS params) override;
     virtual bool prepareDecoderContext(AVCodecContext* context, AVDictionary** options) override;
     virtual void cleanupRenderContext() override;
+    virtual void waitToRender() override;
     virtual void renderFrame(AVFrame* frame) override;
     virtual bool testRenderFrame(AVFrame* frame) override;
     virtual void notifyOverlayUpdated(Overlay::OverlayType) override;
@@ -45,11 +46,16 @@ private:
     IFFmpegRenderer *m_Backend;
     unsigned int m_VAO;
     bool m_BlockingSwapBuffers;
+    EGLSync m_LastRenderSync;
     AVFrame* m_LastFrame;
     PFNGLEGLIMAGETARGETTEXTURE2DOESPROC m_glEGLImageTargetTexture2DOES;
     PFNGLGENVERTEXARRAYSOESPROC m_glGenVertexArraysOES;
     PFNGLBINDVERTEXARRAYOESPROC m_glBindVertexArrayOES;
     PFNGLDELETEVERTEXARRAYSOESPROC m_glDeleteVertexArraysOES;
+    PFNEGLCREATESYNCPROC m_eglCreateSync;
+    PFNEGLCREATESYNCKHRPROC m_eglCreateSyncKHR;
+    PFNEGLDESTROYSYNCPROC m_eglDestroySync;
+    PFNEGLCLIENTWAITSYNCPROC m_eglClientWaitSync;
     int m_GlesMajorVersion;
     int m_GlesMinorVersion;
     bool m_HasExtUnpackSubimage;

app/streaming/video/ffmpeg-renderers/renderer.h

@@ -21,9 +21,19 @@ extern "C" {
 #ifndef EGL_VERSION_1_5
 typedef intptr_t EGLAttrib;
 typedef void *EGLImage;
+
+typedef void *EGLSync;
+#define EGL_NO_SYNC                       ((EGLSync)0)
+#define EGL_SYNC_FENCE                    0x30F9
+#define EGL_FOREVER                       0xFFFFFFFFFFFFFFFFull
+#define EGL_SYNC_FLUSH_COMMANDS_BIT       0x0001
 #endif
 
 #if !defined(EGL_VERSION_1_5) || !defined(EGL_EGL_PROTOTYPES)
+typedef EGLSync (EGLAPIENTRYP PFNEGLCREATESYNCPROC) (EGLDisplay dpy, EGLenum type, const EGLAttrib *attrib_list);
+typedef EGLBoolean (EGLAPIENTRYP PFNEGLDESTROYSYNCPROC) (EGLDisplay dpy, EGLSync sync);
+typedef EGLint (EGLAPIENTRYP PFNEGLCLIENTWAITSYNCPROC) (EGLDisplay dpy, EGLSync sync, EGLint flags, EGLTime timeout);
+
 typedef EGLImage (EGLAPIENTRYP PFNEGLCREATEIMAGEPROC) (EGLDisplay dpy, EGLContext ctx, EGLenum target, EGLClientBuffer buffer, const EGLAttrib *attrib_list);
 typedef EGLBoolean (EGLAPIENTRYP PFNEGLDESTROYIMAGEPROC) (EGLDisplay dpy, EGLImage image);
 typedef EGLDisplay (EGLAPIENTRYP PFNEGLGETPLATFORMDISPLAYPROC) (EGLenum platform, void *native_display, const EGLAttrib *attrib_list);
@@ -40,6 +50,10 @@ typedef EGLBoolean (EGLAPIENTRYP PFNEGLDESTROYIMAGEKHRPROC) (EGLDisplay dpy, EGL
 typedef EGLDisplay (EGLAPIENTRYP PFNEGLGETPLATFORMDISPLAYEXTPROC) (EGLenum platform, void *native_display, const EGLint *attrib_list);
 #endif
 
+#if !defined(EGL_KHR_fence_sync) || !defined(EGL_EGLEXT_PROTOTYPES)
+typedef EGLSyncKHR (EGLAPIENTRYP PFNEGLCREATESYNCKHRPROC) (EGLDisplay dpy, EGLenum type, const EGLint *attrib_list);
+#endif
+
 #ifndef EGL_EXT_image_dma_buf_import
 #define EGL_LINUX_DMA_BUF_EXT             0x3270
 #define EGL_LINUX_DRM_FOURCC_EXT          0x3271