Fixup H.264 SPS for VideoToolbox compatibility. Fixes #98

.gitmodules

@@ -10,3 +10,6 @@
 [submodule "soundio/libsoundio"]
 	path = soundio/libsoundio
 	url = https://github.com/andrewrk/libsoundio.git
+[submodule "h264bitstream/h264bitstream"]
+	path = h264bitstream/h264bitstream
+	url = https://github.com/aizvorski/h264bitstream.git

app/app.pro

@@ -258,6 +258,13 @@ else:unix: LIBS += -L$$OUT_PWD/../soundio/ -lsoundio
 INCLUDEPATH += $$PWD/../soundio/libsoundio
 DEPENDPATH += $$PWD/../soundio/libsoundio
 
+win32:CONFIG(release, debug|release): LIBS += -L$$OUT_PWD/../h264bitstream/release/ -lh264bitstream
+else:win32:CONFIG(debug, debug|release): LIBS += -L$$OUT_PWD/../h264bitstream/debug/ -lh264bitstream
+else:unix: LIBS += -L$$OUT_PWD/../h264bitstream/ -lh264bitstream
+
+INCLUDEPATH += $$PWD/../h264bitstream/h264bitstream
+DEPENDPATH += $$PWD/../h264bitstream/h264bitstream
+
 unix:!macx: {
     isEmpty(PREFIX) {
         PREFIX = /usr/local

app/streaming/session.cpp

@@ -350,6 +350,20 @@ void Session::initialize()
                                           m_StreamConfig.width,
                                           m_StreamConfig.height,
                                           m_StreamConfig.fps);
+        {
+            // Prior to GFE 3.11, GFE did not allow us to constrain
+            // the number of reference frames, so we have to fixup the SPS
+            // to allow decoding via VideoToolbox on macOS. Since we don't
+            // have fixup code for HEVC, just avoid it if GFE is too old.
+            QVector<int> gfeVersion = NvHTTP::parseQuad(m_Computer->gfeVersion);
+            if (gfeVersion.isEmpty() || // Very old versions don't have GfeVersion at all
+                    gfeVersion[0] < 3 ||
+                    (gfeVersion[0] == 3 && gfeVersion[1] < 11)) {
+                SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                            "Disabling HEVC on macOS due to old GFE version");
+                m_StreamConfig.supportsHevc = false;
+            }
+        }
         m_StreamConfig.enableHdr = false;
         break;
     case StreamingPreferences::VCC_FORCE_H264:

app/streaming/video/ffmpeg.cpp

@@ -1,6 +1,7 @@
 #include <Limelight.h>
 #include "ffmpeg.h"
 #include "streaming/streamutils.h"
+#include <h264_stream.h>
 
 #ifdef Q_OS_WIN32
 #include "ffmpeg-renderers/dxva2.h"
@@ -21,6 +22,8 @@
 // This is gross but it allows us to use sizeof()
 #include "ffmpeg_videosamples.cpp"
 
+#define MAX_SPS_EXTRA_SIZE 16
+
 #define FAILED_DECODES_RESET_THRESHOLD 20
 
 bool FFmpegVideoDecoder::isHardwareAccelerated()
@@ -62,7 +65,8 @@ FFmpegVideoDecoder::FFmpegVideoDecoder()
       m_ConsecutiveFailedDecodes(0),
       m_Pacer(nullptr),
       m_LastFrameNumber(0),
-      m_StreamFps(0)
+      m_StreamFps(0),
+      m_NeedsSpsFixup(false)
 {
     av_init_packet(&m_Pkt);
     SDL_AtomicSet(&m_QueuedFrames, 0);
@@ -225,6 +229,17 @@ bool FFmpegVideoDecoder::completeInitialization(AVCodec* decoder, SDL_Window* wi
             return false;
         }
     }
+    else {
+        if ((videoFormat & VIDEO_FORMAT_MASK_H264) &&
+                !(m_Renderer->getDecoderCapabilities() & CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC)) {
+            SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
+                        "Using H.264 SPS fixup");
+            m_NeedsSpsFixup = true;
+        }
+        else {
+            m_NeedsSpsFixup = false;
+        }
+    }
 
 #ifdef QT_DEBUG
     // Restore default log level before streaming
@@ -424,6 +439,49 @@ bool FFmpegVideoDecoder::initialize(
     }
 }
 
+void FFmpegVideoDecoder::writeBuffer(PLENTRY entry, int& offset)
+{
+    if (m_NeedsSpsFixup && entry->bufferType == BUFFER_TYPE_SPS) {
+        const char naluHeader[] = {0x00, 0x00, 0x00, 0x01};
+        h264_stream_t* stream = h264_new();
+        int nalStart, nalEnd;
+
+        // Read the old NALU
+        find_nal_unit((uint8_t*)entry->data, entry->length, &nalStart, &nalEnd);
+        read_nal_unit(stream,
+                      (unsigned char *)&entry->data[nalStart],
+                      nalEnd - nalStart);
+
+        SDL_assert(nalStart == sizeof(naluHeader));
+        SDL_assert(nalEnd == entry->length);
+
+        // Fixup the SPS to what OS X needs to use hardware acceleration
+        stream->sps->num_ref_frames = 1;
+        stream->sps->vui.max_dec_frame_buffering = 1;
+
+        int initialOffset = offset;
+
+        // Copy the modified NALU data. This assumes a 3 byte prefix and
+        // begins writing from the 2nd byte, so we must write the data
+        // first, then go back and write the Annex B prefix.
+        offset += write_nal_unit(stream, (uint8_t*)&m_DecodeBuffer.data()[initialOffset + 3],
+                                 MAX_SPS_EXTRA_SIZE + entry->length - sizeof(naluHeader));
+
+        // Copy the NALU prefix over from the original SPS
+        memcpy(&m_DecodeBuffer.data()[initialOffset], naluHeader, sizeof(naluHeader));
+        offset += sizeof(naluHeader);
+
+        h264_free(stream);
+    }
+    else {
+        // Write the buffer as-is
+        memcpy(&m_DecodeBuffer.data()[offset],
+               entry->data,
+               entry->length);
+        offset += entry->length;
+    }
+}
+
 int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
 {
     PLENTRY entry = du->bufferList;
@@ -461,23 +519,23 @@ int FFmpegVideoDecoder::submitDecodeUnit(PDECODE_UNIT du)
         m_LastFrameNumber = du->frameNumber;
     }
 
-    if (du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE > m_DecodeBuffer.length()) {
-        m_DecodeBuffer = QByteArray(du->fullLength + AV_INPUT_BUFFER_PADDING_SIZE, 0);
+    int requiredBufferSize = du->fullLength;
+    if (du->frameType == FRAME_TYPE_IDR) {
+        // Add some extra space in case we need to do an SPS fixup
+        requiredBufferSize += MAX_SPS_EXTRA_SIZE;
+    }
+    if (requiredBufferSize + AV_INPUT_BUFFER_PADDING_SIZE > m_DecodeBuffer.length()) {
+        m_DecodeBuffer = QByteArray(requiredBufferSize + AV_INPUT_BUFFER_PADDING_SIZE, 0);
     }
 
     int offset = 0;
     while (entry != nullptr) {
-        memcpy(&m_DecodeBuffer.data()[offset],
-               entry->data,
-               entry->length);
-        offset += entry->length;
+        writeBuffer(entry, offset);
         entry = entry->next;
     }
 
-    SDL_assert(offset == du->fullLength);
-
     m_Pkt.data = reinterpret_cast<uint8_t*>(m_DecodeBuffer.data());
-    m_Pkt.size = du->fullLength;
+    m_Pkt.size = offset;
 
     m_ActiveWndVideoStats.totalReassemblyTime += LiGetMillis() - du->receiveTimeMs;
 

app/streaming/video/ffmpeg.h

@@ -40,6 +40,8 @@ private:
 
     void reset();
 
+    void writeBuffer(PLENTRY entry, int& offset);
+
     static
     enum AVPixelFormat ffGetFormat(AVCodecContext* context,
                                    const enum AVPixelFormat* pixFmts);
@@ -57,6 +59,7 @@ private:
     VIDEO_STATS m_GlobalVideoStats;
     int m_LastFrameNumber;
     int m_StreamFps;
+    bool m_NeedsSpsFixup;
 
     static const uint8_t k_H264TestFrame[];
     static const uint8_t k_HEVCTestFrame[];

h264bitstream/h264bitstream

@@ -0,0 +1 @@
+Subproject commit 34f3c58afa3c47b6cf0a49308a68cbf89c5e0bff

h264bitstream/h264bitstream.pro

@@ -0,0 +1,41 @@
+#-------------------------------------------------
+#
+# Project created by QtCreator 2018-10-12T15:50:59
+#
+#-------------------------------------------------
+
+
+QT       -= core gui
+
+TARGET = h264bitstream
+TEMPLATE = lib
+
+# Support debug and release builds from command line for CI
+CONFIG += debug_and_release
+
+# Ensure symbols are always generated
+CONFIG += force_debug_info
+
+# Build a static library
+CONFIG += staticlib
+
+# Disable warnings
+CONFIG += warn_off
+
+SRC_DIR = $$PWD/h264bitstream
+
+SOURCES += \
+    $$SRC_DIR/h264_avcc.c           \
+    $$SRC_DIR/h264_nal.c            \
+    $$SRC_DIR/h264_sei.c            \
+    $$SRC_DIR/h264_slice_data.c     \
+    $$SRC_DIR/h264_stream.c
+
+HEADERS += \
+    $$SRC_DIR/bs.h              \
+    $$SRC_DIR/h264_avcc.h       \
+    $$SRC_DIR/h264_sei.h        \
+    $$SRC_DIR/h264_slice_data.h \
+    $$SRC_DIR/h264_stream.h
+
+INCLUDEPATH += $$INC_DIR

moonlight-qt.pro

@@ -3,10 +3,11 @@ SUBDIRS = \
     moonlight-common-c \
     qmdnsengine \
     app \
-    soundio
+    soundio \
+    h264bitstream
 
 # Build the dependencies in parallel before the final app
-app.depends = qmdnsengine moonlight-common-c soundio
+app.depends = qmdnsengine moonlight-common-c soundio h264bitstream
 
 # Support debug and release builds from command line for CI
 CONFIG += debug_and_release