From a81c6a1c5ec9715ddcd858aa02f9d8ae0ecd0eaf Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Mon, 3 Jul 2023 00:45:36 -0500 Subject: [PATCH] Initial work on AV1 support --- app/backend/nvhttp.cpp | 2 +- app/streaming/session.cpp | 73 ++++++++++--------- .../video/ffmpeg-renderers/d3d11va.cpp | 30 ++++++++ .../video/ffmpeg-renderers/dxva2.cpp | 9 ++- app/streaming/video/ffmpeg.cpp | 48 +++++++++++- moonlight-common-c/moonlight-common-c | 2 +- 6 files changed, 127 insertions(+), 37 deletions(-) diff --git a/app/backend/nvhttp.cpp b/app/backend/nvhttp.cpp index 258f6593..04b829dc 100644 --- a/app/backend/nvhttp.cpp +++ b/app/backend/nvhttp.cpp @@ -216,7 +216,7 @@ NvHTTP::startApp(QString verb, "&additionalStates=1&sops="+QString::number(sops ? 1 : 0)+ "&rikey="+QByteArray(streamConfig->remoteInputAesKey, sizeof(streamConfig->remoteInputAesKey)).toHex()+ "&rikeyid="+QString::number(riKeyId)+ - (streamConfig->enableHdr ? + ((streamConfig->supportedVideoFormats & VIDEO_FORMAT_MASK_10BIT) ? "&hdrMode=1&clientHdrCapVersion=0&clientHdrCapSupportedFlagsInUint32=0&clientHdrCapMetaDataId=NV_STATIC_METADATA_TYPE_1&clientHdrCapDisplayData=0x0x0x0x0x0x0x0x0x0x0" : "")+ "&localAudioPlayMode="+QString::number(localAudio ? 1 : 0)+ diff --git a/app/streaming/session.cpp b/app/streaming/session.cpp index 2c06359a..b5abbc62 100644 --- a/app/streaming/session.cpp +++ b/app/streaming/session.cpp @@ -440,10 +440,20 @@ bool Session::populateDecoderProperties(SDL_Window* window) { IVideoDecoder* decoder; + int videoFormat; + if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_MAIN10) { + videoFormat = VIDEO_FORMAT_H265_MAIN10; + } + else if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265) { + videoFormat = VIDEO_FORMAT_H265; + } + else { + videoFormat = VIDEO_FORMAT_H264; + } + if (!chooseDecoder(m_Preferences->videoDecoderSelection, window, - m_StreamConfig.enableHdr ? VIDEO_FORMAT_H265_MAIN10 : - (m_StreamConfig.supportsHevc ? VIDEO_FORMAT_H265 : VIDEO_FORMAT_H264), + videoFormat, m_StreamConfig.width, m_StreamConfig.height, m_StreamConfig.fps, @@ -600,17 +610,22 @@ bool Session::initialize() "Audio channel mask: %X", CHANNEL_MASK_FROM_AUDIO_CONFIGURATION(m_StreamConfig.audioConfiguration)); + // H.264 is always supported + m_StreamConfig.supportedVideoFormats = VIDEO_FORMAT_H264; + switch (m_Preferences->videoCodecConfig) { case StreamingPreferences::VCC_AUTO: // TODO: Determine if HEVC is better depending on the decoder - m_StreamConfig.supportsHevc = - isHardwareDecodeAvailable(testWindow, - m_Preferences->videoDecoderSelection, - VIDEO_FORMAT_H265, - m_StreamConfig.width, - m_StreamConfig.height, - m_StreamConfig.fps); + if (isHardwareDecodeAvailable(testWindow, + m_Preferences->videoDecoderSelection, + VIDEO_FORMAT_H265, + m_StreamConfig.width, + m_StreamConfig.height, + m_StreamConfig.fps)) { + m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265; + } + #ifdef Q_OS_DARWIN { // Prior to GFE 3.11, GFE did not allow us to constrain @@ -623,23 +638,18 @@ bool Session::initialize() (gfeVersion[0] == 3 && gfeVersion[1] < 11)) { SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, "Disabling HEVC on macOS due to old GFE version"); - m_StreamConfig.supportsHevc = false; + m_StreamConfig.supportedVideoFormats &= ~VIDEO_FORMAT_H265; } } #endif - m_StreamConfig.enableHdr = false; break; case StreamingPreferences::VCC_FORCE_H264: - m_StreamConfig.supportsHevc = false; - m_StreamConfig.enableHdr = false; break; case StreamingPreferences::VCC_FORCE_HEVC: - m_StreamConfig.supportsHevc = true; - m_StreamConfig.enableHdr = false; + m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265; break; case StreamingPreferences::VCC_FORCE_HEVC_HDR: - m_StreamConfig.supportsHevc = true; - m_StreamConfig.enableHdr = true; + m_StreamConfig.supportedVideoFormats |= VIDEO_FORMAT_H265 | VIDEO_FORMAT_H265_MAIN10; break; } @@ -723,7 +733,7 @@ bool Session::validateLaunch(SDL_Window* testWindow) if (m_Preferences->videoDecoderSelection == StreamingPreferences::VDS_FORCE_SOFTWARE) { if (m_Preferences->videoCodecConfig == StreamingPreferences::VCC_FORCE_HEVC_HDR) { emitLaunchWarning(tr("HDR is not supported with software decoding.")); - m_StreamConfig.enableHdr = false; + m_StreamConfig.supportedVideoFormats &= ~VIDEO_FORMAT_MASK_10BIT; } else { emitLaunchWarning(tr("Your settings selection to force software decoding may cause poor streaming performance.")); @@ -738,7 +748,7 @@ bool Session::validateLaunch(SDL_Window* testWindow) } } - if (m_StreamConfig.supportsHevc) { + if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_H265) { bool hevcForced = m_Preferences->videoCodecConfig == StreamingPreferences::VCC_FORCE_HEVC || m_Preferences->videoCodecConfig == StreamingPreferences::VCC_FORCE_HEVC_HDR; @@ -750,7 +760,7 @@ bool Session::validateLaunch(SDL_Window* testWindow) // Moonlight-common-c will handle this case already, but we want // to set this explicitly here so we can do our hardware acceleration // check below. - m_StreamConfig.supportsHevc = false; + m_StreamConfig.supportedVideoFormats &= ~VIDEO_FORMAT_MASK_H265; } else if (m_Preferences->videoDecoderSelection == StreamingPreferences::VDS_AUTO && // Force hardware decoding checked below hevcForced && // Auto VCC is already checked in initialize() @@ -764,7 +774,7 @@ bool Session::validateLaunch(SDL_Window* testWindow) } } - if (!m_StreamConfig.supportsHevc && + if (!(m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_H265) && m_Preferences->videoDecoderSelection == StreamingPreferences::VDS_AUTO && !isHardwareDecodeAvailable(testWindow, m_Preferences->videoDecoderSelection, @@ -792,13 +802,11 @@ bool Session::validateLaunch(SDL_Window* testWindow) } } - if (m_StreamConfig.enableHdr) { - // Turn HDR back off unless all criteria are met. - m_StreamConfig.enableHdr = false; - + if (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_10BIT) { // Check that the server GPU supports HDR - if (!(m_Computer->serverCodecModeSupport & 0x200)) { + if (!(m_Computer->serverCodecModeSupport & 0x2200)) { emitLaunchWarning(tr("Your host PC doesn't support HDR streaming.")); + m_StreamConfig.supportedVideoFormats &= ~VIDEO_FORMAT_MASK_10BIT; } else if (!isHardwareDecodeAvailable(testWindow, m_Preferences->videoDecoderSelection, @@ -807,12 +815,10 @@ bool Session::validateLaunch(SDL_Window* testWindow) m_StreamConfig.height, m_StreamConfig.fps)) { emitLaunchWarning(tr("This PC's GPU doesn't support HEVC Main10 decoding for HDR streaming.")); + m_StreamConfig.supportedVideoFormats &= ~VIDEO_FORMAT_MASK_10BIT; } else { // TODO: Also validate display capabilites - - // Validation successful so HDR is good to go - m_StreamConfig.enableHdr = true; } } @@ -862,17 +868,17 @@ bool Session::validateLaunch(SDL_Window* testWindow) emit displayLaunchError(tr("Your host PC's GPU doesn't support streaming video resolutions over 4K.")); return false; } - else if (!m_StreamConfig.supportsHevc) { - emit displayLaunchError(tr("Video resolutions over 4K are only supported by the HEVC codec.")); + else if ((m_StreamConfig.supportedVideoFormats & ~VIDEO_FORMAT_MASK_H264) == 0) { + emit displayLaunchError(tr("Video resolutions over 4K are not supported by the H.264 codec.")); return false; } } if (m_Preferences->videoDecoderSelection == StreamingPreferences::VDS_FORCE_HARDWARE && - !m_StreamConfig.enableHdr && // HEVC Main10 was already checked for hardware decode support above + !(m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_10BIT) && // HDR was already checked for hardware decode support above !isHardwareDecodeAvailable(testWindow, m_Preferences->videoDecoderSelection, - m_StreamConfig.supportsHevc ? VIDEO_FORMAT_H265 : VIDEO_FORMAT_H264, + (m_StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_H265) ? VIDEO_FORMAT_H265 : VIDEO_FORMAT_H264, m_StreamConfig.width, m_StreamConfig.height, m_StreamConfig.fps)) { @@ -1236,6 +1242,7 @@ bool Session::startConnectionAsync() SERVER_INFORMATION hostInfo; hostInfo.address = hostnameStr.data(); hostInfo.serverInfoAppVersion = siAppVersion.data(); + hostInfo.serverCodecModeSupport = m_Computer->serverCodecModeSupport; // Older GFE versions didn't have this field QByteArray siGfeVersion; diff --git a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp index b94783e6..23c7db65 100644 --- a/app/streaming/video/ffmpeg-renderers/d3d11va.cpp +++ b/app/streaming/video/ffmpeg-renderers/d3d11va.cpp @@ -988,6 +988,36 @@ bool D3D11VARenderer::checkDecoderSupport(IDXGIAdapter* adapter) } break; + case VIDEO_FORMAT_AV1_MAIN8: + if (FAILED(videoDevice->CheckVideoDecoderFormat(&D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0, DXGI_FORMAT_NV12, &supported))) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "GPU doesn't support AV1 decoding"); + videoDevice->Release(); + return false; + } + else if (!supported) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "GPU doesn't support AV1 decoding to NV12 format"); + videoDevice->Release(); + return false; + } + break; + + case VIDEO_FORMAT_AV1_MAIN10: + if (FAILED(videoDevice->CheckVideoDecoderFormat(&D3D11_DECODER_PROFILE_AV1_VLD_PROFILE0, DXGI_FORMAT_P010, &supported))) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "GPU doesn't support AV1 Main10 decoding"); + videoDevice->Release(); + return false; + } + else if (!supported) { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "GPU doesn't support AV1 Main10 decoding to P010 format"); + videoDevice->Release(); + return false; + } + break; + default: SDL_assert(false); videoDevice->Release(); diff --git a/app/streaming/video/ffmpeg-renderers/dxva2.cpp b/app/streaming/video/ffmpeg-renderers/dxva2.cpp index 5b941322..4d827fee 100644 --- a/app/streaming/video/ffmpeg-renderers/dxva2.cpp +++ b/app/streaming/video/ffmpeg-renderers/dxva2.cpp @@ -19,6 +19,7 @@ #include DEFINE_GUID(DXVADDI_Intel_ModeH264_E, 0x604F8E68,0x4951,0x4C54,0x88,0xFE,0xAB,0xD2,0x5C,0x15,0xB3,0xD6); +DEFINE_GUID(DXVA2_ModeAV1_VLD_Profile0,0xb8be4ccb,0xcf53,0x46ba,0x8d,0x59,0xd6,0xb8,0xa6,0xda,0x5d,0x2a); #define SAFE_COM_RELEASE(x) if (x) { (x)->Release(); } @@ -202,6 +203,12 @@ bool DXVA2Renderer::initializeDecoder() break; } } + else if (m_VideoFormat == VIDEO_FORMAT_AV1_MAIN8 || m_VideoFormat == VIDEO_FORMAT_AV1_MAIN10) { + if (IsEqualGUID(guids[i], DXVA2_ModeAV1_VLD_Profile0)) { + chosenDeviceGuid = guids[i]; + break; + } + } } CoTaskMemFree(guids); @@ -774,7 +781,7 @@ bool DXVA2Renderer::initialize(PDECODER_PARAMETERS params) // the screen in full-screen mode at 720p/1080p unless we use 32 pixel alignment. // This appears to work without issues on AMD and Nvidia GPUs too, so we will // do it unconditionally for now. - if (m_VideoFormat & VIDEO_FORMAT_MASK_H265) { + if (!(m_VideoFormat & VIDEO_FORMAT_MASK_H264)) { alignment = 32; } else { diff --git a/app/streaming/video/ffmpeg.cpp b/app/streaming/video/ffmpeg.cpp index 265ee9ab..d4c28aad 100644 --- a/app/streaming/video/ffmpeg.cpp +++ b/app/streaming/video/ffmpeg.cpp @@ -78,6 +78,9 @@ static const QList k_NonHwaccelHEVCCodecs = { {"hevc_omx", 0}, }; +static const QList k_NonHwaccelAV1Codecs = { +}; + bool FFmpegVideoDecoder::isHardwareAccelerated() { return m_HwDecodeCfg != nullptr || @@ -633,6 +636,19 @@ void FFmpegVideoDecoder::stringifyVideoStats(VIDEO_STATS& stats, char* output) } break; + case VIDEO_FORMAT_AV1_MAIN8: + codecString = "AV1"; + break; + + case VIDEO_FORMAT_AV1_MAIN10: + if (LiGetCurrentHostDisplayHdrMode()) { + codecString = "AV1 10-bit HDR"; + } + else { + codecString = "AV1 10-bit SDR"; + } + break; + default: SDL_assert(false); codecString = "UNKNOWN"; @@ -996,6 +1012,23 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params) } } } + { + QString av1DecoderHint = qgetenv("AV1_DECODER_HINT"); + if (!av1DecoderHint.isEmpty() && (params->videoFormat & VIDEO_FORMAT_MASK_AV1)) { + QByteArray decoderString = av1DecoderHint.toLocal8Bit(); + if (tryInitializeRendererForDecoderByName(decoderString.constData(), params)) { + SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION, + "Using custom AV1 decoder (AV1_DECODER_HINT): %s", + decoderString.constData()); + return true; + } + else { + SDL_LogError(SDL_LOG_CATEGORY_APPLICATION, + "Custom AV1 decoder (AV1_DECODER_HINT) failed to load: %s", + decoderString.constData()); + } + } + } const AVCodec* decoder; @@ -1005,6 +1038,9 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params) else if (params->videoFormat & VIDEO_FORMAT_MASK_H265) { decoder = avcodec_find_decoder(AV_CODEC_ID_HEVC); } + else if (params->videoFormat & VIDEO_FORMAT_MASK_AV1) { + decoder = avcodec_find_decoder(AV_CODEC_ID_AV1); + } else { Q_ASSERT(false); decoder = nullptr; @@ -1042,13 +1078,23 @@ bool FFmpegVideoDecoder::initialize(PDECODER_PARAMETERS params) } } } - else { + else if (params->videoFormat & VIDEO_FORMAT_MASK_H265) { for (const codec_info_t& codecInfo : k_NonHwaccelHEVCCodecs) { if (tryInitializeRendererForDecoderByName(codecInfo.codec, params)) { return true; } } } + else if (params->videoFormat & VIDEO_FORMAT_MASK_AV1) { + for (const codec_info_t& codecInfo : k_NonHwaccelAV1Codecs) { + if (tryInitializeRendererForDecoderByName(codecInfo.codec, params)) { + return true; + } + } + } + else { + Q_ASSERT(false); + } // Look for the first matching hwaccel hardware decoder (pass 1) // This picks up "second-tier" hwaccels like CUDA. diff --git a/moonlight-common-c/moonlight-common-c b/moonlight-common-c/moonlight-common-c index c0792168..9ad56cdd 160000 --- a/moonlight-common-c/moonlight-common-c +++ b/moonlight-common-c/moonlight-common-c @@ -1 +1 @@ -Subproject commit c0792168f5a7ed48fc6feeb7fce01b83df405df2 +Subproject commit 9ad56cdd8e3db91381ef30cb828b310a545cad57