Fix a bunch of memory leaks in the VTRenderer

It appears the leaked AVSampleBufferDisplayLayer may be causing the hang in #973
This commit is contained in:
Cameron Gutman 2023-09-11 20:49:35 -05:00
parent b968886594
commit 2ac7176787

View file

@ -56,7 +56,7 @@ public:
} }
virtual ~VTRenderer() override virtual ~VTRenderer() override
{ { @autoreleasepool {
// We may have overlay update blocks enqueued for execution. // We may have overlay update blocks enqueued for execution.
// We must cancel those to avoid a UAF. // We must cancel those to avoid a UAF.
for (int i = 0; i < Overlay::OverlayMax; i++) { for (int i = 0; i < Overlay::OverlayMax; i++) {
@ -100,13 +100,19 @@ public:
for (int i = 0; i < Overlay::OverlayMax; i++) { for (int i = 0; i < Overlay::OverlayMax; i++) {
if (m_OverlayTextFields[i] != nullptr) { if (m_OverlayTextFields[i] != nullptr) {
[m_OverlayTextFields[i] removeFromSuperview]; [m_OverlayTextFields[i] removeFromSuperview];
[m_OverlayTextFields[i] release];
} }
} }
if (m_StreamView != nullptr) { if (m_StreamView != nullptr) {
[m_StreamView removeFromSuperview]; [m_StreamView removeFromSuperview];
[m_StreamView release];
} }
}
if (m_DisplayLayer != nullptr) {
[m_DisplayLayer release];
}
}}
static static
CVReturn CVReturn
@ -252,7 +258,7 @@ public:
// Caller frees frame after we return // Caller frees frame after we return
virtual void renderFrame(AVFrame* frame) override virtual void renderFrame(AVFrame* frame) override
{ { @autoreleasepool {
OSStatus status; OSStatus status;
CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]); CVPixelBufferRef pixBuf = reinterpret_cast<CVPixelBufferRef>(frame->data[3]);
@ -352,10 +358,10 @@ public:
[m_DisplayLayer enqueueSampleBuffer:sampleBuffer]; [m_DisplayLayer enqueueSampleBuffer:sampleBuffer];
CFRelease(sampleBuffer); CFRelease(sampleBuffer);
} }}
virtual bool initialize(PDECODER_PARAMETERS params) override virtual bool initialize(PDECODER_PARAMETERS params) override
{ { @autoreleasepool {
int err; int err;
if (params->videoFormat & VIDEO_FORMAT_MASK_H264) { if (params->videoFormat & VIDEO_FORMAT_MASK_H264) {
@ -549,10 +555,10 @@ public:
} }
return true; return true;
} }}
void updateOverlayOnMainThread(Overlay::OverlayType type) void updateOverlayOnMainThread(Overlay::OverlayType type)
{ { @autoreleasepool {
// Lazy initialization for the overlay // Lazy initialization for the overlay
if (m_OverlayTextFields[type] == nullptr) { if (m_OverlayTextFields[type] == nullptr) {
m_OverlayTextFields[type] = [[NSTextField alloc] initWithFrame:m_StreamView.bounds]; m_OverlayTextFields[type] = [[NSTextField alloc] initWithFrame:m_StreamView.bounds];
@ -584,7 +590,7 @@ public:
// Unhide if it's enabled // Unhide if it's enabled
[m_OverlayTextFields[type] setHidden: !Session::get()->getOverlayManager().isOverlayEnabled(type)]; [m_OverlayTextFields[type] setHidden: !Session::get()->getOverlayManager().isOverlayEnabled(type)];
} }}
virtual void notifyOverlayUpdated(Overlay::OverlayType type) override virtual void notifyOverlayUpdated(Overlay::OverlayType type) override
{ {